diff --git a/.config/nextest.toml b/.config/nextest.toml index 6ef3821f0..79608e2ff 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -34,14 +34,15 @@ path = "junit.xml" [test-groups] live-validators = { max-threads = 6 } -# Live-package tuning: capped concurrency (via the group), a slow-timeout for -# validator startup, and retries to absorb RPC-timing flakiness. Scoped to the -# live packages so production unit tests keep retries = 0 and no slow-timeout. +# Live-package tuning: capped concurrency (via the group) and a slow-timeout for +# validator startup. Scoped to the live packages so production unit tests keep +# no slow-timeout. Retries are 0 everywhere: a flaky live test is a signal to +# fix, not to absorb. [[profile.default.overrides]] filter = 'package(e2e) | package(clientless)' test-group = "live-validators" slow-timeout = { period = "60s", terminate-after = "10" } -retries = 2 +retries = 0 # The canonical test-target inventory consumed by the update/validate-test-targets # scripts via `cargo nextest list` -- NOT the CI test run (that is `default`). diff --git a/.env.testing-artifacts b/.env.testing-artifacts index 9bb6238e8..e1e14d228 100644 --- a/.env.testing-artifacts +++ b/.env.testing-artifacts @@ -5,7 +5,7 @@ # zcashd Git tag (https://github.com/zcash/zcash/releases) ZCASH_VERSION=6.20.0 -ZEBRA_VERSION=5.2.0 +ZEBRA_VERSION=6.0.0 # zcash-devtool git ref (https://github.com/zingolabs/zcash-devtool) — # the wallet client driven by wallet-tests via zcash_local_net, built @@ -14,4 +14,6 @@ ZEBRA_VERSION=5.2.0 # (get-ci-image-tag.sh → resolve_devtool_rev), so a branch HEAD advancing # changes the tag automatically and the rebuild always bakes the current # binary — no manual SHA bump needed. Pin a tag here only to freeze it. -DEVTOOL_VERSION=add_regtest +# Pinned to the NU6.3/Ironwood support commit: tip of +# `ironwood-valar-portables`, under review as zcash/zcash-devtool#208. +DEVTOOL_VERSION=2efc8e9c690c6b4a844d6bb0a49bc66ba6fcbb14 diff --git a/.github/workflows/build-n-push-ci-image.yaml b/.github/workflows/build-n-push-ci-image.yaml index d837af914..e29b963f2 100644 --- a/.github/workflows/build-n-push-ci-image.yaml +++ b/.github/workflows/build-n-push-ci-image.yaml @@ -49,10 +49,15 @@ jobs: source .env.testing-artifacts set +a RUST_VERSION=$(cargo run -q --manifest-path tools/workbench/Cargo.toml --bin get-rust-version) + # Resolve ZEBRA_VERSION (the Docker Hub tag) to the git ref the + # zebra-builder source stage checks out; fails the job here on an + # unresolvable value instead of mid-build. + ZEBRA_GIT_REF=$(cargo run -q --manifest-path tools/workbench/Cargo.toml --bin get-zebra-git-ref -- "$ZEBRA_VERSION") echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" echo "RUST_VERSION=$RUST_VERSION" >> "$GITHUB_ENV" echo "ZCASH_VERSION=$ZCASH_VERSION" >> "$GITHUB_ENV" echo "ZEBRA_VERSION=$ZEBRA_VERSION" >> "$GITHUB_ENV" + echo "ZEBRA_GIT_REF=$ZEBRA_GIT_REF" >> "$GITHUB_ENV" echo "DEVTOOL_VERSION=$DEVTOOL_VERSION" >> "$GITHUB_ENV" - name: Define build target @@ -79,4 +84,5 @@ jobs: RUST_VERSION=${{ env.RUST_VERSION }} ZCASH_VERSION=${{ env.ZCASH_VERSION }} ZEBRA_VERSION=${{ env.ZEBRA_VERSION }} + ZEBRA_GIT_REF=${{ env.ZEBRA_GIT_REF }} DEVTOOL_VERSION=${{ env.DEVTOOL_VERSION }} diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml new file mode 100644 index 000000000..f2fb2e948 --- /dev/null +++ b/.github/workflows/publish-dry-run.yml @@ -0,0 +1,105 @@ +name: Publish dry-run + +on: + workflow_dispatch: + push: + branches: + - 'rc/**' + - stable + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + publish-dry-run: + name: cargo publish --dry-run + runs-on: ubuntu-latest + # Blocking context: direct pushes to rc/** or stable — the branch is in + # release-prep state and must publish cleanly. Advisory context: everything + # else, including PRs that *target* rc/** or stable. Those PRs carry + # unbumped dev code; version bumps happen on the RC branch after merge, so + # a publish failure at PR time is expected, not actionable. + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + # rust-toolchain.toml auto-installs the pinned toolchain on first cargo + # invocation, so no explicit toolchain setup step is needed. + + - name: Compute check mode + id: mode + run: | + if [[ "${{ github.event_name }}" == "push" ]]; then + target="${{ github.ref_name }}" + else + # PRs (including those targeting rc/** or stable) are always + # advisory — they carry pre-bump code by design. + target="" + fi + if [[ "$target" == rc/* || "$target" == stable ]]; then + echo "mode=blocking" >> "$GITHUB_OUTPUT" + else + echo "mode=advisory" >> "$GITHUB_OUTPUT" + fi + + # A publishable crate whose exact version is already on crates.io with + # different content cannot be released without a bump. Advisory mode + # warns (unbumped-but-changed is the normal bump-at-release state on + # feature branches); blocking mode fails. Also emits the `violations` + # output consumed by the dry-run step below. + - name: Check for version-reuse violations + id: version-check + run: | + cargo run --manifest-path tools/workbench/Cargo.toml \ + --bin check-published-versions -- --mode "${{ steps.mode.outputs.mode }}" + + # --workspace resolves sibling dependencies against a local overlay + # registry, so packaging succeeds even when members depend on + # unpublished sibling changes — emulating a real dependency-ordered + # release. A per-crate dry-run loop cannot do this: nothing is uploaded + # between iterations, so dependents resolve siblings from crates.io and + # any cross-crate change fails spuriously. + # + # The verify build is weaker than packaging: when a local crate reuses + # an already-published version number, the registry artifact shadows + # the overlay copy, so dependents verify-compile against the published + # content, not the tree's. While version-reuse violations exist + # (advisory contexts only — blocking contexts already failed above), + # verification is therefore meaningless and the dry-run degrades to + # packaging/resolution checks. Release prep bumps the versions, and the + # full verify then runs where this job gates. + - name: Dry-run publish (workspace overlay) + run: | + set -euo pipefail + + verify_flag="" + if [[ "${{ steps.version-check.outputs.violations }}" == "true" ]]; then + verify_flag="--no-verify" + echo "Version-reuse violations present: packaged crates would verify-compile" + echo "against the already-published versions that shadow the workspace overlay." + echo "Running packaging and resolution checks only (--no-verify) until the" + echo "versions are bumped at release prep." + echo "" + fi + + if ! cargo publish --workspace --dry-run $verify_flag 2>&1; then + echo "" + echo "============================================" + echo "cargo publish --workspace --dry-run FAILED" + echo "============================================" + echo "" + echo "The workspace does not publish cleanly as a set. Common causes:" + echo " - a [patch.crates-io] override pulling in unreleased upstream" + echo " code (published crates cannot depend on unpublished revisions)" + echo " - a dependency on a git revision or path outside the workspace" + echo " - a crate that does not compile from its packaged form" + echo " - new cross-crate API or features that exist in the workspace but" + echo " not in the published dependency; resolved by the version bump +" + echo " publish of that dependency at the next release" + echo "" + if [[ "${{ steps.mode.outputs.mode }}" == "advisory" ]]; then + echo "::warning::publish dry-run failed — the workspace would not release cleanly. Advisory on this branch, but it must be resolved before the next release." + exit 0 + fi + echo "::error::publish dry-run failed — the workspace would not release cleanly." + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 621a927ce..e07073cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,46 @@ and this library adheres to Rust's notion of ## Unreleased ### Changed +- `zaino-state`: `FetchService` and `StateService` are merged into a single + generic `NodeBackedIndexerService` (module + `zaino_state::indexer::node_backed_indexer`; the former `backends` module is + gone). The validator connection is now selected at runtime rather than by type: + `NodeBackedIndexerServiceConfig { common, connection }` carries a + `ValidatorConnectionType` of either `Rpc` (JSON-RPC, formerly `Fetch`) or + `Direct(DirectConnectionConfig)` (Zebra `ReadStateService`, formerly `State`). + The per-backend `Fetch/StateServiceConfig`, `Fetch/StateServiceError`, and + `BackendConfig` types are replaced by `NodeBackedIndexerServiceConfig`, + `NodeBackedIndexerServiceError`, and `ValidatorConnectionType`. +- **Breaking** — config: `zainod.toml`'s `backend` selector is renamed + `state` → `direct` and `fetch` → `rpc`. The legacy `"state"` / `"fetch"` + values are still accepted as aliases, so existing config files keep working. +- `zaino-state`: the `ChainIndex` trait is split into `ChainIndex` (the + wallet-essential core: chain/tx/address/mempool access) and a + `ChainIndexRpcExt: ChainIndex` extension (compact-block serving, subtree + roots, and the block-explorer / mining / node-passthrough RPCs). The split is + a provisional first pass to be refined into finer capability traits later. +- `zaino-state`: all remaining backend-split RPC functionality has moved out of + the `FetchService` (`JsonRpSeeConnector`) and `StateService` + (`ReadStateService`) backends and into `BlockchainSource` / + `ChainIndex`. Both backends now resolve every fetch through their `ChainIndex` + indexer — building responses from Zaino's own indexed state where possible and + delegating to the `ValidatorConnector` (`BlockchainSource`) only for + validator-only or passthrough data. Validator connection/syncer spawning also + moves into `ValidatorConnector::spawn_fetch` / `spawn_state`, so each + service/subscriber now holds only `{ indexer, data, config }`. This readies + the two backends for their eventual merge into a single + `ValidatorBackedIndexerService`. No behaviour change. +- TLS: zaino now installs rustls's **aws-lc-rs** CryptoProvider as its + preferred process-level default (was ring) and enables rustls's + `prefer-post-quantum` feature, so the X25519MLKEM768 hybrid key exchange + leads zaino's outbound handshakes (ADR-0006). Installation remains + first-install-wins: an embedder that installs a provider before zaino + keeps its choice. + +### Deprecated +- Classical TLS key exchange (X25519, SECP256R1, SECP384R1) is deprecated: + still offered and accepted for wallet compatibility, slated for refusal + once major wallet stacks negotiate hybrid key exchange (ADR-0006). - **Breaking** — config: `storage.database.sync_write_batch_bytes` (bytes) is renamed to `sync_write_batch_size` and given in **GiB** (default raised from 4 GiB to 32 GiB); this budget now also bounds the txout-set accumulator diff --git a/CLAUDE.md b/CLAUDE.md index 3731d8d8b..a01a2863a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,15 @@ # Zaino AI Contributor Guidelines +## Tool selection + +Always prefer Rust-native tools in domains where they are designed to operate. +Dependency and manifest changes go through `cargo add` / `cargo remove` / +`cargo update`. Code navigation and refactors go through rust-analyzer (see +the LSP section below). Verification goes through `cargo check` / +`cargo clippy` / `cargo fmt` / `cargo nextest`. Do not reach for Python, sed, +or regex sweeps over Rust source or `Cargo.toml` when a Rust tool covers the +job. + ## Visibility: minimum required scope All items (functions, methods, structs, enums, fields, modules) MUST use the diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 000000000..6246105d7 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,98 @@ +# Zaino + +Zaino is a Zcash indexing service. This glossary pins down the canonical +terms for concepts where the team has picked one word among several. + +## Language + +### Release engineering + +**Publishable set**: +The workspace members released to crates.io as a unit — every member not +marked `publish = false`. Derived from workspace metadata, never from a +hard-coded list. +_Avoid_: crate list, publish list + +**Blocking context**: +A CI context in which release checks must pass: pushes to `rc/**` or +`stable`, and pull requests targeting them. +_Avoid_: strict mode, release mode + +**Advisory context**: +Any CI context that is not a blocking context. Release checks report +findings (warnings, annotations) but do not fail the build there. +_Avoid_: soft mode, informational mode + +**Version-reuse violation**: +A publishable crate whose exact version already exists on crates.io while +its packaged content differs. The tree cannot be released until that crate's +version is bumped. An unchanged crate keeping its published version is not a +violation. +_Avoid_: stale version, forgotten bump + +### Chains and networks + +**The Public Testnet**: +The public Zcash test network, and only that. Its regimes are +non-hermetic — state is shared with other participants, and an epoch +the public chain has left (e.g. pre-NU6.3 once NU6.3 activates there) +cannot be re-entered. Always this exact phrase in prose; identifiers +use `the_pub_testnet` and types use `PubTestnet`. +_Avoid_: bare "Testnet"/"testnet", and especially "testnet" for any +locally-launched chain, even one launched under a testnet network kind + +**Regtest net**: +A hermetic, locally-launched chain whose activation heights the +launcher chooses. Every hermetic local net is a regtest net, whatever +network-kind flag it runs under. +_Avoid_: local testnet, custom testnet + +### Pools and upgrades + +**Ironwood / Orchard (era naming)**: +Eras, fixtures, and predicates that speak of shielded pools are named by +pool — Orchard, Ironwood — and a name that mentions one pool pairs with +the other pool's name, never with the upgrade's. **NU6.3** names only the +network upgrade itself: activation heights, consensus branch ID, +consensus rules. +_Avoid_: mixing vocabularies in one name or one sibling set (e.g. an +`ORCHARD_ONLY_*` fixture whose sibling is `NU6_3_ACTIVE_*` — the sibling +is `IRONWOOD_ONLY_*`) + +**Cross-address restriction**: +The post-NU6.3 rule the Orchard Action circuit enforces: "(g_d, pk_d) +of the output note must equal (g_d, pk_d) of the spent note" — the +output note must carry the same expanded receiver (diversified base +g_d, diversified transmission key pk_d) as its spent note, so each +Orchard action is either change to the spent note's own address or a +withdrawal (positive value balance). Orchard-to-Orchard transfers to +any other address — including another address of the same wallet — are +prohibited. A companion transaction-level rule forbids new value +entering the pool. Source: + +_Avoid_: "exit-only" (overclaims — same-receiver change still lands in +the pool and its commitment tree still grows) + +### TLS and cryptography + +**Preferred CryptoProvider**: +The TLS cryptography provider zaino installs as the process-wide default +when no provider is installed yet. A preference, not a mandate: an +embedder (e.g. zallet) that installs a provider before zaino keeps its +choice, and zaino handshakes through it. +_Avoid_: "enforced provider" (implies zaino overrides an embedder's +already-installed provider; it never does) + +**Hybrid key exchange**: +A TLS 1.3 key-exchange group combining a classical curve with a +post-quantum KEM (key-encapsulation mechanism), e.g. X25519MLKEM768. +Secure if either component holds. Hybrids are the post-quantum +deployment vehicle, not a classical fallback. +_Avoid_: calling hybrids "classical" because they contain a classical +component + +**Classical key exchange**: +A key-exchange group with no post-quantum component (X25519, SECP256R1, +SECP384R1). Deprecated in zaino: still accepted for client +compatibility, slated for refusal once major wallet stacks negotiate +hybrids. diff --git a/Cargo.lock b/Cargo.lock index e7b1fdd13..5b75d7196 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,26 +306,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.13.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex 1.3.0", - "syn 2.0.118", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -339,25 +319,11 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.3", + "rustc-hash", "shlex 1.3.0", "syn 2.0.118", ] -[[package]] -name = "bip0039" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568b6890865156d9043af490d4c4081c385dd68ea10acd6ca15733d511e6b51c" -dependencies = [ - "hmac 0.12.1", - "pbkdf2", - "rand 0.8.6", - "sha2 0.10.9", - "unicode-normalization", - "zeroize", -] - [[package]] name = "bip32" version = "0.6.0-pre.1" @@ -365,7 +331,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143f5327f23168716be068f8e1014ba2ea16a6c91e8777bc8927da7b51e1df1f" dependencies = [ "bs58", - "hmac 0.13.0-pre.4", + "hmac", "rand_core 0.6.4", "ripemd 0.2.0-pre.4", "secp256k1", @@ -525,9 +491,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bzip2-sys" @@ -550,9 +516,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -703,14 +669,9 @@ name = "clientless" version = "0.2.0" dependencies = [ "anyhow", - "corez", "futures", "hex", - "serde_json", - "tempfile", "tokio", - "tower 0.4.13", - "tracing", "wire_serialized_transaction_test_data", "zaino-common", "zaino-fetch", @@ -719,10 +680,9 @@ dependencies = [ "zaino-testutils", "zainod", "zcash_local_net", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", - "zip32", + "zebra-chain", + "zebra-rpc", + "zebra-state", ] [[package]] @@ -920,18 +880,18 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -939,18 +899,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1235,9 +1195,9 @@ dependencies = [ "zainod", "zcash_local_net", "zcash_primitives", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-rpc", + "zebra-state", ] [[package]] @@ -1395,7 +1355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", - "rand 0.8.6", + "rand 0.8.7", "rustc-hex", "static_assertions", ] @@ -1638,11 +1598,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -1652,9 +1610,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -1725,7 +1685,7 @@ dependencies = [ "halo2_proofs", "lazy_static", "pasta_curves", - "rand 0.8.6", + "rand 0.8.7", "sinsemilla", "subtle", "uint 0.9.5", @@ -1860,15 +1820,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "hmac" version = "0.13.0-pre.4" @@ -1899,9 +1850,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1909,9 +1860,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -2270,15 +2221,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2349,11 +2291,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] @@ -2368,12 +2310,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" - [[package]] name = "jsonrpsee" version = "0.24.11" @@ -2402,8 +2338,8 @@ dependencies = [ "http-body-util", "jsonrpsee-types", "parking_lot", - "rand 0.8.6", - "rustc-hash 2.1.3", + "rand 0.8.7", + "rustc-hash", "serde", "serde_json", "thiserror 1.0.69", @@ -2510,12 +2446,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "left-right" version = "0.11.7" @@ -2560,14 +2490,13 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.16.0+8.10.0" +version = "0.17.3+10.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" dependencies = [ - "bindgen 0.69.5", + "bindgen", "bzip2-sys", "cc", - "glob", "libc", "libz-sys", "lz4-sys", @@ -2590,7 +2519,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f8ce05b56f3cbc65ec7d0908adb308ed91281e022f61c8c3a0c9388b5380b17" dependencies = [ - "bindgen 0.72.1", + "bindgen", "cc", "thiserror 2.0.18", "tracing", @@ -2708,9 +2637,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memuse" @@ -2760,7 +2689,7 @@ dependencies = [ "hashbrown 0.16.1", "metrics", "quanta", - "rand 0.9.4", + "rand 0.9.5", "rand_xoshiro", "rapidhash", "sketches-ddsketch", @@ -2789,9 +2718,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -2838,12 +2767,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549e471b99ccaf2f89101bec68f4d244457d5a95a9c3d0672e9564124397741d" -[[package]] -name = "nonempty" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2855,9 +2778,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -2982,9 +2905,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "orchard" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a54f8d29bfb1e76a9d4e868a1a08cce2e57dd2bdc66232982822ad3114b91ab3" +checksum = "cca2ede6a4a77bd729eed3be9303d98a08b217edc85190dcf4dbe004086d5a65" dependencies = [ "aes", "bitvec", @@ -3001,9 +2924,9 @@ dependencies = [ "incrementalmerkletree", "lazy_static", "memuse", - "nonempty 0.11.0", + "nonempty", "pasta_curves", - "rand 0.8.6", + "rand 0.8.7", "rand_core 0.6.4", "reddsa", "serde", @@ -3092,17 +3015,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "password-hash" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "pasta_curves" version = "0.5.1" @@ -3113,7 +3025,7 @@ dependencies = [ "ff", "group", "lazy_static", - "rand 0.8.6", + "rand 0.8.7", "static_assertions", "subtle", ] @@ -3124,16 +3036,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest 0.10.7", - "password-hash", -] - [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3359,7 +3261,7 @@ dependencies = [ "bit-vec", "bitflags 2.13.0", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -3523,7 +3425,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.3", + "rustc-hash", "rustls", "socket2", "thiserror 2.0.18", @@ -3534,17 +3436,17 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ - "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", - "rustc-hash 2.1.3", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3556,16 +3458,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3610,9 +3512,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3621,9 +3523,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3712,6 +3614,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -3732,9 +3643,9 @@ dependencies = [ [[package]] name = "rapidhash" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a224897b65b7ce38bf7b0adb569e7d77e11460d0cc3577908b263d8b5a89aa" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" dependencies = [ "rustversion", ] @@ -3842,9 +3753,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" dependencies = [ "aho-corasick", "memchr", @@ -3854,9 +3765,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" dependencies = [ "aho-corasick", "memchr", @@ -3928,7 +3839,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "quinn", "rustls", "rustls-pki-types", "rustls-platform-verifier", @@ -3988,9 +3898,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" dependencies = [ "libc", "librocksdb-sys", @@ -4004,15 +3914,9 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" [[package]] name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - -[[package]] -name = "rustc-hash" -version = "1.1.0" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc-hash" @@ -4050,9 +3954,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", @@ -4127,9 +4031,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "rusty-fork" @@ -4181,7 +4085,7 @@ dependencies = [ "jubjub", "lazy_static", "memuse", - "rand 0.8.6", + "rand 0.8.7", "rand_core 0.6.4", "redjubjub", "subtle", @@ -4435,9 +4339,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4555,9 +4459,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -4575,7 +4479,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.8.6", + "rand 0.8.7", "sha1", ] @@ -4603,9 +4507,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -4768,9 +4672,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -4817,9 +4721,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -5068,9 +4972,9 @@ dependencies = [ [[package]] name = "tower-batch-control" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e6cf52578f98b4da47335c26c4f883f7993b1a9b9d2f5420eb8dbfd5dd19a28" +checksum = "0a49cdaf2b00c1150c93a8dbbb63cae193573903a3c9a07b5d4d61251b9e32f0" dependencies = [ "futures", "futures-core", @@ -5085,9 +4989,9 @@ dependencies = [ [[package]] name = "tower-fallback" -version = "0.2.41" +version = "0.2.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434e19ee996ee5c6aa42f11463a355138452592e5c5b5b73b6f0f19534556af" +checksum = "700cd6732c9c9e2c77501cc663708b2229abc9e8fa27d3deae15126330a1ed87" dependencies = [ "futures-core", "pin-project", @@ -5287,15 +5191,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -5717,15 +5612,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -5759,30 +5645,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -5795,12 +5664,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -5813,12 +5676,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -5831,24 +5688,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -5861,12 +5706,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -5879,12 +5718,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -5897,12 +5730,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -5915,17 +5742,11 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ "memchr", ] @@ -6000,26 +5821,22 @@ dependencies = [ [[package]] name = "zaino-common" -version = "0.3.0" +version = "0.4.0" dependencies = [ - "hex", - "nu-ansi-term", + "rustls", "serde", - "thiserror 2.0.18", "time", "tracing", "tracing-subscriber", "tracing-tree", - "zebra-chain 10.1.0", - "zingo_common_components", + "zebra-chain", ] [[package]] name = "zaino-fetch" -version = "0.3.0" +version = "0.4.0" dependencies = [ "base64", - "byteorder", "derive_more", "hex", "http", @@ -6030,7 +5847,6 @@ dependencies = [ "reqwest 0.13.1", "serde", "serde_json", - "sha2 0.10.9", "thiserror 2.0.18", "tokio", "tonic", @@ -6039,26 +5855,26 @@ dependencies = [ "wire_serialized_transaction_test_data", "zaino-common", "zaino-proto", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", + "zebra-chain", + "zebra-rpc", ] [[package]] name = "zaino-proto" -version = "0.1.4" +version = "0.2.0" dependencies = [ "prost", "tonic", "tonic-prost", "tonic-prost-build", "which", - "zebra-chain 10.1.0", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-state", ] [[package]] name = "zaino-serve" -version = "0.4.0" +version = "0.5.0" dependencies = [ "futures", "jsonrpsee", @@ -6069,25 +5885,24 @@ dependencies = [ "tonic", "tower 0.4.13", "tracing", + "zaino-common", "zaino-fetch", "zaino-proto", "zaino-state", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", + "zebra-chain", + "zebra-rpc", ] [[package]] name = "zaino-state" -version = "0.4.1" +version = "0.5.0" dependencies = [ "arc-swap", "bitflags 2.13.0", "blake2", - "bs58", "chrono", "corez", "dashmap", - "derive_more", "futures", "hex", "incrementalmerkletree", @@ -6095,11 +5910,8 @@ dependencies = [ "lmdb", "lmdb-sys", "metrics", - "nonempty 0.12.0", - "once_cell", "primitive-types 0.14.0", "proptest", - "prost", "rand 0.10.2", "reqwest 0.13.1", "sapling-crypto", @@ -6125,9 +5937,9 @@ dependencies = [ "zcash_primitives", "zcash_protocol", "zcash_transparent", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-rpc", + "zebra-state", ] [[package]] @@ -6148,14 +5960,14 @@ dependencies = [ "zainod", "zcash_local_net", "zcash_protocol", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-rpc", + "zebra-state", ] [[package]] name = "zainod" -version = "0.5.1" +version = "0.6.0" dependencies = [ "clap", "config", @@ -6168,22 +5980,20 @@ dependencies = [ "tokio", "toml", "tracing", - "tracing-subscriber", "zaino-common", "zaino-fetch", "zaino-serve", "zaino-state", "zcash_address", "zcash_protocol", - "zebra-chain 10.1.0", - "zebra-state 9.0.1", + "zebra-state", ] [[package]] name = "zcash_address" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58342d0aaa8e2fa98849636f52800ac4bf020574c944c974742fc933db58cac2" +checksum = "5a854b28c07dba372f4410ea8ad62b4bf7d5c2bf8be32fc4b31bc0db6521a975" dependencies = [ "bech32", "bs58", @@ -6201,14 +6011,14 @@ checksum = "1440921903cdb86133fb9e2fe800be488015db2939a30bedb413078a1acb0306" dependencies = [ "corez", "hex", - "nonempty 0.11.0", + "nonempty", ] [[package]] name = "zcash_history" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fde17bf53792f9c756b313730da14880257d7661b5bfc69d0571c3a7c11a76d" +checksum = "69d2ed9a9fa7b466a7adedab1ad5a21f8330e4943756912fc776cf7f3beae32b" dependencies = [ "blake2b_simd", "byteorder", @@ -6217,9 +6027,9 @@ dependencies = [ [[package]] name = "zcash_keys" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbcdfbb5c8edb247439d72a397abaae9b7dd14a1c070e7e4fc3536924f9065f" +checksum = "36391ebfce7df2510c6564f79e608ed93f1c0692c6464e2397b248e06dae3912" dependencies = [ "bech32", "blake2b_simd", @@ -6229,7 +6039,7 @@ dependencies = [ "document-features", "group", "memuse", - "nonempty 0.11.0", + "nonempty", "orchard", "rand_core 0.6.4", "sapling-crypto", @@ -6245,31 +6055,27 @@ dependencies = [ [[package]] name = "zcash_local_net" -version = "0.6.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083#0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083" +version = "0.7.0" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" dependencies = [ - "getset", "hex", - "json", "reqwest 0.12.28", + "serde", "serde_json", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "tokio", "tracing", - "zcash_protocol", - "zebra-chain 9.0.0", - "zebra-node-services 7.0.0", - "zebra-rpc 9.0.0", - "zingo_common_components", + "zingo-consensus", "zingo_test_vectors", ] [[package]] name = "zcash_note_encryption" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77efec759c3798b6e4d829fcc762070d9b229b0f13338c40bf993b7b609c2272" +checksum = "e1cb1b9170c94370e3d66c5cc0877661db743337588b64de7711239eed462198" dependencies = [ "chacha20 0.9.1", "chacha20poly1305", @@ -6280,9 +6086,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69e07f5eb3f682a6467b4b08ee4956f1acd1e886d70b21c4766953b3a1beba2" +checksum = "bdbeccc05bfe63b6dee9e989c6ff5027421741562a4853c36504dd621f6a81b1" dependencies = [ "blake2b_simd", "block-buffer 0.11.0-rc.3", @@ -6295,7 +6101,7 @@ dependencies = [ "incrementalmerkletree", "jubjub", "memuse", - "nonempty 0.11.0", + "nonempty", "orchard", "rand_core 0.6.4", "redjubjub", @@ -6311,9 +6117,9 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3de6b0ca82e08a9d38b1121f87c5b180b5feac19fecba074cb582882210d2371" +checksum = "b91fb0b420fb66a765f1fa92ab7a6b631aa54c08415415ef6185256826e74fbd" dependencies = [ "bellman", "blake2b_simd", @@ -6334,9 +6140,9 @@ dependencies = [ [[package]] name = "zcash_protocol" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bec496a0bd62dae98c4b26f51c5dab112d0c5350bbc2ccfdfd05bb3454f714d" +checksum = "2973d210e74ebd81adc50828fbbb284ab6aaa099308097c42c47dc63cf3420fc" dependencies = [ "corez", "document-features", @@ -6373,9 +6179,9 @@ dependencies = [ [[package]] name = "zcash_transparent" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15df1908b428d4edeb7c7caae5692e05e2e92e5c38007a40b20ac098efdffd96" +checksum = "72e0f8c142bed366ed5886dcfa8772ec90b5420cc127e6cdb76b6744c53a287b" dependencies = [ "bip32", "bs58", @@ -6383,7 +6189,7 @@ dependencies = [ "document-features", "getset", "hex", - "nonempty 0.11.0", + "nonempty", "ripemd 0.1.3", "secp256k1", "sha2 0.10.9", @@ -6398,73 +6204,9 @@ dependencies = [ [[package]] name = "zebra-chain" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ed3a4e00e8f0b2197f3d8fd6cad02351a25ddbffcf0a6dc1fe463d7ea4ccfa" -dependencies = [ - "bech32", - "bitflags 2.13.0", - "bitflags-serde-legacy", - "bitvec", - "blake2b_simd", - "blake2s_simd", - "bounded-vec", - "bs58", - "byteorder", - "chrono", - "derive-getters", - "dirs", - "ed25519-zebra", - "equihash", - "futures", - "group", - "halo2_proofs", - "hex", - "humantime", - "incrementalmerkletree", - "itertools 0.14.0", - "jubjub", - "lazy_static", - "num-integer", - "orchard", - "primitive-types 0.12.2", - "rand_core 0.6.4", - "rayon", - "reddsa", - "redjubjub", - "ripemd 0.1.3", - "sapling-crypto", - "schemars 1.2.1", - "secp256k1", - "serde", - "serde-big-array", - "serde_json", - "serde_with", - "sha2 0.10.9", - "sinsemilla", - "static_assertions", - "strum", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tracing", - "uint 0.10.0", - "x25519-dalek", - "zcash_address", - "zcash_encoding", - "zcash_history", - "zcash_note_encryption", - "zcash_primitives", - "zcash_protocol", - "zcash_script", - "zcash_transparent", -] - -[[package]] -name = "zebra-chain" -version = "10.1.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a32562497425712e17b78c42d73bb69560053205653bb6b6e3e1aed6217ec" +checksum = "014adde138ea00b1cf6c873aaf56aacb5aa0d55b8e5451a3e3b1b06c461bd107" dependencies = [ "bech32", "bitflags 2.13.0", @@ -6494,7 +6236,7 @@ dependencies = [ "primitive-types 0.12.2", "proptest", "proptest-derive", - "rand 0.8.6", + "rand 0.8.7", "rand_chacha 0.3.1", "rand_core 0.6.4", "rayon", @@ -6531,52 +6273,9 @@ dependencies = [ [[package]] name = "zebra-consensus" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15475adf8c271d03de99d18d622a8ae4c512c401e3e29da1f27a0ba62b22057c" -dependencies = [ - "bellman", - "blake2b_simd", - "bls12_381", - "chrono", - "derive-getters", - "futures", - "futures-util", - "halo2_proofs", - "jubjub", - "lazy_static", - "libzcash_script", - "metrics", - "mset", - "once_cell", - "orchard", - "rand 0.8.6", - "rayon", - "sapling-crypto", - "serde", - "thiserror 2.0.18", - "tokio", - "tower 0.4.13", - "tower-batch-control", - "tower-fallback", - "tracing", - "tracing-futures", - "zcash_primitives", - "zcash_proofs", - "zcash_protocol", - "zcash_script", - "zcash_transparent", - "zebra-chain 9.0.0", - "zebra-node-services 7.0.0", - "zebra-script 8.0.0", - "zebra-state 8.0.0", -] - -[[package]] -name = "zebra-consensus" -version = "9.0.1" +version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c5b49c3cfef4df307ed6f6e8b1bd8cf0baa475c6841620bc17f9e9f53f8e77" +checksum = "4481ced3223226db18e33b9fb2e8eab7f7cefeae1681f4e8d8d7e3c2285a5c4d" dependencies = [ "bellman", "blake2b_simd", @@ -6593,7 +6292,7 @@ dependencies = [ "mset", "once_cell", "orchard", - "rand 0.8.6", + "rand 0.8.7", "rayon", "sapling-crypto", "serde", @@ -6609,55 +6308,17 @@ dependencies = [ "zcash_protocol", "zcash_script", "zcash_transparent", - "zebra-chain 10.1.0", - "zebra-node-services 8.0.0", - "zebra-script 9.0.0", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-node-services", + "zebra-script", + "zebra-state", ] [[package]] name = "zebra-network" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ba4f7dcd795eb84a5a33f5c4f29df60dd4971be363d2cd98d5fba5ce0477f2" -dependencies = [ - "bitflags 2.13.0", - "byteorder", - "bytes", - "chrono", - "dirs", - "futures", - "hex", - "humantime-serde", - "indexmap 2.14.0", - "itertools 0.14.0", - "lazy_static", - "metrics", - "num-integer", - "ordered-map", - "pin-project", - "rand 0.8.6", - "rayon", - "regex", - "schemars 1.2.1", - "serde", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "tokio-util", - "tower 0.4.13", - "tracing", - "tracing-error", - "tracing-futures", - "zebra-chain 9.0.0", -] - -[[package]] -name = "zebra-network" -version = "9.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d957623f215fcc049ef1178efba186440ac17c05ceeb68edf8f2999ce7f6eca8" +checksum = "9a59dc9e9ac723c4c01d985a5ecf3ddd85c0f1681574457d52173c43263905f2" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -6674,7 +6335,7 @@ dependencies = [ "num-integer", "ordered-map", "pin-project", - "rand 0.8.6", + "rand 0.8.7", "rayon", "regex", "schemars 1.2.1", @@ -6688,30 +6349,14 @@ dependencies = [ "tracing", "tracing-error", "tracing-futures", - "zebra-chain 10.1.0", -] - -[[package]] -name = "zebra-node-services" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abeece0fb4503a1df00c5ab736754b55c702613968f622f6aa3c2f9842aed2b2" -dependencies = [ - "color-eyre", - "jsonrpsee-types", - "reqwest 0.12.28", - "serde", - "serde_json", - "tokio", - "tower 0.4.13", - "zebra-chain 9.0.0", + "zebra-chain", ] [[package]] name = "zebra-node-services" -version = "8.0.0" +version = "9.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799017e2ade4fd2169bd4e14ddcbe180b62332e0b8d2ec9745d3256d8482684d" +checksum = "006992c62b9c4742a13a306f2e7443646fa0b5b884aac63a6fba5b0be273f864" dependencies = [ "color-eyre", "jsonrpsee-types", @@ -6720,74 +6365,14 @@ dependencies = [ "serde_json", "tokio", "tower 0.4.13", - "zebra-chain 10.1.0", -] - -[[package]] -name = "zebra-rpc" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b460869e352c9f1b49a00ed9beaae04ca3e6498381c7189d4b90a79e51c260bd" -dependencies = [ - "base64", - "chrono", - "color-eyre", - "derive-getters", - "derive-new", - "futures", - "hex", - "http-body-util", - "hyper", - "indexmap 2.14.0", - "jsonrpsee", - "jsonrpsee-proc-macros", - "jsonrpsee-types", - "lazy_static", - "metrics", - "nix", - "openrpsee", - "orchard", - "phf", - "prost", - "rand 0.8.6", - "sapling-crypto", - "schemars 1.2.1", - "semver", - "serde", - "serde_json", - "serde_with", - "strum", - "strum_macros", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "tonic", - "tonic-prost", - "tonic-prost-build", - "tonic-reflection", - "tower 0.4.13", - "tracing", - "which", - "zcash_address", - "zcash_keys", - "zcash_primitives", - "zcash_proofs", - "zcash_protocol", - "zcash_script", - "zcash_transparent", - "zebra-chain 9.0.0", - "zebra-consensus 8.0.0", - "zebra-network 8.0.0", - "zebra-node-services 7.0.0", - "zebra-script 8.0.0", - "zebra-state 8.0.0", + "zebra-chain", ] [[package]] name = "zebra-rpc" -version = "10.0.1" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eedf51b44db5c6c8b21aad40e88746a858185b812680e210a7eb44189489cf9" +checksum = "4d071b177740cdc1107f90dd50fac67e1b440a84c8509ae1529b3609fb6f801d" dependencies = [ "base64", "chrono", @@ -6809,7 +6394,7 @@ dependencies = [ "orchard", "phf", "prost", - "rand 0.8.6", + "rand 0.8.7", "sapling-crypto", "schemars 1.2.1", "semver", @@ -6818,6 +6403,7 @@ dependencies = [ "serde_with", "strum", "strum_macros", + "subtle", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -6835,85 +6421,33 @@ dependencies = [ "zcash_protocol", "zcash_script", "zcash_transparent", - "zebra-chain 10.1.0", - "zebra-consensus 9.0.1", - "zebra-network 9.0.0", - "zebra-node-services 8.0.0", - "zebra-script 9.0.0", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-consensus", + "zebra-network", + "zebra-node-services", + "zebra-script", + "zebra-state", ] [[package]] name = "zebra-script" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fc38c19388671df8a77c767e279b135ffaecd5e7df6ed7cd096390c080b4a3" -dependencies = [ - "libzcash_script", - "rand 0.8.6", - "thiserror 2.0.18", - "zcash_primitives", - "zcash_script", - "zebra-chain 9.0.0", -] - -[[package]] -name = "zebra-script" -version = "9.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "851f8533916035873c1543945bc425d7b7f3b1daa68a1ba2ba700a0714e75a8d" +checksum = "f7c967badb1bb99573dba430948c98fbca940daf23915f2e9736e1a1d2a166f3" dependencies = [ "libzcash_script", - "rand 0.8.6", + "rand 0.8.7", "thiserror 2.0.18", "zcash_primitives", "zcash_script", - "zebra-chain 10.1.0", + "zebra-chain", ] [[package]] name = "zebra-state" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b351fde5047dce0d505c9dc26863ee818b7579e9cf8d8e44489deb9decfbb7" -dependencies = [ - "bincode", - "chrono", - "crossbeam-channel", - "derive-getters", - "derive-new", - "dirs", - "futures", - "hex", - "hex-literal", - "human_bytes", - "humantime-serde", - "indexmap 2.14.0", - "itertools 0.14.0", - "lazy_static", - "metrics", - "mset", - "rayon", - "regex", - "rlimit", - "rocksdb", - "sapling-crypto", - "semver", - "serde", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tower 0.4.13", - "tracing", - "zebra-chain 9.0.0", - "zebra-node-services 7.0.0", -] - -[[package]] -name = "zebra-state" -version = "9.0.1" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f75fbe6845c042d64608664f3c66550c6cd9ed09efbd71a3849c617c14da137" +checksum = "39295d344c1cc56ed725646cfc7bb7e16f8edae7518ea5a9e599665f979e9cf6" dependencies = [ "bincode", "chrono", @@ -6938,20 +6472,21 @@ dependencies = [ "sapling-crypto", "semver", "serde", + "serde-big-array", "tempfile", "thiserror 2.0.18", "tokio", "tower 0.4.13", "tracing", - "zebra-chain 10.1.0", - "zebra-node-services 8.0.0", + "zebra-chain", + "zebra-node-services", ] [[package]] name = "zebra-test" -version = "3.0.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd2b1e773cbe0e9377e6c73f7a45be4e33db3bd92738545e877131a054c60aec" +checksum = "93c3fbb37b81e2727172ae184582297ffa91016be660ddd594b4830b16d168c9" dependencies = [ "color-eyre", "futures", @@ -6964,7 +6499,7 @@ dependencies = [ "once_cell", "owo-colors", "proptest", - "rand 0.8.6", + "rand 0.8.7", "regex", "spandoc", "thiserror 2.0.18", @@ -6977,18 +6512,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -7070,21 +6605,14 @@ dependencies = [ ] [[package]] -name = "zingo_common_components" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3378e81bdef45a9659736a09deaef2b5e5cb3d1556031468debfca3a21d4fa76" -dependencies = [ - "hex", -] +name = "zingo-consensus" +version = "0.1.0" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083#0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083" -dependencies = [ - "bip0039", -] +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" [[package]] name = "zip32" @@ -7101,6 +6629,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index d1bc50f04..e49ef17fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,23 +38,18 @@ license = "Apache-2.0" [workspace.dependencies] -# Zingo - -zingo_common_components = "0.3.1" - # Librustzcash incrementalmerkletree = "0.8" -zcash_address = "0.12" -zcash_keys = "0.14" -zcash_protocol = "0.9" -zcash_primitives = "0.28" -zcash_transparent = "0.8" -zcash_client_backend = "0.23" +zcash_address = "0.13" +zcash_keys = "0.15" +zcash_protocol = "0.10" +zcash_primitives = "0.29" +zcash_transparent = "0.9" # Zebra -zebra-chain = "10.0" -zebra-state = "9.0" -zebra-rpc = "10.0" +zebra-chain = "11.1" +zebra-state = "10.1" +zebra-rpc = "11.1" # Runtime tokio = { version = "1.38", features = ["full"] } @@ -72,7 +67,6 @@ tracing-subscriber = { version = "0.3.20", features = [ "time", "json", ] } -tracing-futures = "0.2" tracing-tree = "0.4" nu-ansi-term = "0.50" time = { version = "0.3", features = ["macros", "formatting"] } @@ -82,29 +76,22 @@ http = "1.1" url = "2.5" reqwest = { version = "0.13", default-features = false, features = [ "cookies", - "rustls", + "rustls-no-provider", "webpki-roots", ] } tower = { version = "0.4", features = ["buffer", "util"] } tonic = { version = "0.14" } -tonic-build = "0.14" prost = "0.14" serde = "1.0" serde_json = "1.0" -jsonrpsee-core = "0.24" jsonrpsee-types = "0.24" jsonrpsee = { version = "0.24", features = ["server", "macros"] } -hyper = "1.6" # Hashmaps, channels, DBs indexmap = "2.2.6" -crossbeam-channel = "0.5" dashmap = "6.1" lmdb = "0.8" lmdb-sys = "0.8" - -# Async -async-stream = "0.3" futures = "0.3.30" # Metrics @@ -113,9 +100,7 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false, feat # Utility thiserror = "2.0" -lazy-regex = "3.3" once_cell = "1.20" -ctrlc = "3.4" chrono = "0.4" which = "8" whoami = "2.1" @@ -129,7 +114,6 @@ blake2 = "0.10" hex = "0.4" toml = "1.1" primitive-types = "0.14" -bs58 = "0.5" bitflags = "2.9" # Test @@ -138,38 +122,36 @@ tempfile = "3.2" # no longer need `default-features = false` to drop it: their default feature # set is already empty. Enable zcashd end-to-end with `--features zcashd_support`. zainod = { path = "packages/zainod" } -zaino-common = { path = "packages/zaino-common", version = "0.3.0" } +zaino-common = { path = "packages/zaino-common", version = "0.4.0" } # default-features = false so the default-on `zcashd_support` feature can be # dropped end-to-end via `--no-default-features`; each consumer re-enables it # by forwarding `zcashd_support` in its own `default`. See # docs/adr/0001-zcashd-support-feature-gate.md. -zaino-fetch = { path = "packages/zaino-fetch", version = "0.3.0", default-features = false } -zaino-proto = { path = "packages/zaino-proto", version = "0.1.4" } -zaino-state = { path = "packages/zaino-state", version = "0.4.1", default-features = false } -zaino-serve = { path = "packages/zaino-serve", version = "0.4.0", default-features = false } +zaino-fetch = { path = "packages/zaino-fetch", version = "0.4.0", default-features = false } +zaino-proto = { path = "packages/zaino-proto", version = "0.2.0" } +zaino-state = { path = "packages/zaino-state", version = "0.5.0", default-features = false } +zaino-serve = { path = "packages/zaino-serve", version = "0.5.0", default-features = false } zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). -# TEMPORARY (zingolabs/infrastructure#269): pinned to an exact commit on the -# `add_client_support` branch so every machine resolves the same code; restore -# the release tag once tagged upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083" } +# Pinned to an exact commit past zcash_local_net_v0.7.0: the v0.7.0 launcher's +# zebrad config writer silently drops NU6.3 activation heights +# (https://github.com/zingolabs/zaino/issues/1368). This rev additionally +# makes the Validator the sole compile-time source of activation-height +# truth (infrastructure ADR 0003, PR #278): wallet clients derive their +# heights via `WalletNetwork::from_validator`, and `ZainodConfig` carries a +# payload-free network kind. Restore a release tag once one is cut upstream. +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "85502cd0b5faf9308e590865dae51bf31fdd342e" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } -nonempty = "0.12.0" proptest = "~1.11" -zip32 = "0.2.1" - -# Patch for vulnerable dependency -slab = "0.4.11" anyhow = "1.0" arc-swap = "1.7.1" -cargo-lock = "10.1.0" derive_more = "2.0.1" -lazy_static = "1.5.0" [profile.test] opt-level = 3 debug = true +[patch.crates-io] diff --git a/Makefile.toml b/Makefile.toml index f38a60907..2bcaec624 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -93,7 +93,7 @@ script.main = 'source "./tools/scripts/push-image.sh"' [tasks.container-test] clear = true -description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [package|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." +description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [packages|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." # This task runs tests inside the container built from live-tests/test_environment. # The entrypoint.sh script in the container sets up test binaries (zcashd, zebrad, zcash-cli) # by creating symlinks from /home/container_user/artifacts to the expected live-tests/test_binaries/bins location. @@ -132,16 +132,16 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # =================================================================== # Developer-facing test front door: a single `makers test [SET]` task. # -# makers test packaged (the default) -# makers test package packages/* production-crate tests (no validator) +# makers test packages (the default) +# makers test packages packages/* production-crate tests (no validator) # makers test e2e the e2e live partition # makers test clientless the clientless live partition # makers test live both live partitions + combined summary -# makers test all package then live (everything) +# makers test all packages then live (everything) # -# SET names mirror the crate/dir names (`e2e`, `clientless`); `package`, -# `live`, and `all` are the groupings. `live` = all non-package tests; `all` -# = `package` + `live` (see live-tests/CONTEXT.md). Each set delegates to an +# SET names mirror the crate/dir names (`e2e`, `clientless`); `packages`, +# `live`, and `all` are the groupings. `live` = all non-packages tests; `all` +# = `packages` + `live` (see live-tests/CONTEXT.md). Each set delegates to an # existing engine rather than re-implementing a run. `--with-zcashd` (handled # by the engines) adds the zcashd-backed tests to any set. # =================================================================== @@ -150,42 +150,56 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # clear = true fully replaces cargo-make's built-in `test` task (which runs # `cargo test`); without it our `script` would merge with the builtin `command`. clear = true -description = "Run a test set: `makers test [package|e2e|clientless|live|all]` (default: package). Pass --with-zcashd for the zcashd-backed tests." +description = "Run a test set: `makers test [packages|e2e|clientless|live|all|ironwood]` (default: packages). Pass --with-zcashd for the zcashd-backed tests." script_runner = "bash" script = ''' set -uo pipefail usage() { - echo "usage: makers test [package|e2e|clientless|live|all] [-- nextest args]" >&2 - echo " package (default) packages/* tests, no live validator" >&2 + echo "usage: makers test [packages|e2e|clientless|live|all|ironwood] [-- nextest args]" >&2 + echo " packages (default) packages/* tests, no live validator" >&2 echo " e2e | clientless that single live partition" >&2 echo " live both live partitions + combined summary" >&2 - echo " all package then live (everything)" >&2 + echo " all packages then live (everything)" >&2 + echo " ironwood the ironwood tests of every set, packages then clientless then e2e" >&2 } # A non-flag first arg selects the set and must be a known name; a flag first -# arg (or none) keeps the default `package` and is forwarded to nextest. So -# `makers test --with-zcashd` runs package with the flag, while a mistyped set +# arg (or none) keeps the default `packages` and is forwarded to nextest. So +# `makers test --with-zcashd` runs packages with the flag, while a mistyped set # name errors instead of silently running the default. -set="package" +set="packages" if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then case "$1" in - package|e2e|clientless|live|all) set="$1"; shift ;; + packages|e2e|clientless|live|all|ironwood) set="$1"; shift ;; *) echo "makers test: unknown set '$1'" >&2; usage; exit 2 ;; esac fi case "$set" in - package) exec makers container-test "$@" ;; + packages) exec makers container-test "$@" ;; e2e) exec makers live-e2e "$@" ;; clientless) exec makers live-clientless "$@" ;; live) exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" ;; all) - # Run both even if the first fails, then fail if either did, so the - # omnibus reflects the whole suite. + # The summary runner's --all mode runs the packages set and both live + # partitions (each set runs even when an earlier one fails, and the exit + # code reflects any failure), so the combined table covers everything. + exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- --all "$@" ;; + ironwood) + # The one definition of the ironwood selection: the two dedicated test + # binaries, plus every test whose name carries the pool's name (the + # CONTEXT.md era-naming convention keeps new ironwood tests inside this + # net). The same expression serves every engine — a selector naming a + # binary a partition lacks simply matches nothing there. The engines are + # invoked directly rather than through the `live` summary runner, which + # forwards no nextest args. Run every set even if an earlier one fails, + # as in `all`. + ironwood_filter='binary(ironwood_activation) | binary(the_pub_testnet_ironwood_boundary) | test(ironwood)' rc=0 - makers container-test "$@" || rc=1 - cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" || rc=1 + makers container-test -E "$ironwood_filter" "$@" || rc=1 + makers live-clientless -E "$ironwood_filter" "$@" || rc=1 + makers live-e2e -E "$ironwood_filter" "$@" || rc=1 exit "$rc" ;; esac ''' diff --git a/README.md b/README.md index a72a9a495..0b8ca92f4 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ The test suites run inside a **podman** container via `makers` (cargo-make): ```sh makers test # packages/* tests that need no live validator (default) makers test live # both live partitions (clientless + e2e) + combined summary -makers test all # everything: package then live +makers test all # everything: packages then live ``` zcashd-backed tests are **off by default**; add `--with-zcashd` to include them diff --git a/docs/adr/0004-rename-integration-partition-to-clientless.md b/docs/adr/0004-rename-integration-partition-to-clientless.md index c98de87d4..cb80fbfa7 100644 --- a/docs/adr/0004-rename-integration-partition-to-clientless.md +++ b/docs/adr/0004-rename-integration-partition-to-clientless.md @@ -50,3 +50,6 @@ unchanged. "clientless" is no longer avoided. - ADR-0003's naming bullet and its three-front-door task surface are superseded; its two-crate split and the `live` umbrella decision still stand. + +> **Later note:** the `package` set was renamed `packages` (plural, matching the +> `packages/` dir); `makers test package` now errors. See `live-tests/CONTEXT.md`. diff --git a/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md b/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md new file mode 100644 index 000000000..13306503c --- /dev/null +++ b/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md @@ -0,0 +1,90 @@ +# aws-lc-rs is the preferred CryptoProvider; classical key exchange is deprecating + +## Status + +accepted (reverses the ring choice made on zingolabs/zaino#1366 before it +merged) + +## Context and decision + +Zaino installs a rustls `CryptoProvider` as the process-wide default at +its two TLS boundaries (`zaino-serve`'s gRPC server, `zaino-fetch`'s +jsonrpsee connector) via `zaino_common::crypto::ensure_default_crypto_provider` +(zingolabs/zaino#1360). The provider was initially pinned to **ring** for +two reasons: zebra used ring (shared attack surface across the deployed +stack), and ring avoided the `aws-lc-sys` + `cmake` build dependencies. + +Daira Emma argued for **aws-lc-rs** instead: post-quantum TLS key +exchange in the rustls ecosystem requires the aws-lc-rs provider — the +ring provider has no ML-KEM support at all — and the provider +architecture makes the swap cheap +(, +). + +We decide: + +1. **aws-lc-rs is zaino's preferred CryptoProvider** — a *preference, + not a mandate*: installation stays first-install-wins, so an embedder + (e.g. zallet) that installs a provider before zaino keeps its choice. + Enforcement lives in the feature graph (`rustls` `aws_lc_rs` feature, + tonic `tls-aws-lc`), not at runtime. +2. **rustls's `prefer-post-quantum` feature is enabled**, reordering the + provider's default key-exchange groups so the hybrid X25519MLKEM768 + leads (same membership; in rustls 0.23 the hybrid is in the default + set either way). Verified effect in rustls 0.23.41: on zaino's + *outbound* (client-role) TLS the hybrid share is offered upfront, and + rustls adds the hybrid's X25519 component as a free second key share, + so classical-only servers negotiate without a HelloRetryRequest — the + cost is ~1.2 KB of ClientHello, not a round trip. On the *server* + side rustls follows the client's group preference, so inbound + negotiation outcomes are unchanged by the feature; hybrid uptake on + inbound connections is entirely client-driven, which is why the + deprecation below is gated on client stacks. +3. **Classical key exchange (X25519, SECP256R1, SECP384R1) remains + accepted but is deprecated.** Refusing it today would break both + flagship wallets: zingolib's TLS stack is feature-pinned to ring (no + ML-KEM possible), and ZODL rides mobile platform TLS where hybrid + support is still rolling out. The refusal precondition — zingolib on + aws-lc-rs with hybrid preferred, and ZODL's Android/iOS platform + stacks negotiating X25519MLKEM768 — is tracked in a dedicated issue; + deprecation is documented in the changelog rather than signalled at + runtime. +4. **Ring stays in `Cargo.lock`, tolerated and fenced, until zebra drops + it.** `zebra-rpc` hard-enables `zebra-node-services/rpc-client`, + whose reqwest 0.12 `rustls-tls` feature compiles ring in — reqwest + 0.12 offers no aws-lc alternative, only `*-no-provider` variants. This + is dormant code, not a live handshake path: reqwest consults the + process-default provider first and falls back to ring only when none + is installed, so both zaino (which installs aws-lc-rs early) and + embedders (whose earlier install wins) preempt it. A CI lint pins + `cargo tree --invert ring` to exactly this one known path and fails on + drift in *either* direction: a new ring consumer is a regression to + investigate; the path vanishing means zebra dropped ring, the + allowlist gets deleted, and the guard tightens to "ring absent". + +## Considered options + +- **Stay on ring, aligned with zebra.** Rejected: forecloses post-quantum + key exchange entirely; the harvest-now-decrypt-later argument applies + to wallet↔zaino TLS traffic today. +- **Refuse classical key exchange (hybrid-only).** Rejected for now: + hard-locks out every wallet whose TLS stack lacks X25519MLKEM768, + which today is all of them. Revisit when the tracked precondition is + met. +- **Fork-pin zebra to purge ring from the lock immediately.** Rejected: + the fork tax (pin churn here and in the wallet-tests workspace) buys + only the removal of dormant code; downstream is already protected by + provider preemption. Instead we upstream the fix to zebra — + `zebra-node-services` switches reqwest to a `*-no-provider` feature + and zebrad installs its own default provider (the same pattern as + zaino's #1360 fix) — and pick it up at the next zebra bump. + +## Consequences + +- `aws-lc-sys` and `cmake` return to the build graph (both build images + already install cmake). +- Zaino diverges from zebra's provider until zebra moves; the CI lint + makes the eventual convergence loud instead of silent. +- TLS 1.2 remains available only as long as classical key exchange + does; the refusal that ends the classical deprecation also implies + TLS 1.3-only. diff --git a/docs/notes/ironwood-activation-plan.md b/docs/notes/ironwood-activation-plan.md new file mode 100644 index 000000000..abb2d3b97 --- /dev/null +++ b/docs/notes/ironwood-activation-plan.md @@ -0,0 +1,227 @@ +# Ironwood activation: test design and heights source-of-truth roadmap + +State capture, 2026-07-06. Design session artifacts live in this tree +(uncommitted at time of writing); coordination artifacts live in +`zingolabs/infrastructure` (PR #278 worktree). This note is the durable +record of the decisions and the facts they rest on. + +## Domain facts, with sources + +- **Ironwood is a new shielded pool** activated by NU6.3 + (). It shares the Action-circuit shape + with Orchard but is a distinct pool. +- **Public testnet activated NU6.3 at height 4,134,000, ~2026-07-04.** + Defined identically in zebra-chain 11.0.0 + (`parameters/constants.rs::activation_heights::testnet::NU6_3`) and + zcash_protocol 0.10.0-pre.0 (`consensus.rs`), consensus branch ID + `0x37a5165b` in both. Wallet stack and validator stack therefore flip eras + at the same height with no drift. **Mainnet has no NU6.3 height** in those + pins. (Gotcha: public explorers can mislabel testnet/mainnet heights — + verify against a synced node.) +- **ZIP 318 (zcash/zips PR #1317), "Orchard to Ironwood migration"**: wallet + best-practices for a two-phase scheduled migration. Normative anchor: "The + user MUST be able to migrate Orchard-pool funds to the Ironwood pool, and + MUST be informed that doing so is necessary to retain access to those + funds." The wallet-visible migration transaction is an Orchard spend with + Ironwood outputs. +- **Cross-address restriction** + (): + post-NU6.3 the Orchard Action circuit requires "(g_d, pk_d) of the output + note must equal (g_d, pk_d) of the spent note" — every Orchard action is + either change to the spent note's own address or a withdrawal (positive + value balance). Cross-address Orchard transfers are circuit-impossible, + including between two addresses of one wallet. A companion + transaction-level rule forbids new value entering the pool. + - Do **not** describe Orchard as "exit-only": same-receiver change still + lands in the pool, so the Orchard note-commitment tree keeps growing + after activation. There is **no frozen-finalRoot predicate**; the + correct chain-walk predicate is **Orchard pool value non-increasing from + the boundary** (observable via `valuePools`). +- **No Orchard TAZ is obtainable** (and the testnet pre-epoch is closed): + post-activation nothing new can enter Orchard, so pre-activation Orchard + notes exist only in wallets that held them before the flip. Zaino holds + none. Consequence: the migration cell is exercisable **only on hermetic + regtest transition chains** — permanently. + +## Language decisions (canonical in CONTEXT.md) + +- **Testnet** names only the public test network. Every hermetic local net + is a **regtest net**, whatever network-kind flag it runs under. +- **Pool names pair with pool names** (Orchard ↔ Ironwood); **NU6.3** names + only the upgrade itself (activation heights, branch ID, consensus rules). + Applied renames: `NU6_3_ACTIVE_ACTIVATION_HEIGHTS` → + `IRONWOOD_ONLY_ACTIVATION_HEIGHTS` (zaino-testutils), + `NU6_3_ACTIVE_HEIGHTS` → `IRONWOOD_ONLY_HEIGHTS` + (`proptest_blockgen.rs`). `NU6_3_TRANSITION_BOUNDARY` keeps the upgrade + vocabulary: an activation height is the upgrade's concept. + +## Design model: predicates × regimes + +A **regime** is a (network, era) pair; a **predicate** is a +wallet-observable claim whose truth is fixed per regime. The suite is the +truth table; tests instantiate predicates in every reachable regime. + +| predicate | Orchard era | Ironwood era | +|---|---|---| +| unified-address receipt lands in Orchard | true | false | +| unified-address receipt lands in Ironwood | false (pool inactive) | true | +| shielded-receiver coinbase pays the era pool | Orchard | Ironwood | +| Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a | true | +| `shield` deposits into the era pool | Orchard | Ironwood | +| anything can land in Orchard from outside | true | false (no-new-value rule) | +| Orchard pool value non-increasing | n/a | true (cross-address restriction) | + +Regime venues: + +- **Regtest, Ironwood-only** (`IRONWOOD_ONLY_ACTIVATION_HEIGHTS`, canonical + NU6.3-at-2): the existing devtool wallet tests (`send_to_ironwood`, + `shield_for_validator`, `receives_mining_reward`, `send_to_all` in + `live-tests/e2e/tests/devtool.rs`). +- **Regtest, Orchard-only** (`ORCHARD_ONLY_ACTIVATION_HEIGHTS`): clientless + / wire mining fixtures only (devtool wallet cannot run there today). +- **Regtest, transition** (`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`, + NU6.3 at `NU6_3_TRANSITION_BOUNDARY` = 6): coinbase-routing and era + composition covered clientless + (`clientless/tests/compact_block_consistency.rs`) and over the wire + (`e2e/tests/compact_block_wire.rs`); **wallet cells live in + `e2e/tests/ironwood_activation.rs`** (see below). +- **Public testnet**: observational boundary walk across height 4,134,000 — + durable forever, historical blocks don't expire; third-party ZIP 318 + migrations supply the Orchard-spend traffic zaino must index but cannot + author. Active cells post-flip only, Orchard-free: faucet TAZ → + transparent receipt → `shield` → Ironwood send/receive. Env-gated manual + runs; record txids/heights in a dated evidence log + (`docs/notes/ironwood-activation-evidence.md`, created on first run). + `ZEBRAD_TESTNET_CACHE_DIR` (zaino-testutils) anticipates the cached sync. + +## Landed artifacts (this tree) + +- **`live-tests/e2e/tests/ironwood_activation.rs`** — the transition-regime + wallet cells, full real bodies, gated + `#[should_panic(expected = "UnsupportedActivationHeights")]`: + - `unified_receipt_lands_in_orchard_before_boundary` + - `orchard_note_spends_to_ironwood_across_boundary` — the migration cell; + recipient unified address generated *after* activation; asserts the + faucet's Orchard balance strictly shrinks, guarding against devtool note + selection dodging the migration by spending the boundary-height Ironwood + coinbase instead (sound under the cross-address restriction: change + returns only to the spent note's receiver, so a genuine Orchard spend + nets sent-plus-fee out of the pool). + - Verified: both pass as should_panic in ~5 s each + (`cargo nextest run -p e2e -E 'binary(ironwood_activation)'`). + - Registered on one backend (FetchService) while known-red: the panic + fires before the backend is exercised; mirror the fetch/state matrix + when the cells go live. +- **`e2e::devtool::build_clients_at(port, &heights)`** — devtool wallets on + caller-chosen regtest heights; `build_clients` delegates with the + canonical set. +- CONTEXT.md glossary entries; the renames above. + +## The blocker, and the infra handoff + +`zcash_local_net`'s devtool client rejects every regtest heights set except +its canonical NU6.3-at-2 one at wallet launch +(`ClientError::UnsupportedActivationHeights`; the guard exists because +heights drift between wallet and validator produces "incorrect consensus +branch id" — see zaino#1368). The guard predates the current devtool, which +reads heights at `init` from the `--activation-heights` TOML the client +already writes. + +Handoff: **`~/src/zingolabs/infras/dev/zaino-ironwood-activation-infra-spec.md`** +(for zingolabs/infrastructure PR #278, branch `bump_to_NU6.3`; zaino pins +rev `0dc4a51f...`). Asks: lift the equality guard (canonical set stays the +default), retire the error variant (zaino's known-red tests key on its name +so the pin bump flips them loudly), keep the caller-responsibility doc +contract, plus an infra-side integration test proving era-correct scanning +and branch-ID selection from file heights. Known unknown flagged: devtool +note-selection policy across pools. + +Scope addition (confirmed): infra also narrows `ZainodConfig.network` from +`NetworkType` to a payload-free `NetworkKind` in the same release — the +heights payload there is dead code (discarded at +`network_type_to_string()`; the Indexer never received heights that way). +Validator configs keep full `NetworkType`: they configure the source of +truth. + +## Roadmap: source of truth first + +Current directive (supersedes "unblock the suite next"): **first** make the +Validator the single source of truth for activation heights, with +regression tests that keep that source unique, and fix zaino#1076. + +Invariant: *the Indexer learns activation heights from the Validator at +startup and accepts them from nowhere else; configs carry network kind +only.* + +Audit facts grounding the implementation: + +- zainod's TOML is already kind-only: `NetworkSerde` serializes + "Mainnet"/"Testnet"/"Regtest", and deserializing "Regtest" injects the + compiled-in `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` + (`zaino-common/src/config/network.rs`). The real second channel is the + library seam: `Network::Regtest(ActivationHeights)` constructed + programmatically (e.g. by `TestManager`) and consumed via + `config.get_network()` → `to_zebra_network()`. +- The validator channel already exists in zaino's types: + `GetBlockchainInfoResponse.upgrades` is parsed (zaino-fetch + `jsonrpsee/response.rs`) and consumed (`backends/fetch.rs`, + `latest_network_upgrade`). The work is deriving the runtime + `zebra_chain::parameters::Network` from that response at startup and + narrowing the config type to kind-only. +- Regression tests that maintain uniqueness: launch the validator at + non-canonical heights (the transition fixture) with a kind-only-configured + zainod and assert era-correct serving (fails if anyone re-couples zainod + to a compiled-in set); the narrowed config type itself is the compile-time + regression test; a serde test pins the TOML shape. +- Payoff: the `InvalidData("Block commitment could not be computed")` + config-drift failure class becomes structurally impossible — the same + invariant the infra guard lift expresses on the wallet side (wallets get + heights from a file matched to the validator; the indexer gets them from + the validator directly). +- **#1076 residue** (issue text predates the all-at-2 canonical set, which + already replaced the NU6.1-at-1000 sentinel): wire + `lockbox_disbursements` into zebrad-launching fixtures (plumbing landed + at pinned rev, infra#243); add one NU6.1 boundary-crossing test (same + transition-fixture pattern as the Ironwood suite); watch for + block-commitment regressions. + +## Sequencing + +1. Zaino: heights source-of-truth implementation + uniqueness regression + tests (**landed in this tree**: `regtest_network_from_upgrades` + + spawn-time adoption in the zaino-state backends, de-mirrored testutils, + unit tests beside the mapping, live regressions in + `clientless/tests/validator_heights.rs`); remaining from this item: the + config-type split (kind-only config type, heights only post-handshake) + and the #1076 residue (lockbox fixtures, NU6.1 boundary test). +2. Infra: `NetworkKind` narrowing plus the strict form of the wallet-side + change on `bump_to_NU6.3`. The standing directive (2026-07-06) is that + the Validator is the *only* source of truth, enforced at compile time: + the harness derives the wallet's `activation-heights.toml` from + `Validator::get_activation_heights()`, and no client API accepts + caller-supplied heights (the spec's original "lift the guard, accept + caller heights" shape is superseded — a caller heights parameter is + itself a second source). +3. Zaino pin bump: adapt to the narrowed `ZainodConfig`; the two + `ironwood_activation` tests flip red (their expected + `UnsupportedActivationHeights` panic disappears along with the error + variant); replace `e2e::devtool::build_clients_at(port, &heights)` — + which exists only to express the caller-supplied shape — with a + derivation-based builder, so a fixture's heights are typed in exactly + one place: the zebrad launch config. First live runs settle devtool + note selection and the compact form of Orchard-spend data. +4. Deferred cells: testnet observational walk + post-flip Ironwood active + cells (env-gated, evidence log); chain-walk cells for the cross-address + restriction (Orchard pool-value monotonicity, same-receiver-change-only + commitments); the Orchard spend-window/freeze sub-regime waits on a + consensus source. + +## Unrelated finding from the same session + +The red `Integration tests / RPC tests shard-2/3` statuses on zaino commits +(zcash/integration-tests runs) are an upstream zallet break, not zaino's: +every run since 2026-07-03 ~23:25 UTC fails `zcashd_key_import{,_db}.py` in +zallet's `migrate-zcashd-wallet` with `UNIQUE constraint failed: +addresses.cached_transparent_receiver_address`, immediately after zallet +main's librustzcash/NU6.3 dependency bumps. No zallet issue existed for it +as of 2026-07-06. diff --git a/docs/notes/pool-indexed-dispatch.md b/docs/notes/pool-indexed-dispatch.md new file mode 100644 index 000000000..e028655a5 --- /dev/null +++ b/docs/notes/pool-indexed-dispatch.md @@ -0,0 +1,73 @@ +# Follow-up: pool-indexed dispatch in the chain index + +Status: proposed (not started), tracked in +[#1367](https://github.com/zingolabs/zaino/issues/1367). Scope: `zaino-state` +chain index. Origin: the PR #1362 (Ironwood/NU6.3) review. + +## Problem + +Ironwood was added to the chain index by copy-paste: every place that handled +"sapling and orchard" gained a third clone. The review of that PR found six +bugs, and every one of them was a copy-paste artifact of exactly this shape: + +- a sapling-output skip reusing the orchard-refactor's spend-width variable; +- the ironwood JSON field reading the `"orchard"` key; +- an error message describing the ironwood pool as "active NU5"; +- per-pool activation gating re-implemented (and drifted) at three sites; +- per-pool tuples widened positionally, letting same-typed roots transpose + silently. + +The review's DRY pass consolidated the *implementations* (shared cursor walks, +`required_pool_root`/`optional_pool_root`, `extract_block_pool_lists`, +`BlockRowEntries`), so each per-pool behaviour now has one definition. What it +did not change is the *dispatch*: pools are still reached by field name +(`self.sapling`, `tx.ironwood()`, `treestate.orchard`), so adding the next +pool still means hand-editing every call site, and the compiler cannot point +at the sites that were missed. + +## Proposal + +Make `ShieldedPool` (already defined in `chain_index.rs`) the index for +per-pool data and behaviour: + +1. **Table dispatch**: `DbV1::pool_table(&self, ShieldedPool) -> lmdb::Database` + replacing direct field access in the pool read/write paths, plus a + `MissingRow` policy per pool (`ShieldedPool::missing_row_policy()` — dense + for sapling/orchard, sparse for ironwood and later pools). +2. **Activation dispatch**: `ShieldedPool::activation_upgrade() -> + NetworkUpgrade` (Sapling → Sapling, Orchard → Nu5, Ironwood → Nu6_3), + consumed by `required_pool_root`/`optional_pool_root` callers instead of + per-site `is_nu_active` triples. +3. **Per-transaction data dispatch**: a method on the indexed transaction type + returning the pool's compact data by `ShieldedPool` value, so + `extract_block_pool_lists` and the compact-block builders iterate pools + instead of naming them. +4. **Exhaustiveness as the safety net**: every per-pool `match` on + `ShieldedPool` (no wildcard arms). Adding the NU7 pool then fails + compilation at every site that needs a decision — the same mechanism that + surfaced the `PoolType::Shielded(ShieldedPool::Ironwood)` non-exhaustive + match errors in the devtool work. + +## Constraints and cautions + +- **Behaviour preserving.** No schema, wire, or semantics change; this is a + dispatch refactor over the already-consolidated helpers. +- **Type asymmetry is real**: sapling has spends+outputs and its own types; + orchard and ironwood share the Orchard compact types. The dispatch layer + must not force a premature unification of the per-pool value types — + associated types or per-pool `match` arms returning distinct types are + acceptable; a trait object is not required. +- **Overlap warning**: this touches the same table-handle layer as the paused + DB/wire orthogonalization work (branch `clean_block_id_ident_relation`). + Reconcile with that plan before starting; doing both independently will + conflict. +- The lightwalletd protocol side (proto field per pool) stays copy-per-pool by + nature of protobuf; this proposal covers the indexer internals only. + +## Suggested shape of the work + +One branch, roughly three commits: (1) `ShieldedPool` gains +`activation_upgrade` / `missing_row_policy` and the connector/finalised-state +call sites consume them; (2) `DbV1::pool_table` + pool-iterating read/write +paths; (3) per-transaction pool-data dispatch and compact-block builders. +Each commit behaviour-preserving with the full `zaino-state` suite green. diff --git a/docs/testing.md b/docs/testing.md index f5c6d8f10..9bfd24575 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,14 +26,14 @@ The `makers` tasks below build and run the test suites inside a **podman** container, so you don't need the validator binaries on your host `$PATH`. The container image is built or pulled automatically on first run. -The single front door is `makers test [SET]`, where `SET` defaults to `package`: +The single front door is `makers test [SET]`, where `SET` defaults to `packages`: -- `makers test` (or `makers test package`) — the **package** set: the +- `makers test` (or `makers test packages`) — the **packages** set: the `packages/*` production-crate tests that need no live validator. - `makers test e2e` / `makers test clientless` — one live partition. - `makers test live` — both live partitions (`clientless` then `e2e`) against a live validator, with a combined pass/fail summary. -- `makers test all` — the whole suite: package then live. +- `makers test all` — the whole suite: packages then live. (`container-test`, `live-clientless`, and `live-e2e` are the internal engines the `test` front door delegates to; invoke them directly only when you need to diff --git a/live-tests/CONTEXT.md b/live-tests/CONTEXT.md index d00e0f064..1bef4076e 100644 --- a/live-tests/CONTEXT.md +++ b/live-tests/CONTEXT.md @@ -40,15 +40,17 @@ owns shared fixtures and the feature-forwarding surface (`zcashd_support`, `transparent_address_history_experimental`). _Avoid_: test helpers, testlib. -**package (test set)**: +**packages (test set)**: The tests of the `packages/*` production crates — the workspace `default-members`, exercised by bare `cargo nextest run` and by `makers test` -(the default with no argument, equivalently `makers test package`). Named for +(the default with no argument, equivalently `makers test packages`). Named for where the crates live (`packages/`); the functional reason they run apart from the live partitions is that they need **no** live validator. They are *not* network-free, though: e.g. zaino-serve's gRPC `spawn` regression test binds a loopback socket and stands up a tonic server. The absent ingredient is a validator, not the network. -_Avoid_: offline (overclaims "no network" — these tests bind loopback sockets); -unit test (the set includes crate-level integration tests); container test -(names the run mechanism, which the live suite shares). +_Avoid_: package (singular — the set is plural, matching the `packages/` dir; +`makers test package` now errors with `unknown set`); offline (overclaims "no +network" — these tests bind loopback sockets); unit test (the set includes +crate-level integration tests); container test (names the run mechanism, which +the live suite shares). diff --git a/live-tests/clientless/Cargo.toml b/live-tests/clientless/Cargo.toml index b7af22cdb..c8945a81a 100644 --- a/live-tests/clientless/Cargo.toml +++ b/live-tests/clientless/Cargo.toml @@ -36,12 +36,18 @@ zaino-fetch = { workspace = true } zaino-testutils = { workspace = true } # The lib's z_validate helper calls `ZcashIndexer::z_validate_address`. zaino-state = { workspace = true, features = ["test_dependencies"] } -tracing.workspace = true [dev-dependencies] anyhow = { workspace = true } +# Enable the tractable seam depth (fast-test-seam) so the chain_cache tests exercise +# finalisation/eviction at small chain heights. Dev-dependency only — the feature +# must never reach a shipped build (see the zaino-state feature comment). +zaino-state = { workspace = true, features = ["fast-test-seam"] } # Test utility +# Not imported by any test source; present so the +# transparent_address_history_experimental feature can forward to +# zainod/transparent_address_history_experimental in the spawned daemon. zainod = { workspace = true } zaino-testutils = { workspace = true } zaino-fetch = { workspace = true } @@ -51,13 +57,8 @@ wire_serialized_transaction_test_data = { workspace = true } zebra-chain = { workspace = true } zebra-state = { workspace = true } zebra-rpc = { workspace = true } -zip32 = { workspace = true } -corez = { workspace = true } -serde_json = { workspace = true } futures = { workspace = true } -tempfile = { workspace = true } -tower = { workspace = true } hex = { workspace = true } # Runtime diff --git a/live-tests/clientless/src/lib.rs b/live-tests/clientless/src/lib.rs index 9e7314757..12d93f9be 100644 --- a/live-tests/clientless/src/lib.rs +++ b/live-tests/clientless/src/lib.rs @@ -83,7 +83,17 @@ pub mod rpc { // assert_eq!(fs_sprout, expected_sprout); - // Sapling (differs by validator) + // Sapling + let expected_sapling = ValidZValidateAddress::sapling( + VALID_SAPLING_ADDRESS.to_string(), + Some(VALID_DIVERSIFIER.to_string()), + Some(VALID_DIVERSIFIED_TRANSMISSION_KEY.to_string()), + ); + assert_known_valid_eq( + rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, + expected_sapling, + "Sapling", + ); // Unified (differs by validator) let expected_unified = @@ -101,70 +111,15 @@ pub mod rpc { assert_eq!(all_zeroes, ZValidateAddressResponse::invalid()); } - pub async fn run_z_validate_sapling(rpc_call: &F) - where - F: Fn(String) -> Fut, - Fut: Future, - { - let expected_sapling = ValidZValidateAddress::sapling( - VALID_SAPLING_ADDRESS.to_string(), - Some(VALID_DIVERSIFIER.to_string()), - Some(VALID_DIVERSIFIED_TRANSMISSION_KEY.to_string()), - ); - assert_known_valid_eq( - rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, - expected_sapling, - "Sapling", - ); - } - - /// zebrad's JSON-RPC passthrough (via FetchService) omits `diversifier` - /// and `diversifiedtransmissionkey` from the Sapling response. This is - /// the safer behavior: address component extraction should happen - /// client-side, not by delegating to a remote actor. - /// - /// See [`DEPRECATION_NOTICE`](zaino_fetch::jsonrpsee::response::z_validate_address::DEPRECATION_NOTICE). - pub async fn run_z_validate_sapling_zebrad_passthrough_fetchservice(rpc_call: &F) - where - F: Fn(String) -> Fut, - Fut: Future, - { - let expected_sapling = ValidZValidateAddress::sapling( - VALID_SAPLING_ADDRESS.to_string(), - None::, - None::, - ); - assert_known_valid_eq( - rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, - expected_sapling, - "Sapling (zebrad passthrough via FetchService — keys omitted)", - ); - } - - /// Which sapling suite to run after the shared suite. - pub enum SaplingSuite { - /// Full response — diversifier and diversifiedtransmissionkey present. - Standard, - /// zebrad's JSON-RPC passthrough (via FetchService) omits those keys. - ZebradPassthroughFetchService, - } - /// Build the `z_validate_address` rpc-call closure from `subscriber` and - /// run the shared validation suite plus the chosen sapling suite. Factors - /// the identical closure + suite-call preamble shared by the four - /// `z_validate_address` tests (fetch_service zcashd/zebrad, state_service, - /// json_server). + /// run the shared validation suite. Factors the identical closure + + /// suite-call preamble shared by the four `z_validate_address` tests + /// (fetch_service zcashd/zebrad, state_service, json_server). #[allow(deprecated)] - pub async fn run_z_validate_for(subscriber: &S, sapling: SaplingSuite) { + pub async fn run_z_validate_for(subscriber: &S) { let rpc_call = |addr: String| async move { subscriber.z_validate_address(addr).await.unwrap() }; run_z_validate_suite(&rpc_call).await; - match sapling { - SaplingSuite::Standard => run_z_validate_sapling(&rpc_call).await, - SaplingSuite::ZebradPassthroughFetchService => { - run_z_validate_sapling_zebrad_passthrough_fetchservice(&rpc_call).await - } - } } } } diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index 8f724bd3c..eb173f2e0 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -1,24 +1,20 @@ use zaino_common::network::ActivationHeights; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; -use zaino_state::{ZcashIndexer, ZcashService}; -use zaino_testutils::{TestManager, ValidatorExt, ValidatorKind}; -use zainodlib::error::IndexerError; +use zaino_testutils::{Direct, Rpc, TestManager, ValidatorExt, ValidatorKind}; #[allow(deprecated)] -async fn create_test_manager_and_connector( +async fn create_test_manager_and_connector( validator: &ValidatorKind, activation_heights: Option, chain_cache: Option, enable_zaino: bool, enable_clients: bool, -) -> (TestManager, JsonRpSeeConnector) +) -> (TestManager, JsonRpSeeConnector) where T: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch( + let test_manager = TestManager::::launch( validator, None, activation_heights, @@ -46,8 +42,11 @@ mod chain_query_interface { source::ValidatorConnector, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, ShieldedPool, }, - test_dependencies::{chain_index::ChainIndex, ChainIndexConfig}, - FetchService, Height, StateService, StateServiceConfig, ZcashService, + test_dependencies::{ + chain_index::{ChainIndex, ChainIndexRpcExt}, + ChainIndexConfig, + }, + Height, NodeBackedIndexerService, NodeBackedIndexerServiceConfig, ZcashService, }; #[cfg(feature = "zcashd_support")] use zcash_local_net::validator::zcashd::Zcashd; @@ -63,26 +62,24 @@ mod chain_query_interface { use super::*; #[allow(deprecated)] - async fn create_test_manager_and_chain_index( + async fn create_test_manager_and_chain_index( validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, enable_clients: bool, ephemeral: bool, ) -> ( - TestManager, + TestManager, JsonRpSeeConnector, - Option, + Option, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, ) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, json_service) = create_test_manager_and_connector::( + let (test_manager, json_service) = create_test_manager_and_connector::( validator, None, chain_cache.clone(), @@ -114,6 +111,7 @@ mod chain_query_interface { nu6: local_net_activation_heights.nu6(), nu6_1: local_net_activation_heights.nu6_1(), nu6_2: local_net_activation_heights.nu6_2(), + nu6_3: local_net_activation_heights.nu6_3(), nu7: local_net_activation_heights.nu7(), }, )) @@ -122,42 +120,43 @@ mod chain_query_interface { NetworkKind::Testnet => zebra_chain::parameters::Network::new_default_testnet(), NetworkKind::Mainnet => zebra_chain::parameters::Network::Mainnet, }; - // FIXME: when state service is integrated into chain index this initialization must change - let state_service = StateService::spawn(StateServiceConfig::new( - zebra_state::Config { - cache_dir: state_chain_cache_dir, - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - // todo: does this matter? - should_backup_non_finalized_state: true, - debug_skip_non_finalized_state_backup_task: false, - }, - test_manager.full_node_rpc_listen_address.to_string(), - test_manager.full_node_grpc_listen_address, - false, - None, - None, - None, - ServiceConfig::default(), - StorageConfig { - cache: CacheConfig::default(), - database: DatabaseConfig { - path: test_manager - .data_dir - .as_path() - .to_path_buf() - .join("state-service-zaino"), - ..Default::default() + // FIXME: when the direct connection is integrated into chain index this initialization must change + let state_service = + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_direct( + zebra_state::Config { + cache_dir: state_chain_cache_dir, + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + // todo: does this matter? + should_backup_non_finalized_state: true, + debug_skip_non_finalized_state_backup_task: false, }, - }, - false, - network.into(), - None, - )) - .await - .unwrap(); + test_manager.full_node_rpc_listen_address.to_string(), + test_manager.full_node_grpc_listen_address, + false, + None, + None, + None, + ServiceConfig::default(), + StorageConfig { + cache: CacheConfig::default(), + database: DatabaseConfig { + path: test_manager + .data_dir + .as_path() + .to_path_buf() + .join("state-service-zaino"), + ..Default::default() + }, + }, + false, + network.into(), + None, + )) + .await + .unwrap(); let config = ChainIndexConfig { storage: StorageConfig { database: DatabaseConfig { @@ -172,9 +171,12 @@ mod chain_query_interface { }, ephemeral, db_version: 1, - network: zaino_common::Network::Regtest(ActivationHeights::from( - test_manager.local_net.get_activation_heights().await, - )), + // This fixture derives its runtime network from the + // heights the harness launched the validator with. + network: zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ) + .to_regtest_network(), }; // **NOTE** The "fetch" backend is currently the backend used in the wild, and @@ -216,9 +218,12 @@ mod chain_query_interface { }, ephemeral, db_version: 1, - network: zaino_common::Network::Regtest( - test_manager.local_net.get_activation_heights().await.into(), - ), + // This fixture derives its runtime network from the + // heights the harness launched the validator with. + network: zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ) + .to_regtest_network(), }; let chain_index = NodeBackedChainIndex::new( ValidatorConnector::Fetch(json_service.clone()), @@ -237,25 +242,23 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_block_range_zebrad() { - get_block_range::(&ValidatorKind::Zebrad).await + get_block_range::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_block_range_zcashd() { - get_block_range::(&ValidatorKind::Zcashd).await + get_block_range::(&ValidatorKind::Zcashd).await } - async fn get_block_range(validator: &ValidatorKind) + async fn get_block_range(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -277,7 +280,7 @@ mod chain_query_interface { #[tokio::test(flavor = "multi_thread")] async fn ephemeral_serves_finalised_blocks_zebrad() { - ephemeral_serves_finalised_blocks::(&ValidatorKind::Zebrad).await + ephemeral_serves_finalised_blocks::(&ValidatorKind::Zebrad).await } /// Ephemeral mode on regtest: the chain index opens no persistent @@ -285,39 +288,41 @@ mod chain_query_interface { /// validator via the ephemeral passthrough. /// /// In ephemeral mode `db_height` is `0`, so the non-finalised cache retains - /// blocks down to `tip - MAX_NFS_DEPTH` (110). We therefore generate well - /// past that depth and query a height below `tip - 110`, so the reads are + /// blocks only down to `tip - MAX_NFS_DEPTH` (a small margin past the seam). We + /// therefore generate well past that and query a height below it, so the reads are /// genuinely served by the ephemeral *finalised* passthrough rather than the /// non-finalised cache. The test then: /// - fetches a finalised chain (indexed) block by height, re-fetches it by /// its hash, and asserts the two are identical; /// - streams compact blocks across the finalised / non-finalised boundary; /// - asserts nothing was persisted to disk. - async fn ephemeral_serves_finalised_blocks(validator: &ValidatorKind) + async fn ephemeral_serves_finalised_blocks(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use zaino_proto::proto::utils::PoolTypeFilter; let (test_manager, json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, true) + create_test_manager_and_chain_index::(validator, None, false, false, true) .await; - // Generate well past MAX_NFS_DEPTH (110) so low heights are evicted from - // the non-finalised cache and served by the ephemeral finalised passthrough. + // The finalised floor sits at `tip - seam`; the non-finalised cache retains a + // little past that. Generate well beyond it so low heights are evicted from the + // cache and served by the ephemeral finalised passthrough. `fast-test-seam` + // shrinks the seam to `FAST_TEST_MAX_NONFINALISED_DEPTH`, so a small chain suffices. + let seam = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; test_manager - .generate_blocks_and_wait_for_tip(150, &indexer) + .generate_blocks_and_wait_for_tip(seam + 50, &indexer) .await; let snapshot = indexer.snapshot_nonfinalized_state().await.unwrap(); let chain_height: u32 = json_service.get_blockchain_info().await.unwrap().blocks.0; - // `start_height` is below `tip - 110` (evicted from the NFS cache, served - // by the passthrough); `end_height` is above `tip - 100` (non-finalised). - let start_height: u32 = chain_height - 120; - let end_height: u32 = chain_height - 40; + // `start_height` is comfortably below the retention window → evicted from the NFS + // cache, served by the passthrough; `end_height` is above the finalised floor + // (`tip - seam`) → non-finalised. + let start_height: u32 = chain_height - (seam + 20); + let end_height: u32 = chain_height - seam / 2; let finalised_height = Height::try_from(start_height).unwrap(); // --- chain (indexed) block: fetch by height, then by its hash --- @@ -372,25 +377,23 @@ mod chain_query_interface { #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn sync_large_chain_zebrad() { - sync_large_chain::(&ValidatorKind::Zebrad).await + sync_large_chain::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn sync_large_chain_zcashd() { - sync_large_chain::(&ValidatorKind::Zcashd).await + sync_large_chain::(&ValidatorKind::Zcashd).await } - async fn sync_large_chain(validator: &ValidatorKind) + async fn sync_large_chain(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, json_service, option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -416,9 +419,11 @@ mod chain_query_interface { let snapshot = indexer.snapshot_nonfinalized_state().await.unwrap(); let chain_height = json_service.get_blockchain_info().await.unwrap().blocks.0; - let finalised_start = Height::try_from(chain_height - 150).unwrap(); - let finalised_tip = Height::try_from(chain_height - 100).unwrap(); - let end = Height::try_from(chain_height - 50).unwrap(); + // Finalised floor is `tip - seam`; pick a range straddling it. + let seam = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; + let finalised_start = Height::try_from(chain_height - (seam + 50)).unwrap(); + let finalised_tip = Height::try_from(chain_height - seam).unwrap(); + let end = Height::try_from(chain_height - seam / 2).unwrap(); let finalized_blocks = indexer .get_block_range(&snapshot, finalised_start, Some(finalised_tip)) @@ -448,25 +453,23 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_subtree_roots_zebrad() { - get_subtree_roots::(&ValidatorKind::Zebrad).await + get_subtree_roots::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_subtree_roots_zcashd() { - get_subtree_roots::(&ValidatorKind::Zcashd).await + get_subtree_roots::(&ValidatorKind::Zcashd).await } - async fn get_subtree_roots(validator: &ValidatorKind) + async fn get_subtree_roots(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -553,30 +556,26 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream_fresh_snapshot_repeated_zebrad() { - get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zebrad) - .await + get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream_fresh_snapshot_repeated_zcashd() { - get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zcashd) - .await + get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zcashd).await } - async fn get_mempool_stream_fresh_snapshot_repeated(validator: &ValidatorKind) + async fn get_mempool_stream_fresh_snapshot_repeated(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; use tokio::time::{timeout, Duration}; let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -612,28 +611,26 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn zallet_like_steady_state_loop_zebrad() { - zallet_like_steady_state_loop::(&ValidatorKind::Zebrad).await + zallet_like_steady_state_loop::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn zallet_like_steady_state_loop_zcashd() { - zallet_like_steady_state_loop::(&ValidatorKind::Zcashd).await + zallet_like_steady_state_loop::(&ValidatorKind::Zcashd).await } - async fn zallet_like_steady_state_loop(validator: &ValidatorKind) + async fn zallet_like_steady_state_loop(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::{StreamExt as _, TryStreamExt as _}; use tokio::time::{timeout, Duration}; let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs new file mode 100644 index 000000000..18bf0116a --- /dev/null +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -0,0 +1,361 @@ +//! Per-block consistency between served compact-block content and its chain metadata. +//! +//! A compact block's `chainMetadata` commitment-tree sizes are cumulative counts of the +//! note commitments the chain has produced. A scanning wallet advances its trees by the +//! actions/outputs each served block carries, so whenever a served block's tree-size +//! delta disagrees with its served commitment count the wallet observes a tree-size +//! discontinuity and treats it as a chain reorg. This walk pins that invariant per +//! block, per pool, for the request shape real (including pre-Ironwood) light clients +//! send: an empty `poolTypes` filter. + +use zaino_common::network::ActivationHeights; +use zaino_fetch::jsonrpsee::response::GetBlockResponse; +#[allow(deprecated)] +use zaino_state::ZcashIndexer as _; +use zaino_testutils::{ + MinerPool, Rpc, TestManager, ValidatorKind, IRONWOOD_ONLY_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, +}; +use zcash_local_net::validator::zebrad::Zebrad; +use zebra_chain::serialization::ZcashDeserialize as _; + +/// multi_thread required: the test manager spawns the validator and indexer services. +#[allow(deprecated)] +#[tokio::test(flavor = "multi_thread")] +async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { + let mut test_manager = TestManager::::launch_mining_to( + // Shielded mining: from NU6.3 an orchard-receiver coinbase is built as Ironwood + // actions (the coinbase's Orchard component must be empty from NU6.3), so every + // generated block carries ironwood data for the walk to check. A transparent + // miner would leave the ironwood assertions vacuous. + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(IRONWOOD_ONLY_ACTIVATION_HEIGHTS), + None, + true, + false, + false, + ) + .await + .unwrap(); + let subscriber = test_manager.subscriber().clone(); + + test_manager + .generate_blocks_and_wait_for_tip(8, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.unwrap().0); + + // The empty pool filter is what unfiltered (pre-Ironwood) clients send; the served + // stream must include every shielded pool's actions. + let blocks = zaino_testutils::collect_block_range(&subscriber, 0, tip, vec![]).await; + assert!(!blocks.is_empty(), "no compact blocks served"); + + let connector = test_manager.full_node_jsonrpc_connector().await; + + let (mut prev_sapling, mut prev_orchard, mut prev_ironwood) = (0u32, 0u32, 0u32); + let mut total_orchard_actions = 0usize; + let mut total_ironwood_actions = 0usize; + for (index, block) in blocks.iter().enumerate() { + assert_eq!( + block.height, index as u64, + "served blocks must be contiguous from genesis for the walk's running totals" + ); + let metadata = block + .chain_metadata + .as_ref() + .expect("every served compact block carries chain metadata"); + + let sapling_outputs: u32 = block.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(); + let orchard_actions: u32 = block.vtx.iter().map(|tx| tx.actions.len() as u32).sum(); + let ironwood_actions: u32 = block + .vtx + .iter() + .map(|tx| tx.ironwood_actions.len() as u32) + .sum(); + total_orchard_actions += orchard_actions as usize; + total_ironwood_actions += ironwood_actions as usize; + + assert_eq!( + metadata.sapling_commitment_tree_size, + prev_sapling + sapling_outputs, + "sapling tree-size delta must equal the served output count at height {}", + block.height + ); + assert_eq!( + metadata.orchard_commitment_tree_size, + prev_orchard + orchard_actions, + "orchard tree-size delta must equal the served action count at height {}", + block.height + ); + // The regression this walk exists for: a served block whose metadata counts + // commitments from actions the block omits (e.g. ironwood stripped from an + // unfiltered request) reads to a scanning wallet as a phantom chain reorg. + assert_eq!( + metadata.ironwood_commitment_tree_size, + prev_ironwood + ironwood_actions, + "ironwood tree-size delta must equal the served action count at height {}", + block.height + ); + + // Clientless-exclusive predicate — oracle parity: zebrad's verbose getblock + // reports the validator's own per-block tree sizes, an independent + // implementation's answer to compare zaino's served metadata against. Package + // tests cannot express this: their "source of truth" is the object being + // served, so any such comparison is circular. + // Verbosity 1: txids-as-strings plus the `trees` field the oracle needs + // (verbosity 2 returns full transaction objects, which BlockObject's + // string-typed `tx` field rejects). + let oracle_trees = match connector + .get_block(block.height.to_string(), Some(1)) + .await + .expect("validator serves verbose blocks") + { + GetBlockResponse::Object(block_object) => block_object.trees, + other => panic!("verbosity-2 getblock must return a block object, got {other:?}"), + }; + assert_eq!( + u64::from(metadata.sapling_commitment_tree_size), + oracle_trees.sapling(), + "served sapling tree size must match the validator's own at height {}", + block.height + ); + assert_eq!( + u64::from(metadata.orchard_commitment_tree_size), + oracle_trees.orchard(), + "served orchard tree size must match the validator's own at height {}", + block.height + ); + assert_eq!( + u64::from(metadata.ironwood_commitment_tree_size), + oracle_trees.ironwood(), + "served ironwood tree size must match the validator's own at height {}", + block.height + ); + + prev_sapling = metadata.sapling_commitment_tree_size; + prev_orchard = metadata.orchard_commitment_tree_size; + prev_ironwood = metadata.ironwood_commitment_tree_size; + } + + assert!( + total_ironwood_actions > 0, + "the fixture produced no ironwood actions; the walk asserted nothing about ironwood" + ); + // The counterpart of the guard above: the miner *asked* for Orchard, and from NU6.3 + // consensus requires an empty Orchard coinbase component, routing the reward into + // Ironwood actions instead. With coinbase-only blocks, served orchard actions must + // therefore be exactly zero. Together the two totals distinguish failure modes: + // pool-swap (orchard > 0, ironwood == 0: ironwood served under the orchard field) + // vs pool-drop (both zero) vs a broken routing premise. + assert_eq!( + total_orchard_actions, 0, + "an Orchard-receiver coinbase must carry no Orchard actions from NU6.3" + ); + + test_manager.close().await; +} + +/// Class-1 (consensus) predicate: in the NU5-through-NU6.2 era, a shielded +/// (orchard-receiver) miner's coinbase carries the reward as Orchard actions. +/// +/// The action count uses `>= 1` rather than the padded exact count so the predicate +/// does not couple to the Orchard bundle-padding rule. +fn is_valid_orchard_coinbase(block: &zebra_chain::block::Block) -> bool { + let Some(coinbase) = block.transactions.first() else { + return false; + }; + coinbase.is_coinbase() + && coinbase.version() == 5 + && coinbase.sapling_outputs().count() == 0 + && coinbase.orchard_actions().count() >= 1 + && coinbase.ironwood_actions().count() == 0 +} + +/// Class-1 (consensus) predicate: from NU6.3 the same miner's coinbase must have an +/// empty Orchard component, with the reward routed to Ironwood actions instead. +/// +/// Known open question at the activation boundary: the first NU6.3 block's coinbase +/// has been observed served as Orchard (one action, zero ironwood) — see +/// for the hypotheses (zebrad +/// builder off-by-one vs missing routing vs a zaino pool-swap). This predicate over +/// raw validator blocks is that issue's disambiguator. +fn is_valid_ironwood_coinbase(block: &zebra_chain::block::Block) -> bool { + let Some(coinbase) = block.transactions.first() else { + return false; + }; + coinbase.is_coinbase() + && coinbase.version() == 6 + && coinbase.sapling_outputs().count() == 0 + && coinbase.orchard_actions().count() == 0 + && coinbase.ironwood_actions().count() >= 1 +} + +/// One-line coinbase summary for assertion messages, so a predicate failure names the +/// violated clause instead of reporting a bare boolean. +fn describe_coinbase(block: &zebra_chain::block::Block) -> String { + match block.transactions.first() { + Some(coinbase) => format!( + "coinbase: v{}, sapling outputs {}, orchard actions {}, ironwood actions {}, txs in block {}", + coinbase.version(), + coinbase.sapling_outputs().count(), + coinbase.orchard_actions().count(), + coinbase.ironwood_actions().count(), + block.transactions.len(), + ), + None => "block has no transactions".to_string(), + } +} + +/// Which shielded pool the fixture's coinbase reward must land in at a given height. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CoinbaseEra { + /// Below NU5: the shielded-receiver reward cannot land in either pool. + Neither, + /// NU5 through NU6.2: the orchard-receiver reward is paid as Orchard actions. + Orchard, + /// From NU6.3: the same receiver's reward is routed to Ironwood actions. + Ironwood, +} + +/// Launches an orchard-receiver mining fixture on `activation_heights`, generates +/// `blocks` blocks, and asserts each height's raw validator block satisfies exactly +/// the era predicate `expected_era` assigns it. Raw blocks come straight from the +/// validator — zaino runs (the harness needs its subscriber as the block-generation +/// pollable) but is never consulted, so a failure here is a class-1 +/// (consensus/routing) or fixture fact, never a zaino one. +/// +/// Mismatches are collected across the whole chain and reported together rather than +/// aborting at the first failing height, so one run distinguishes the hypotheses of +/// : a boundary-only mismatch is the +/// zebrad builder off-by-one (A1), every-post-activation-height mismatches mean the +/// routing is absent (A2), and a clean pass here while the e2e wire tests fail moves +/// the blame to a zaino pool-swap (B). +async fn assert_coinbase_routing( + activation_heights: ActivationHeights, + blocks: u32, + expected_era: impl Fn(u64) -> CoinbaseEra, +) { + #[allow(deprecated)] + let mut test_manager = TestManager::::launch_mining_to( + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(activation_heights), + None, + true, + false, + false, + ) + .await + .unwrap(); + let subscriber = test_manager.subscriber().clone(); + + test_manager + .generate_blocks_and_wait_for_tip(blocks, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.unwrap().0); + + let connector = test_manager.full_node_jsonrpc_connector().await; + let mut violations: Vec = Vec::new(); + for height in 0..=tip { + let block = match connector + .get_block(height.to_string(), Some(0)) + .await + .unwrap() + { + GetBlockResponse::Raw(raw) => { + zebra_chain::block::Block::zcash_deserialize(raw.as_ref()) + .expect("validator serves deserializable blocks") + } + other => panic!("verbosity-0 getblock must return a raw block, got {other:?}"), + }; + + let expected = expected_era(height); + let (want_orchard, want_ironwood) = match expected { + CoinbaseEra::Neither => (false, false), + CoinbaseEra::Orchard => (true, false), + CoinbaseEra::Ironwood => (false, true), + }; + let got_orchard = is_valid_orchard_coinbase(&block); + let got_ironwood = is_valid_ironwood_coinbase(&block); + if got_orchard != want_orchard || got_ironwood != want_ironwood { + violations.push(format!( + "height {height}: expected {expected:?}, predicates say \ + (orchard: {got_orchard}, ironwood: {got_ironwood}) — {}", + describe_coinbase(&block) + )); + } + } + + assert!( + violations.is_empty(), + "coinbase routing mismatches ({} of {} heights; see \ + https://github.com/zingolabs/zaino/issues/1368 for the hypothesis map):\n{}", + violations.len(), + tip + 1, + violations.join("\n") + ); + + test_manager.close().await; +} + +/// Orchard-only era: NU6.3 never activates, so every post-NU5 coinbase stays an +/// Orchard coinbase and no ironwood ever appears. (The zebrad *default* heights are +/// now the canonical NU6.3-at-2 set, so this fixture is explicit.) +/// +/// multi_thread required: the test manager spawns the validator and indexer services. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_only_coinbase_routing_zebrad() { + assert_coinbase_routing(ORCHARD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }) + .await; +} + +/// Ironwood-only era: NU6.3 active from height 2 (with every prior upgrade), so every +/// post-activation coinbase is an Ironwood coinbase and no Orchard coinbase ever +/// appears. +/// +/// multi_thread required: the test manager spawns the validator and indexer services. +#[tokio::test(flavor = "multi_thread")] +async fn ironwood_only_coinbase_routing_zebrad() { + assert_coinbase_routing(IRONWOOD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Ironwood + } else { + CoinbaseEra::Neither + } + }) + .await; +} + +/// The transition: the same unchanged orchard-receiver miner produces Orchard +/// coinbases through NU6.2 and Ironwood coinbases from the NU6.3 activation height — +/// each predicate exactly delimiting its era, so a mis-timed flip fails on both sides +/// of the boundary. +/// +/// multi_thread required: the test manager spawns the validator and indexer services. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_coinbase_routing_flips_to_ironwood_at_activation_zebrad() { + // Two blocks past the boundary, so both eras carry more than one block. + assert_coinbase_routing( + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY + 2, + |height| { + if height >= u64::from(NU6_3_TRANSITION_BOUNDARY) { + CoinbaseEra::Ironwood + } else if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }, + ) + .await; +} diff --git a/live-tests/clientless/tests/fetch_service.rs b/live-tests/clientless/tests/fetch_service.rs index 53ea44785..dfcc290ec 100644 --- a/live-tests/clientless/tests/fetch_service.rs +++ b/live-tests/clientless/tests/fetch_service.rs @@ -1,15 +1,14 @@ //! These tests compare the output of `FetchService` with the output of `JsonRpcConnector`. -use clientless::rpc::z_validate_address::{run_z_validate_for, SaplingSuite}; +use clientless::rpc::z_validate_address::run_z_validate_for; use futures::StreamExt as _; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; use zaino_proto::proto::compact_formats::CompactBlock; use zaino_proto::proto::service::{BlockId, BlockRange, GetSubtreeRootsArg}; -#[allow(deprecated)] use zaino_state::{ - FetchService, FetchServiceSubscriber, LightWalletIndexer, Status, StatusType, ZcashIndexer, + LightWalletIndexer, NodeBackedIndexerServiceSubscriber, Status, StatusType, ZcashIndexer, }; -use zaino_testutils::{TestManager, ValidatorExt, ValidatorKind}; +use zaino_testutils::{Rpc, TestManager, ValidatorExt, ValidatorKind}; use zebra_chain::parameters::subsidy::ParameterSubsidy as _; use zebra_rpc::client::ValidateAddressResponse; use zebra_rpc::methods::{GetBlock, GetBlockHash}; @@ -95,7 +94,7 @@ async fn fetch_service_get_latest_block(validator: &ValidatorKi #[allow(deprecated)] async fn assert_subscriber_matches_rpc( validator: &ValidatorKind, - fetch_query: impl FnOnce(FetchServiceSubscriber) -> FFut, + fetch_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> FFut, rpc_query: impl FnOnce(JsonRpSeeConnector) -> RFut, ) where V: ValidatorExt, @@ -214,7 +213,6 @@ async fn fetch_service_get_block_subsidy(validator: &ValidatorK // first halving boundary plus a margin on both validators. let height_limit = fetch_service_subscriber .network() - .to_zebra_network() .height_for_first_halving() .0 + 10; @@ -321,7 +319,7 @@ async fn fetch_service_get_block_header(validator: &ValidatorKi #[allow(deprecated)] async fn launch_and_mine_five_blocks( validator: &ValidatorKind, -) -> (TestManager, FetchServiceSubscriber) { +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { let (test_manager, fetch_service_subscriber) = zaino_testutils::launch_with_fetch_subscriber::(validator, None).await; @@ -417,11 +415,11 @@ async fn fetch_service_validate_address(validator: &ValidatorKi /// Launch a fetch-backend manager and run the shared `z_validate_address` /// suite against its subscriber. -async fn z_validate(validator: &ValidatorKind, suite: SaplingSuite) { +async fn z_validate(validator: &ValidatorKind) { let (mut test_manager, fetch_service_subscriber) = zaino_testutils::launch_with_fetch_subscriber::(validator, None).await; - run_z_validate_for(&fetch_service_subscriber, suite).await; + run_z_validate_for(&fetch_service_subscriber).await; test_manager.close().await; } @@ -649,7 +647,7 @@ mod zcashd { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub(crate) async fn z_validate_address() { - z_validate::(&ValidatorKind::Zcashd, SaplingSuite::Standard).await; + z_validate::(&ValidatorKind::Zcashd).await; } } @@ -720,11 +718,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub(crate) async fn z_validate_address() { - z_validate::( - &ValidatorKind::Zebrad, - SaplingSuite::ZebradPassthroughFetchService, - ) - .await; + z_validate::(&ValidatorKind::Zebrad).await; } } diff --git a/live-tests/clientless/tests/json_server.rs b/live-tests/clientless/tests/json_server.rs index 199e0709f..b8c17443c 100644 --- a/live-tests/clientless/tests/json_server.rs +++ b/live-tests/clientless/tests/json_server.rs @@ -5,9 +5,8 @@ //! docs/adr/0001-zcashd-support-feature-gate.md. #![cfg(feature = "zcashd_support")] -#[allow(deprecated)] -use zaino_state::{FetchService, FetchServiceSubscriber, ZcashIndexer}; -use zaino_testutils::TestManager; +use zaino_state::{NodeBackedIndexerServiceSubscriber, ZcashIndexer}; +use zaino_testutils::{Rpc, TestManager}; use zcash_local_net::validator::zcashd::Zcashd; /// Assert that `query` returns the same value from the zcashd-backed and @@ -16,11 +15,11 @@ use zcash_local_net::validator::zcashd::Zcashd; /// problem. The pure-compare building block of the json_server tests. #[allow(deprecated)] async fn assert_subscribers_agree( - zcashd_subscriber: &FetchServiceSubscriber, - zaino_subscriber: &FetchServiceSubscriber, + zcashd_subscriber: &NodeBackedIndexerServiceSubscriber, + zaino_subscriber: &NodeBackedIndexerServiceSubscriber, query: Q, ) where - Q: Fn(FetchServiceSubscriber) -> Fut, + Q: Fn(NodeBackedIndexerServiceSubscriber) -> Fut, Fut: std::future::Future, T: std::fmt::Debug + PartialEq, { @@ -34,13 +33,13 @@ async fn assert_subscribers_agree( /// tests that check an invariant holds across the chain. #[allow(deprecated)] async fn compare_over_blocks( - test_manager: &TestManager, - zcashd_subscriber: &FetchServiceSubscriber, - zaino_subscriber: &FetchServiceSubscriber, + test_manager: &TestManager, + zcashd_subscriber: &NodeBackedIndexerServiceSubscriber, + zaino_subscriber: &NodeBackedIndexerServiceSubscriber, blocks: u32, query: Q, ) where - Q: Fn(FetchServiceSubscriber) -> Fut, + Q: Fn(NodeBackedIndexerServiceSubscriber) -> Fut, Fut: std::future::Future, T: std::fmt::Debug + PartialEq, { @@ -430,11 +429,8 @@ mod zcashd { async fn z_validate_address() { let mut services = zaino_testutils::launch_zcashd_dual_fetch_services().await; - clientless::rpc::z_validate_address::run_z_validate_for( - &services.zcashd_subscriber, - clientless::rpc::z_validate_address::SaplingSuite::Standard, - ) - .await; + clientless::rpc::z_validate_address::run_z_validate_for(&services.zcashd_subscriber) + .await; services.test_manager.close().await; } diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index 5655dced3..59b607d40 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -1,11 +1,8 @@ use zaino_fetch::jsonrpsee::response::address_deltas::GetAddressDeltasParams; -#[allow(deprecated)] -use zaino_state::{ - FetchServiceSubscriber, LightWalletIndexer, StateServiceSubscriber, ZcashIndexer, -}; +use zaino_state::{LightWalletIndexer, NodeBackedIndexerServiceSubscriber, ZcashIndexer}; use zaino_testutils::{StateAndFetchServices, ValidatorExt}; -use zaino_testutils::{ValidatorKind, ZEBRAD_TESTNET_CACHE_DIR}; +use zaino_testutils::{ValidatorKind, ZEBRAD_THE_PUB_TESTNET_CACHE_DIR}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::parameters::NetworkKind; use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; @@ -26,7 +23,7 @@ async fn launch_regtest(enable_zaino: bool) -> StateAndFetchServices { async fn launch_testnet_cached() -> StateAndFetchServices { zaino_testutils::launch_state_and_fetch_services::( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), false, Some(NetworkKind::Testnet), ) @@ -42,8 +39,8 @@ async fn launch_testnet_cached() -> StateAndFetchServices { #[allow(deprecated)] async fn assert_subscribers_agree( services: &StateAndFetchServices, - fetch_query: impl FnOnce(FetchServiceSubscriber) -> FFut, - state_query: impl FnOnce(StateServiceSubscriber) -> SFut, + fetch_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> FFut, + state_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> SFut, ) -> T where T: std::fmt::Debug + PartialEq, @@ -386,7 +383,10 @@ mod zebra { #[tokio::test(flavor = "multi_thread")] async fn state_service_chaintip_update_subscriber() { let services = launch_regtest(true).await; - let mut chaintip_subscriber = services.state_subscriber.chaintip_update_subscriber(); + let mut chaintip_subscriber = services + .state_subscriber + .chaintip_update_subscriber() + .expect("the Direct connection exposes a local tip-change stream"); services .test_manager .generate_blocks_and_check_each( @@ -427,7 +427,7 @@ mod zebra { async fn testnet() { state_service_check_info::( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; @@ -576,11 +576,8 @@ mod zebra { ) .await; - clientless::rpc::z_validate_address::run_z_validate_for( - &services.state_subscriber, - clientless::rpc::z_validate_address::SaplingSuite::Standard, - ) - .await; + clientless::rpc::z_validate_address::run_z_validate_for(&services.state_subscriber) + .await; services.test_manager.close().await; } @@ -615,7 +612,7 @@ mod zebra { async fn block_object_testnet() { state_service_get_block_object( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; @@ -631,7 +628,7 @@ mod zebra { async fn block_raw_testnet() { state_service_get_block_raw( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; diff --git a/live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs b/live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs new file mode 100644 index 000000000..41833480d --- /dev/null +++ b/live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs @@ -0,0 +1,157 @@ +//! Observational walk of the real NU6.3 activation boundary on The Public +//! Testnet (glossary: The Public Testnet is the public test network and +//! nothing else). Historical blocks are permanent, so these invariants are +//! checkable forever, long after the epoch that produced them closed. +//! +//! The invariants, enforced one block below the boundary and at it: +//! +//! - Below the activation height the Ironwood pool holds no value. +//! - From the activation height the Orchard pool's value never increases: +//! the no-new-value rule admits only withdrawals and same-receiver change +//! (the cross-address restriction, +//! ). +//! +//! The activation height itself is read from the running validator's +//! `getblockchaininfo.upgrades` — the single source of truth for activation +//! heights — and cross-checked against the observed activation on The +//! Public Testnet at height 4,134,000 (~2026-07-04). +//! +//! Non-hermetic and therefore env-gated: the test needs a zebrad chain +//! cache at `~/.cache/zebra` (`ZEBRAD_THE_PUB_TESTNET_CACHE_DIR`) synced past the +//! activation height, and skips with a message when the cache is absent or +//! short. Run it manually, or from a job that maintains the cache. +//! +//! The validator is launched through `ZebradConfig` directly rather than +//! `TestManager`: the manager's launch path always writes regtest test +//! parameters into the config, while this test needs The Public Testnet +//! (`network_type: NetworkType::Testnet`, which makes zebrad ignore +//! `activation_heights` and `miner_address`). + +use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConnector}; +use zaino_fetch::jsonrpsee::response::GetBlockResponse; +use zaino_testutils::ZEBRAD_THE_PUB_TESTNET_CACHE_DIR; +use zcash_local_net::process::Process as _; +use zcash_local_net::protocol::NetworkType; +use zcash_local_net::validator::zebrad::{Zebrad, ZebradConfig}; +use zcash_local_net::validator::Validator as _; +use zebra_chain::parameters::NetworkUpgrade; + +/// The height The Public Testnet activated NU6.3 at, recorded from the real +/// activation. A mismatch means either a reset of The Public Testnet or a +/// wrong validator pin — both worth failing loudly over. +const OBSERVED_NU6_3_ACTIVATION_ON_THE_PUB_TESTNET: u32 = 4_134_000; + +/// The chain value of `pool_id` as of `height`, from the validator's +/// verbosity-2 block object. +async fn pool_zats(connector: &JsonRpSeeConnector, height: u32, pool_id: &str) -> i64 { + let response = connector + .get_block(height.to_string(), Some(2)) + .await + .expect("getblock verbosity 2"); + let GetBlockResponse::Object(block) = response else { + panic!("verbosity-2 getblock must return a block object"); + }; + block + .value_pools() + .expect("verbosity-2 block object carries value pools") + .iter() + .map(|pool| pool.balance()) + .find(|pool| pool.id() == pool_id) + .unwrap_or_else(|| panic!("value pools must include {pool_id}")) + .chain_value_zat() + .zatoshis() +} + +/// multi_thread required: the test launches the validator process and polls +/// it over RPC. +#[tokio::test(flavor = "multi_thread")] +async fn value_pools_respect_the_boundary_on_the_pub_testnet() { + let Some(cache_dir) = ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone() else { + eprintln!("skipping: no cache dir configured for The Public Testnet"); + return; + }; + if !cache_dir.exists() { + eprintln!( + "skipping: no zebrad chain cache of The Public Testnet at {}", + cache_dir.display() + ); + return; + } + + let config = ZebradConfig { + network_type: NetworkType::Testnet, + chain_cache: Some(cache_dir), + ..ZebradConfig::default() + }; + let mut zebrad = Zebrad::launch(config) + .await + .expect("launch a zebrad on The Public Testnet"); + + let rpc_address = format!("127.0.0.1:{}", zebrad.get_port()); + let connector = JsonRpSeeConnector::new_with_basic_auth( + test_node_and_return_url( + &rpc_address, + None, + Some("xxxxxx".to_string()), + Some("xxxxxx".to_string()), + ) + .await + .expect("validator RPC reachable"), + "xxxxxx".to_string(), + "xxxxxx".to_string(), + ) + .expect("connect to the validator RPC"); + + let blockchain_info = connector + .get_blockchain_info() + .await + .expect("getblockchaininfo"); + + // The validator's reported schedule is the source of truth for the + // boundary; the recorded constant pins the real history of The Public Testnet. + let boundary = blockchain_info + .upgrades + .values() + .find_map(|upgrade_info| { + let (upgrade, height, _status) = upgrade_info.into_parts(); + (upgrade == NetworkUpgrade::Nu6_3).then_some(height.0) + }) + .expect("the validator on The Public Testnet must report an NU6.3 activation height"); + assert_eq!( + boundary, OBSERVED_NU6_3_ACTIVATION_ON_THE_PUB_TESTNET, + "the validator's NU6.3 height must match the observed activation on The Public Testnet" + ); + + let tip = blockchain_info.blocks.0; + if tip <= boundary { + eprintln!( + "skipping: cache tip {tip} of The Public Testnet has not crossed the NU6.3 boundary {boundary}" + ); + zebrad.stop(); + return; + } + + // One block below the boundary and at it, plus a block of margin on + // each side: the Ironwood pool holds no value below the activation + // height, and the Orchard pool never grows from it. + for height in (boundary - 2)..boundary { + assert_eq!( + pool_zats(&connector, height, "ironwood").await, + 0, + "the ironwood pool must hold no value at height {height}, below the boundary" + ); + } + let walk_end = boundary + 1; + let mut previous_orchard = pool_zats(&connector, boundary - 1, "orchard").await; + for height in boundary..=walk_end { + let orchard = pool_zats(&connector, height, "orchard").await; + assert!( + orchard <= previous_orchard, + "the orchard pool must never grow from the boundary; \ + height {height} holds {orchard} zats after {previous_orchard}" + ); + previous_orchard = orchard; + } + + zebrad.stop(); +} diff --git a/live-tests/clientless/tests/validator_heights.rs b/live-tests/clientless/tests/validator_heights.rs new file mode 100644 index 000000000..62986f84d --- /dev/null +++ b/live-tests/clientless/tests/validator_heights.rs @@ -0,0 +1,153 @@ +//! Regression tests for the single source of truth for activation heights +//! (zaino#1076, `zainod-heights-from-validator-spec.md`). +//! +//! The invariant: the validator's configured activation heights are +//! authoritative. zainod's config carries only a network kind — its regtest +//! placeholder is the canonical `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` +//! regardless of the fixture — and both backends adopt the real schedule +//! from `getblockchaininfo.upgrades` at spawn. +//! +//! Two halves, matching the spec's acceptance criteria: +//! +//! 1. [`zainod_syncs_a_schedule_its_config_never_saw`] — boundary sync and +//! the no-recompile proof in one: the same zainod build and (kind-only) +//! configuration that every canonical-heights test runs is here pointed +//! at a validator on the NU6.3-at-6 transition schedule, and must sync +//! across the boundary and serve era-correct compact blocks. Before +//! adoption this exact misalignment killed the chain-index sync with +//! `InvalidData("Block commitment could not be computed")`. +//! 2. [`getblockchaininfo_reports_the_configured_schedule`] — the input +//! contract: what zebrad actually puts in the `upgrades` map for a +//! configured schedule, pinned against a live node rather than assumed. +//! The mapping from that shape to adopted heights is unit-tested next to +//! `activation_heights_from_upgrades` in zaino-state. + +use zaino_state::ZcashIndexer as _; +use zaino_testutils::{ + all_pools_i32, collect_block_range, MinerPool, Rpc, TestManager, ValidatorKind, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, +}; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Launches an orchard-receiver-mining zebrad on the transition schedule +/// with zainod enabled. The harness hands zainod only the canonical +/// placeholder (see `launch_mining_to`), so the launch itself is the +/// deliberate config/validator misalignment under test. +async fn launch_transition_validator() -> TestManager { + assert_ne!( + ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + "premise: zainod's config placeholder must differ from the fixture \ + schedule, or this proves nothing" + ); + + TestManager::::launch_mining_to( + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS), + None, + true, + false, + false, + ) + .await + .expect("launch TestManager") +} + +/// Boundary sync + no-recompile proof: a kind-only-configured zainod adopts +/// the NU6.3-at-6 schedule from the validator, syncs across the boundary, +/// and serves era-correct compact blocks for both eras. +/// +/// multi_thread required: the test manager spawns the validator and indexer +/// services. +#[tokio::test(flavor = "multi_thread")] +async fn zainod_syncs_a_schedule_its_config_never_saw() { + let mut test_manager = launch_transition_validator().await; + let subscriber = test_manager.subscriber().clone(); + + // Two blocks past the boundary, so both eras carry more than one block. + // Reaching the tip at all is the core regression: pre-adoption, the + // chain-index sync died on the first block whose commitment scheme the + // misconfigured heights got wrong. + test_manager + .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY + 1, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.expect("chain height").0); + assert!( + tip > u64::from(NU6_3_TRANSITION_BOUNDARY), + "sync must cross the boundary, tip is {tip}" + ); + + // Era composition of the served chain proves the adopted schedule is the + // validator's, not the placeholder: under the placeholder (NU6.3 at 2) + // the pre-boundary orchard coinbases would be misread as ironwood-era. + let blocks = collect_block_range(&subscriber, 2, tip, all_pools_i32()).await; + for block in &blocks { + let height = block.height; + let has_orchard = block.vtx.iter().any(|tx| !tx.actions.is_empty()); + let has_ironwood = block.vtx.iter().any(|tx| !tx.ironwood_actions.is_empty()); + if height >= u64::from(NU6_3_TRANSITION_BOUNDARY) { + assert!( + has_ironwood && !has_orchard, + "height {height} must be ironwood-era, got orchard={has_orchard} ironwood={has_ironwood}" + ); + } else { + assert!( + has_orchard && !has_ironwood, + "height {height} must be orchard-era, got orchard={has_orchard} ironwood={has_ironwood}" + ); + } + } + + test_manager.close().await; +} + +/// The input contract for adoption: the `upgrades` map a live zebrad reports +/// for the transition schedule, pinned exactly — upgrade set, order, and +/// heights. Establishes (from real output, not reasoning) that nothing +/// pre-Overwinter appears: the map is keyed by consensus branch ID, which +/// pre-Overwinter eras don't have. +/// +/// multi_thread required: the test manager spawns the validator and indexer +/// services. +#[tokio::test(flavor = "multi_thread")] +async fn getblockchaininfo_reports_the_configured_schedule() { + use zebra_chain::parameters::NetworkUpgrade; + + let mut test_manager = launch_transition_validator().await; + + let blockchain_info = test_manager + .full_node_jsonrpc_connector() + .await + .get_blockchain_info() + .await + .expect("getblockchaininfo"); + + let reported: Vec<(NetworkUpgrade, u32)> = blockchain_info + .upgrades + .values() + .map(|upgrade_info| { + let (upgrade, height, _status) = upgrade_info.into_parts(); + (upgrade, height.0) + }) + .collect(); + + assert_eq!( + reported, + vec![ + (NetworkUpgrade::Overwinter, 1), + (NetworkUpgrade::Sapling, 1), + (NetworkUpgrade::Blossom, 1), + (NetworkUpgrade::Heartwood, 1), + (NetworkUpgrade::Canopy, 1), + (NetworkUpgrade::Nu5, 2), + (NetworkUpgrade::Nu6, 2), + (NetworkUpgrade::Nu6_1, 2), + (NetworkUpgrade::Nu6_2, 2), + (NetworkUpgrade::Nu6_3, 6), + ], + ); + + test_manager.close().await; +} diff --git a/live-tests/e2e/Cargo.toml b/live-tests/e2e/Cargo.toml index 2d07ff170..2e6c53f4d 100644 --- a/live-tests/e2e/Cargo.toml +++ b/live-tests/e2e/Cargo.toml @@ -57,3 +57,9 @@ hex = { workspace = true } corez = { workspace = true } tower = { workspace = true } serde_json = { workspace = true } + +[dev-dependencies] +# Enable the tractable seam depth (fast-test-seam) so this crate's live tests can +# exercise finalisation/eviction at small chain heights. Dev-dependency only — the +# feature must never reach a shipped build (see the zaino-state feature comment). +zaino-state = { workspace = true, features = ["fast-test-seam"] } diff --git a/live-tests/e2e/src/devtool.rs b/live-tests/e2e/src/devtool.rs index 31b772d03..13ed1679d 100644 --- a/live-tests/e2e/src/devtool.rs +++ b/live-tests/e2e/src/devtool.rs @@ -4,7 +4,7 @@ //! //! [`DevtoolClients`] mirrors [`crate::Clients`]' method names one-for-one so //! tests can swap backends mechanically. The clients are managed by -//! zcash_local_net's [`zcash_local_net::client`] module: each wallet +//! zcash_local_net's [`zcash_local_net::wallet`] module: each wallet //! operation is a run-to-completion `zcash-devtool` subprocess invocation //! (the binary must be built with `--features regtest_support` and be //! locatable via `TEST_BINARIES_DIR`/`PATH`). @@ -21,9 +21,9 @@ //! constants derived from zingolib note selection (e.g. 235_000 after //! shielding 250_000) must be re-verified on first devtool runs. -use zcash_local_net::client::{ +use zcash_local_net::wallet::{ zcash_devtool::{ZcashDevtool, ZcashDevtoolConfig}, - AddressReceiver, Client as _, GetInfo, WalletBalance, + AddressReceiver, GetInfo, Wallet as _, WalletBalance, }; use zcash_primitives::transaction::TxId; @@ -39,17 +39,27 @@ pub struct DevtoolClients { } /// Launch faucet + recipient devtool wallets against a running Zaino gRPC -/// listener. The devtool analogue of [`crate::build_clients`]; Zaino must -/// already be serving (wallet initialization fetches the chain tip and -/// birthday tree state from it). -pub async fn build_clients(zaino_grpc_listen_port: u16) -> DevtoolClients { - let mut faucet_config = ZcashDevtoolConfig::faucet(); +/// listener, deriving the wallet network from the running `validator`. That +/// derivation is the only construction the client API offers +/// (infrastructure ADR 0003: the Validator is the single source of truth +/// for activation heights), so wallet/validator height drift is +/// unrepresentable — the wallets follow whatever schedule the validator was +/// launched with, fixture or canonical. The devtool analogue of +/// [`crate::build_clients`]; Zaino must already be serving (wallet +/// initialization fetches the chain tip and birthday tree state from it). +pub async fn build_clients( + zaino_grpc_listen_port: u16, + validator: &V, +) -> DevtoolClients { + let network = zcash_local_net::wallet::WalletNetwork::from_validator(validator).await; + + let mut faucet_config = ZcashDevtoolConfig::faucet(network); faucet_config.indexer_port = zaino_grpc_listen_port; let faucet = ZcashDevtool::launch(faucet_config) .await .expect("launch devtool faucet wallet"); - let mut recipient_config = ZcashDevtoolConfig::recipient(); + let mut recipient_config = ZcashDevtoolConfig::recipient(network); recipient_config.indexer_port = zaino_grpc_listen_port; let recipient = ZcashDevtool::launch(recipient_config) .await @@ -234,6 +244,7 @@ impl Pool { pub fn spendable_balance(self, balance: &WalletBalance) -> u64 { match self { Pool::Orchard => balance.orchard_spendable, + Pool::Ironwood => balance.ironwood_spendable, Pool::Sapling => balance.sapling_spendable, Pool::Transparent => balance.transparent_spendable, } diff --git a/live-tests/e2e/src/lib.rs b/live-tests/e2e/src/lib.rs index 6ac1bc325..c2788fd0b 100644 --- a/live-tests/e2e/src/lib.rs +++ b/live-tests/e2e/src/lib.rs @@ -17,8 +17,12 @@ pub mod devtool; /// address string plus a balance-field closure. #[derive(Clone, Copy, Debug)] pub enum Pool { - /// Orchard (funds routed via a unified address). + /// Orchard (funds routed via a unified address through NU6.2; from NU6.3 + /// devtool routes unified-address outputs to Ironwood — use + /// [`Pool::Ironwood`] for the receipt pool on NU6.3-active chains). Orchard, + /// Ironwood (funds routed via a unified address from NU6.3). + Ironwood, /// Sapling. Sapling, /// Transparent. @@ -30,7 +34,7 @@ impl Pool { /// funds into this pool. pub fn address_kind(self) -> &'static str { match self { - Pool::Orchard => "unified", + Pool::Orchard | Pool::Ironwood => "unified", Pool::Sapling => "sapling", Pool::Transparent => "transparent", } @@ -38,7 +42,7 @@ impl Pool { } /// Whether the compact tx with `txid` carries no data for `pool` (transparent -/// `vout` / sapling `outputs` / orchard `actions`). +/// `vout` / sapling `outputs` / orchard `actions` / `ironwood_actions`). fn pool_tx_field_empty(block: &CompactBlock, txid: &TxId, pool: Pool) -> bool { let tx = block .vtx @@ -49,6 +53,7 @@ fn pool_tx_field_empty(block: &CompactBlock, txid: &TxId, pool: Pool) -> bool { Pool::Transparent => tx.vout.is_empty(), Pool::Sapling => tx.outputs.is_empty(), Pool::Orchard => tx.actions.is_empty(), + Pool::Ironwood => tx.ironwood_actions.is_empty(), } } diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs new file mode 100644 index 000000000..ab77775d6 --- /dev/null +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -0,0 +1,205 @@ +//! Wire-tier (zainod gRPC) era tests for compact-block serving. +//! +//! The e2e-exclusive predicate here is **wire fidelity**: the compact blocks a real +//! tonic client receives from a running zainod equal, block for block, what the +//! in-process subscriber produces for the same request. Only this tier crosses the +//! protobuf encode → network → decode boundary and zainod's gRPC server; clientless +//! tests call subscriber methods in-process and can never observe that layer. +//! +//! On top of fidelity, each test asserts the era composition of the served stream for +//! the unfiltered request shape (empty `poolTypes`): orchard-only, ironwood-only, and +//! the orchard→ironwood transition at the NU6.3 activation boundary. + +use zaino_common::network::ActivationHeights; +use zaino_proto::proto::compact_formats::CompactBlock; +use zaino_proto::proto::service::compact_tx_streamer_client::CompactTxStreamerClient; +use zaino_proto::proto::service::{BlockId, BlockRange}; +#[allow(deprecated)] +use zaino_state::ZcashIndexer as _; +use zaino_testutils::{ + make_uri, MinerPool, Rpc, TestManager, ValidatorKind, IRONWOOD_ONLY_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, +}; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Which shielded pool the fixture's coinbase reward must land in at a given height, +/// as observed in the served compact form. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CoinbaseEra { + /// Below NU5: the shielded-receiver reward cannot land in either pool. + Neither, + /// NU5 through NU6.2: the orchard-receiver reward is paid as Orchard actions. + Orchard, + /// From NU6.3: the same receiver's reward is routed to Ironwood actions. + Ironwood, +} + +/// Launches an orchard-receiver mining fixture with zainod enabled, generates +/// `blocks` blocks, streams `[0, tip]` through zainod's real gRPC wire with the empty +/// `poolTypes` filter, and asserts: +/// +/// 1. wire fidelity — the streamed blocks equal the in-process subscriber's blocks; +/// 2. era composition — each height's served orchard/ironwood action presence matches +/// the era `expected_era` assigns it. +async fn assert_wire_served_eras( + activation_heights: ActivationHeights, + blocks: u32, + expected_era: impl Fn(u64) -> CoinbaseEra, +) { + #[allow(deprecated)] + let mut test_manager = TestManager::::launch_mining_to( + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(activation_heights), + None, + true, + false, + false, + ) + .await + .unwrap(); + let subscriber = test_manager.subscriber().clone(); + + test_manager + .generate_blocks_and_wait_for_tip(blocks, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.unwrap().0); + + // The in-process serving result: the same request answered without the wire. + let in_process = zaino_testutils::collect_block_range(&subscriber, 0, tip, vec![]).await; + + // The same request through zainod's real gRPC server. + let uri = make_uri( + test_manager + .zaino_grpc_listen_address + .expect("zaino enabled") + .port(), + ); + let mut grpc_client = CompactTxStreamerClient::connect(uri) + .await + .expect("zainod grpc reachable"); + let mut stream = grpc_client + .get_block_range(BlockRange { + start: Some(BlockId { + height: 0, + hash: vec![], + }), + end: Some(BlockId { + height: tip, + hash: vec![], + }), + // The unfiltered wire request real (including pre-Ironwood) clients send. + pool_types: vec![], + }) + .await + .expect("get_block_range succeeds") + .into_inner(); + let mut wire: Vec = Vec::new(); + while let Some(block) = stream.message().await.expect("stream yields blocks") { + wire.push(block); + } + + // E2e-exclusive predicate — wire fidelity: the protobuf encode/decode and zainod's + // gRPC server must be transparent. + assert_eq!( + wire.len(), + in_process.len(), + "wire and in-process must serve the same block count" + ); + for (wire_block, in_process_block) in wire.iter().zip(&in_process) { + assert_eq!( + wire_block, in_process_block, + "wire round-trip must be transparent at height {}", + wire_block.height + ); + } + + // Era composition of the served stream. Observed failing at the first + // ironwood-era height (served orchard 1 / ironwood 0) — hypotheses and the + // raw-block disambiguation procedure are tracked in + // . + for block in &wire { + let orchard_actions: usize = block.vtx.iter().map(|tx| tx.actions.len()).sum(); + let ironwood_actions: usize = block.vtx.iter().map(|tx| tx.ironwood_actions.len()).sum(); + let (want_orchard, want_ironwood) = match expected_era(block.height) { + CoinbaseEra::Neither => (false, false), + CoinbaseEra::Orchard => (true, false), + CoinbaseEra::Ironwood => (false, true), + }; + assert_eq!( + orchard_actions > 0, + want_orchard, + "served orchard actions mismatch at height {} (orchard {orchard_actions}, \ + ironwood {ironwood_actions})", + block.height + ); + assert_eq!( + ironwood_actions > 0, + want_ironwood, + "served ironwood actions mismatch at height {} (orchard {orchard_actions}, \ + ironwood {ironwood_actions})", + block.height + ); + } + + test_manager.close().await; +} + +/// Orchard-only era over the wire: NU6.3 never activates (explicit fixture — the +/// zebrad default heights are now the canonical NU6.3-at-2 set), so the served +/// stream carries Orchard actions from height 2 and no ironwood anywhere. +/// +/// multi_thread required: the test manager spawns the validator, indexer, and zainod. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_only_wire_serving_zebrad() { + assert_wire_served_eras(ORCHARD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }) + .await; +} + +/// Ironwood-only era over the wire: NU6.3 active from height 2, so the served stream +/// carries Ironwood actions from height 2 and no Orchard actions anywhere. +/// +/// multi_thread required: the test manager spawns the validator, indexer, and zainod. +#[tokio::test(flavor = "multi_thread")] +async fn ironwood_only_wire_serving_zebrad() { + assert_wire_served_eras(IRONWOOD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Ironwood + } else { + CoinbaseEra::Neither + } + }) + .await; +} + +/// The transition over the wire: the same unchanged orchard-receiver miner's served +/// stream flips from Orchard to Ironwood actions exactly at the NU6.3 activation +/// height. +/// +/// multi_thread required: the test manager spawns the validator, indexer, and zainod. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_to_ironwood_transition_wire_serving_zebrad() { + // Two blocks past the boundary, so both eras carry more than one block. + assert_wire_served_eras( + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY + 2, + |height| { + if height >= u64::from(NU6_3_TRANSITION_BOUNDARY) { + CoinbaseEra::Ironwood + } else if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }, + ) + .await; +} diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 5e14d4443..94012c554 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -21,18 +21,17 @@ //! //! Deferred, with the capability each waits on: //! - `send_to_transparent` (heavy / finalization) — runnable now via orchard -//! funding, but the load-bearing 99-block advance across the finalized / +//! funding, but the load-bearing seam-deep advance across the finalized / //! non-finalized boundary costs a halo2 proof per block; waits on cheap -//! filler-block mining (round-3 spec P2). +//! transparent filler-block mining. //! - `monitor_unverified_mempool` — unconfirmed (mempool) wallet balances; -//! devtool sync is block-based (round-3 spec P3 — likely stays on zingolib, -//! the indexer-side mempool views above already cover the surface). +//! devtool sync is block-based (likely stays on zingolib, the indexer-side +//! mempool views above already cover the surface). //! - the zcashd matrix (`json_server`, zcashd send/query) — the devtool wallet -//! rejects zcashd's default regtest activation heights at construction -//! (round-2 spec P0); `json_server` is additionally zcashd-bound (its -//! reference subscriber *is* zcashd). -//! - the `test_vectors` chain builder — transparent-coinbase shielding -//! (round-2 spec P1). +//! rejects zcashd's default regtest activation heights at construction; +//! `json_server` is additionally zcashd-bound (its reference subscriber *is* +//! zcashd). +//! - the `test_vectors` chain builder — devtool transparent-coinbase shielding. //! - `get_mempool_info` — recomputes expected sizes from //! `FetchServiceSubscriber` internals; low value over the mempool surfaces //! already covered. @@ -44,9 +43,8 @@ use e2e::devtool::DevtoolClients; use zaino_proto::proto::service::{ AddressList, BlockId, BlockRange, GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, }; -use zaino_state::{LightWalletIndexer, ZcashIndexer, ZcashService}; -use zaino_testutils::{PollableTip, TestManager, TestService, ValidatorKind}; -use zainodlib::error::IndexerError; +use zaino_state::{LightWalletIndexer, ZcashIndexer}; +use zaino_testutils::{PollableTip, Rpc, TestManager, ValidatorKind}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::subtree::NoteCommitmentSubtreeIndex; use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; @@ -59,15 +57,13 @@ use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; /// spendable orchard notes the faucet ends with — one per send a test makes /// before mining (devtool will not chain unconfirmed change). The shared /// launch+fund preamble of the ports below. -async fn launch_and_fund_faucet( +async fn launch_and_fund_faucet( coinbase_blocks: u32, -) -> (TestManager, DevtoolClients) +) -> (TestManager, DevtoolClients) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, &ValidatorKind::Zebrad, None, @@ -85,6 +81,7 @@ where .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -99,13 +96,11 @@ where /// Launch an orchard-mining zebrad + Zaino on the `Service` backend and build /// devtool faucet/recipient wallets against it, without mining or syncing — the /// minimal preamble for tests that only exercise wallet↔server connectivity. -async fn launch_and_build_clients() -> (TestManager, DevtoolClients) +async fn launch_and_build_clients() -> (TestManager, DevtoolClients) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, &ValidatorKind::Zebrad, None, @@ -123,6 +118,7 @@ where .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -133,13 +129,11 @@ where /// and recipient wallets can report node/server info without erroring. Smoke /// test — the original discards the result. (`chain_name` is "test" for regtest /// and `server_uri` has no trailing slash, so this asserts neither.) -async fn connect_to_node_get_info() +async fn connect_to_node_get_info() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, clients) = launch_and_build_clients::().await; + let (mut test_manager, clients) = launch_and_build_clients::().await; clients.get_info_faucet().await; clients.get_info_recipient().await; @@ -149,18 +143,18 @@ where /// Port of `wallet_to_validator::check_received_mining_reward` (zebrad): the /// faucet's synced wallet sees the orchard coinbase note. -async fn receives_mining_reward() +async fn receives_mining_reward() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, clients) = launch_and_fund_faucet::(1).await; let faucet_balance = dbg!(clients.faucet_balance().await); + // Under the canonical NU6.3-at-2 heights the orchard-receiver mining reward + // is routed to the Ironwood pool. assert!( - faucet_balance.orchard_spendable > 0, - "faucet should hold a spendable orchard coinbase note, got {faucet_balance:?}" + faucet_balance.ironwood_spendable > 0, + "faucet should hold a spendable ironwood coinbase note, got {faucet_balance:?}" ); test_manager.close().await; @@ -169,19 +163,17 @@ where /// Port of the `assert_send_to_pool` family from `wallet_to_validator` /// (zebrad): send 250_000 from the faucet (spending its orchard coinbase) to /// the recipient's `pool` address, mine it in, and assert the recipient's -/// synced wallet shows the receipt in that pool. Covers `send_to_orchard` -/// (unified), `send_to_sapling`, and the basic transparent receipt — the +/// synced wallet shows the receipt in that pool. Covers `send_to_ironwood` +/// (unified, the NU6.3-era receipt pool), `send_to_sapling`, and the basic transparent receipt — the /// per-pool address wiring is the new surface under test. (The original /// `send_to_transparent` additionally mines across the finalization boundary /// under transparent mining; that variant is deferred, as it exercises the /// transparent-coinbase detection path devtool has not yet been run against.) -async fn send_to_pool(pool: e2e::Pool) +async fn send_to_pool(pool: e2e::Pool) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient = clients.get_recipient_address(pool.address_kind()).await; let txid = clients.send_from_faucet(&recipient, 250_000).await; @@ -210,14 +202,12 @@ where /// balance assertions — the sends confirm in one block, and received funds are /// regular (non-coinbase) outputs with no maturity rule — and 100 orchard /// blocks would cost 100 halo2 proofs against the net-speedup criterion. -async fn send_to_all() +async fn send_to_all() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { // Three orchard notes — one per send (devtool will not chain unconfirmed change). - let (mut test_manager, mut clients) = launch_and_fund_faucet::(3).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(3).await; let recipient_ua = clients.get_recipient_address("unified").await; let recipient_zaddr = clients.get_recipient_address("sapling").await; @@ -232,7 +222,11 @@ where clients.sync_recipient().await; let balance = clients.recipient_balance().await; - assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); + // From NU6.3 devtool routes unified-address outputs to Ironwood; the + // orchard pool must stay empty (a nonzero orchard here means the receipt + // was mislabelled, not merely misrouted). + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 0); assert_eq!(e2e::Pool::Sapling.spendable_balance(&balance), 250_000); assert_eq!(e2e::Pool::Transparent.spendable_balance(&balance), 250_000); @@ -241,17 +235,15 @@ where /// Port of `wallet_to_validator::shield_for_validator` (zebrad): the faucet /// sends 250_000 to the recipient's transparent address; the recipient -/// confirms the transparent receipt, shields it to orchard, and confirms the -/// orchard balance net of the ZIP-317 fee (250_000 − 15_000 = 235_000 — the -/// first devtool exercise of `shield`, and the constant flagged as needing -/// re-verification against devtool's note selection). -async fn shield_for_validator() +/// confirms the transparent receipt, shields it (to Ironwood from NU6.3), and +/// confirms the shielded balance net of the ZIP-317 fee (250_000 − 15_000 = +/// 235_000 — the first devtool exercise of `shield`, and the constant flagged +/// as needing re-verification against devtool's note selection). +async fn shield_for_validator() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_taddr = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_taddr, 250_000).await; @@ -272,7 +264,7 @@ where clients.sync_recipient().await; assert_eq!( - e2e::Pool::Orchard.spendable_balance(&clients.recipient_balance().await), + e2e::Pool::Ironwood.spendable_balance(&clients.recipient_balance().await), 235_000 ); @@ -283,14 +275,12 @@ where /// mining) one transparent and one unified send into the mempool. Returns the /// manager and the two broadcast txids (transparent, then unified). The /// devtool analogue of `fund_and_fill_mempool`. -async fn fund_and_fill_mempool() -> (TestManager, String, String) +async fn fund_and_fill_mempool() -> (TestManager, String, String) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { // Two orchard notes — one per unmined send. - let (test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(2).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let recipient_ua = clients.get_recipient_address("unified").await; @@ -308,15 +298,13 @@ where /// txid hex. The txid is in display (reversed) order — devtool prints it that /// way, which matches the txid strings zaino's address queries return, so no /// conversion is needed for those comparisons. -async fn fund_and_send_to( +async fn fund_and_send_to( pool: e2e::Pool, -) -> (TestManager, DevtoolClients, String) +) -> (TestManager, DevtoolClients, String) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient = clients.get_recipient_address(pool.address_kind()).await; let txid_hex = clients.send_from_faucet(&recipient, 250_000).await; @@ -331,14 +319,12 @@ where /// Port of `fetch_service_get_address_tx_ids` (zebrad): after a transparent /// send to the recipient, `get_address_tx_ids` over the recipient's taddr /// returns the send's txid. -async fn get_address_tx_ids() +async fn get_address_tx_ids() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; // A range start a couple of blocks below the tip, covering the send block. @@ -362,14 +348,12 @@ where /// Port of `fetch_service_get_address_utxos` (zebrad): after a transparent /// send to the recipient, `z_get_address_utxos` over the recipient's taddr /// returns a utxo whose txid is the send's. -async fn get_address_utxos() +async fn get_address_utxos() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos = test_manager @@ -387,14 +371,12 @@ where /// Port of `fetch_service_z_get_treestate` (zebrad, smoke): `z_get_treestate` /// at the tip height succeeds. -async fn z_get_treestate() +async fn z_get_treestate() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + fund_and_send_to::(e2e::Pool::Orchard).await; let tip = test_manager.subscriber().tip_height().await; dbg!(test_manager @@ -408,14 +390,12 @@ where /// Port of `fetch_service_z_get_subtrees_by_index` (zebrad, smoke): /// `z_get_subtrees_by_index` for orchard succeeds. -async fn z_get_subtrees_by_index() +async fn z_get_subtrees_by_index() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + fund_and_send_to::(e2e::Pool::Orchard).await; dbg!(test_manager .subscriber() @@ -428,14 +408,11 @@ where /// Port of `fetch_service_get_raw_transaction` (zebrad, smoke): /// `get_raw_transaction` for the orchard send's txid succeeds. -async fn get_raw_transaction() +async fn get_raw_transaction() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, _clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + let (mut test_manager, _clients, txid_hex) = fund_and_send_to::(e2e::Pool::Orchard).await; dbg!(test_manager .subscriber() @@ -449,16 +426,14 @@ where /// Port of `fetch_service_get_taddress_txids` (zebrad, smoke): /// `get_taddress_txids` over the recipient's taddr and a height range around /// the send succeeds. -async fn get_taddress_txids() +async fn get_taddress_txids() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let tip = test_manager.subscriber().tip_height().await; @@ -492,14 +467,12 @@ where /// Port of `fetch_service_get_taddress_utxos` (zebrad, smoke): /// `get_address_utxos` over the recipient's taddr succeeds. -async fn get_taddress_utxos() +async fn get_taddress_utxos() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos_arg = GetAddressUtxosArg { @@ -518,16 +491,14 @@ where /// Port of `fetch_service_get_taddress_utxos_stream` (zebrad, smoke): /// `get_address_utxos_stream` over the recipient's taddr succeeds. -async fn get_taddress_utxos_stream() +async fn get_taddress_utxos_stream() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos_arg = GetAddressUtxosArg { @@ -551,14 +522,12 @@ where /// Port of `fetch_service_get_raw_mempool` (zebrad): with two transactions /// broadcast but unmined, the indexer's `get_raw_mempool` matches the /// validator's own JSON-RPC `getrawmempool`. -async fn get_raw_mempool() +async fn get_raw_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _transparent_txid, _unified_txid) = - fund_and_fill_mempool::().await; + fund_and_fill_mempool::().await; let json_service = test_manager.full_node_jsonrpc_connector().await; @@ -577,17 +546,14 @@ where /// Port of `fetch_service_get_mempool_tx` (zebrad): the `get_mempool_tx` /// stream returns the two unmined transactions keyed by txid (internal byte /// order), and the exclude-by-txid-suffix filter drops the named one. -async fn get_mempool_tx() +async fn get_mempool_tx() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; use zaino_proto::proto::service::GetMempoolTxRequest; - let (mut test_manager, transparent_txid, unified_txid) = - fund_and_fill_mempool::().await; + let (mut test_manager, transparent_txid, unified_txid) = fund_and_fill_mempool::().await; let to_bytes = |hex: &str| -> [u8; 32] { e2e::devtool::txid_internal_bytes(hex) @@ -638,16 +604,14 @@ where /// Port of `fetch_service_get_mempool_stream` (zebrad): a `get_mempool_stream` /// subscription opened before two sends observes those unmined transactions. /// Smoke test — the original only `dbg!`s the collected stream. -async fn get_mempool_stream() +async fn get_mempool_stream() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; // Two orchard notes — one per unmined send. - let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; // Subscribe before the sends so the stream observes them entering the mempool. let subscriber = test_manager.subscriber().clone(); @@ -687,13 +651,11 @@ where /// order. The devtool analogue of `fund_and_send(Pool::Orchard)`; the wallet /// clients are dropped once the transaction is mined (the queries below hit /// zaino, not the wallet). -async fn fund_and_send_orchard() -> (TestManager, Vec) +async fn fund_and_send_orchard() -> (TestManager, Vec) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_hex = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -708,13 +670,11 @@ where /// Port of `fetch_service_get_transaction_mined` (zebrad): the indexer serves /// `get_transaction` for the mined orchard send, keyed by its txid. Also /// confirms `txid_internal_bytes` yields the order the indexer matches on. -async fn get_transaction_mined() +async fn get_transaction_mined() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, txid_bytes) = fund_and_send_orchard::().await; + let (mut test_manager, txid_bytes) = fund_and_send_orchard::().await; let tx_filter = TxFilter { block: None, @@ -734,13 +694,11 @@ where /// Port of `fetch_service_get_transaction_mempool` (zebrad): the indexer /// serves `get_transaction` for an orchard send that is broadcast but not /// mined, keyed by its txid — i.e. from the mempool. -async fn get_transaction_mempool() +async fn get_transaction_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_hex = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -784,6 +742,7 @@ async fn block_range_returns_default_pools() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -875,10 +834,11 @@ async fn block_range_returns_all_pools() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; - // Three orchard coinbase notes (one per send below — devtool will not + // Three ironwood coinbase notes (one per send below — devtool will not // chain unconfirmed change), then one send to each pool's recipient // address, mined into a single block. svc.generate_blocks_and_wait_for_tips(3).await; @@ -889,7 +849,7 @@ async fn block_range_returns_all_pools() { let recipient_u = clients.get_recipient_address("unified").await; let deshielding_txid = clients.send_from_faucet(&recipient_t, 250_000).await; let sapling_txid = clients.send_from_faucet(&recipient_s, 250_000).await; - let orchard_txid = clients.send_from_faucet(&recipient_u, 250_000).await; + let ironwood_txid = clients.send_from_faucet(&recipient_u, 250_000).await; svc.generate_blocks_and_wait_for_tips(1).await; let start_height: u64 = 1; @@ -927,11 +887,10 @@ async fn block_range_returns_all_pools() { &e2e::devtool::txid_from_devtool(&sapling_txid), e2e::Pool::Sapling, ); - e2e::assert_pool_present( - compact_block, - &e2e::devtool::txid_from_devtool(&orchard_txid), - e2e::Pool::Orchard, - ); + let ironwood_txid = e2e::devtool::txid_from_devtool(&ironwood_txid); + e2e::assert_pool_present(compact_block, &ironwood_txid, e2e::Pool::Ironwood); + // The unified-address send must carry no Orchard actions from NU6.3. + e2e::assert_pool_absent(compact_block, &ironwood_txid, e2e::Pool::Orchard); svc.test_manager.close().await; } @@ -944,14 +903,12 @@ async fn block_range_returns_all_pools() { /// Port of `fetch_service_get_address_balance` (zebrad): after a transparent /// send of 250_000 to the recipient, `z_get_address_balance` over that taddr /// reports exactly 250_000. -async fn get_address_balance() +async fn get_address_balance() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let balance = test_manager @@ -970,14 +927,12 @@ where /// Port of `fetch_service_get_taddress_balance` (zebrad): after a transparent /// send of 250_000 to the recipient, `get_taddress_balance` over that taddr /// reports value_zat 250_000. -async fn get_taddress_balance() +async fn get_taddress_balance() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let address_list = AddressList { @@ -1017,6 +972,7 @@ async fn fund_and_send_dual( .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1168,6 +1124,7 @@ async fn fund_and_fill_mempool_dual() -> zaino_testutils::StateAndFetchServices< .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1251,7 +1208,7 @@ async fn transparent_data_in_compact_block() { // The assertion below requires every tx to carry a transparent vout; // the miner's transparent coinbase is that data source, so coinbase // must land on the miner taddr. - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, @@ -1321,8 +1278,7 @@ async fn get_address_balance_fetch_vs_state() { /// `launch_and_build_faucet_request`'s preamble for the faucet-taddr query /// tests. The faucet taddr is the abandon-art transparent receiver, which is /// also the zebrad miner address under transparent mining; the non-vacuity -/// probes in the callers verify that equality empirically (devtool round-2 -/// spec P1(1)). +/// probes in the callers verify that equality empirically. async fn launch_transparent_and_faucet_taddr( blocks: u32, ) -> (zaino_testutils::StateAndFetchServices, String) { @@ -1330,7 +1286,7 @@ async fn launch_transparent_and_faucet_taddr( // These tests query the faucet taddr, which only coinbase funds — // mining must stay transparent or the queries compare empty against // empty. - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, @@ -1343,6 +1299,7 @@ async fn launch_transparent_and_faucet_taddr( .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1356,7 +1313,7 @@ async fn launch_transparent_and_faucet_taddr( /// the fetch and state indexers agree on `get_address_tx_ids` over the faucet's /// coinbase taddr. The non-vacuity probe (`!txids.is_empty()`) guards against a /// silent empty==empty pass and confirms the devtool faucet's transparent -/// receiver equals the zebrad miner address (devtool round-2 spec P1(1)). +/// receiver equals the zebrad miner address. async fn get_taddress_txids_faucet_fetch_vs_state() { let (mut svc, faucet_taddr) = launch_transparent_and_faucet_taddr(100).await; @@ -1384,8 +1341,7 @@ async fn get_taddress_txids_faucet_fetch_vs_state() { /// Port of `state_service_…::get_taddress_balance` (zebrad, faucet-taddr /// cluster): the fetch and state indexers agree on `get_taddress_balance` over /// the faucet's coinbase taddr. The non-vacuity probe (`value_zat > 0`) guards -/// against a silent 0==0 pass and confirms the address equality of round-2 -/// spec P1(1). +/// against a silent 0==0 pass and confirms the address equality. async fn get_taddress_balance_faucet_fetch_vs_state() { let (mut svc, faucet_taddr) = launch_transparent_and_faucet_taddr(5).await; @@ -1484,7 +1440,7 @@ async fn get_address_utxos_stream_faucet_fetch_vs_state() { /// The block-range edge tests need a known 100-block tip and no wallet client. async fn launch_transparent_to_height_100() -> zaino_testutils::StateAndFetchServices { let svc = zaino_testutils::launch_state_and_fetch_services_mining_to::( - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, @@ -1579,24 +1535,22 @@ async fn get_block_range_out_of_range_lower_bound() { /// Port of `send_to_transparent` (wallet_to_validator, heavy/finalization, /// zebrad): a transparent send to the recipient returns the same address txids /// whether served from the non-finalized chain (just mined) or after the -/// 99-block advance pushes the send past the finalization boundary -/// (NON_FINALIZED_DEPTH = 100). +/// seam-deep advance pushes the send past the finalization boundary +/// (the seam depth `FAST_TEST_MAX_NONFINALISED_DEPTH` under `fast-test-seam`). /// -/// `#[ignore]`d (gated): the load-bearing 99-block advance mines orchard +/// `#[ignore]`d (gated): the load-bearing seam-deep advance mines orchard /// coinbase here — the devtool faucet funds via orchard, since the original's /// transparent-mine + shield path needs devtool transparent-coinbase shielding -/// (round-2 P1) — so it costs ~99 halo2 proofs, against the net-speedup +/// — so it costs ~100 halo2 proofs, against the net-speedup /// criterion. The miner pool is fixed at launch and `generate_blocks` has no /// per-call override, so the advance can't be cheap transparent filler until -/// per-call filler mining (round-3 P2) lands; un-ignore and mine the advance to +/// per-call filler mining lands; un-ignore and mine the advance to /// a transparent throwaway then. -async fn send_to_transparent_finalization() +async fn send_to_transparent_finalization() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_taddr = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_taddr, 250_000).await; @@ -1611,11 +1565,14 @@ where .await .unwrap(); - // The load-bearing advance: 99 blocks push the send below NON_FINALIZED_DEPTH - // into the finalized DB. Orchard coinbase here (see #[ignore] rationale). + // The load-bearing advance: these blocks push the send below the seam + // (`FAST_TEST_MAX_NONFINALISED_DEPTH`) into the finalized DB. Orchard coinbase + // here (see #[ignore] rationale). test_manager .generate_blocks_bulk_and_wait_for_tips( - 99, + // Advance past the seam so the send crosses the finalised floor + // (`tip - seam`); a small margin above it keeps the boundary unambiguous. + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5, test_manager.subscriber(), test_manager.subscriber(), ) @@ -1662,7 +1619,7 @@ async fn address_deltas() { let mut svc = zaino_testutils::launch_state_and_fetch_services_mining_to::( // The deltas under test are the faucet taddr's coinbase credits and the // shield's debit — coinbase must land on the miner taddr. - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, @@ -1674,6 +1631,7 @@ async fn address_deltas() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1833,6 +1791,7 @@ async fn get_block_deltas_resolves_transparent_spend() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -2066,10 +2025,13 @@ async fn get_outpoint_spenders_fetch_vs_state() { use zaino_state::chain_index::types::ChainScope; use zaino_state::ChainIndex as _; - // `NON_FINALIZED_DEPTH` is 100; mining this many blocks past a spend buries - // it under the finalised floor so `ChainScope::Finalised` can resolve it. - // A small margin above 100 keeps the boundary unambiguous. - const FINALITY_DEPTH: u32 = 105; + // Bury a spend this many blocks past its block to push it below the finalised + // floor (tip − seam depth) so `ChainScope::Finalised` can resolve it. This crate + // enables `fast-test-seam`, so the live zaino-state uses the tractable + // `FAST_TEST_MAX_NONFINALISED_DEPTH`; a small margin above it keeps the boundary + // unambiguous. (Without the feature the real seam is ~1000, impractical to mine + // here — see zingolabs/zaino#1352.) + const FINALITY_DEPTH: u32 = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5; const FUNDING_AMOUNT: u64 = 250_000; let mut svc = zaino_testutils::launch_state_and_fetch_services_mining_to::( @@ -2085,6 +2047,7 @@ async fn get_outpoint_spenders_fetch_vs_state() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -2148,13 +2111,11 @@ async fn get_outpoint_spenders_fetch_vs_state() { /// sync is block-based so it never scans the mempool). Restore the assertions /// (using `Pool::spendable_balance` for the confirmed ones) and un-ignore when /// devtool surfaces unconfirmed balances. -async fn monitor_unverified_mempool() +async fn monitor_unverified_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_1 = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -2221,13 +2182,11 @@ where /// Port of `test_get_mempool_info` (fetch_service, zebrad): `get_mempool_info` /// matches values recomputed from the fetch subscriber's mempool internals. /// FetchService-only — the recompute reads `FetchServiceSubscriber.indexer`. -#[allow(deprecated)] // FetchService is a deprecated re-export. async fn get_mempool_info_fetch() { use hex::ToHex as _; use zaino_state::ChainIndex as _; - let (mut test_manager, _transparent_txid, _unified_txid) = - fund_and_fill_mempool::().await; + let (mut test_manager, _transparent_txid, _unified_txid) = fund_and_fill_mempool::().await; let subscriber = test_manager.subscriber(); let info = subscriber.get_mempool_info().await.unwrap(); @@ -2263,7 +2222,7 @@ async fn get_mempool_info_state() { let mut svc = fund_and_fill_mempool_dual().await; let info = svc.state_subscriber.get_mempool_info().await.unwrap(); - let entries = svc.state_subscriber.mempool.get_mempool().await; + let entries = svc.state_subscriber.mempool().get_mempool().await; assert_eq!(entries.len() as u64, info.size); assert!(info.size >= 1); @@ -2284,74 +2243,71 @@ async fn get_mempool_info_state() { } mod zebrad { - // FetchService is a deprecated re-export; the deprecation fires at the - // turbofish use sites below, so the allow covers the whole module. - #[allow(deprecated)] mod fetch_service { - use zaino_state::FetchService; + use zaino_testutils::Rpc; #[tokio::test(flavor = "multi_thread")] async fn receives_mining_reward() { - crate::receives_mining_reward::().await; + crate::receives_mining_reward::().await; } #[tokio::test(flavor = "multi_thread")] async fn connect_to_node_get_info() { - crate::connect_to_node_get_info::().await; + crate::connect_to_node_get_info::().await; } #[tokio::test(flavor = "multi_thread")] - async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + async fn send_to_ironwood() { + crate::send_to_pool::(e2e::Pool::Ironwood).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_sapling() { - crate::send_to_pool::(e2e::Pool::Sapling).await; + crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_transparent() { - crate::send_to_pool::(e2e::Pool::Transparent).await; + crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_all() { - crate::send_to_all::().await; + crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] async fn shield_for_validator() { - crate::shield_for_validator::().await; + crate::shield_for_validator::().await; } #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when cheap filler mining lands" )] async fn send_to_transparent_finalization() { - crate::send_to_transparent_finalization::().await; + crate::send_to_transparent_finalization::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mined() { - crate::get_transaction_mined::().await; + crate::get_transaction_mined::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_mempool() { - crate::get_raw_mempool::().await; + crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_mempool_tx() { - crate::get_mempool_tx::().await; + crate::get_mempool_tx::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream() { - crate::get_mempool_stream::().await; + crate::get_mempool_stream::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2365,62 +2321,62 @@ mod zebrad { ignore = "devtool WalletBalance has no unconfirmed_*/confirmed_* fields; balance asserts are commented out — restore + un-ignore when devtool surfaces unconfirmed balances" )] async fn monitor_unverified_mempool() { - crate::monitor_unverified_mempool::().await; + crate::monitor_unverified_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_tx_ids() { - crate::get_address_tx_ids::().await; + crate::get_address_tx_ids::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_utxos() { - crate::get_address_utxos::().await; + crate::get_address_utxos::().await; } #[tokio::test(flavor = "multi_thread")] async fn z_get_treestate() { - crate::z_get_treestate::().await; + crate::z_get_treestate::().await; } #[tokio::test(flavor = "multi_thread")] async fn z_get_subtrees_by_index() { - crate::z_get_subtrees_by_index::().await; + crate::z_get_subtrees_by_index::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_transaction() { - crate::get_raw_transaction::().await; + crate::get_raw_transaction::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_txids() { - crate::get_taddress_txids::().await; + crate::get_taddress_txids::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_utxos() { - crate::get_taddress_utxos::().await; + crate::get_taddress_utxos::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_utxos_stream() { - crate::get_taddress_utxos_stream::().await; + crate::get_taddress_utxos_stream::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mempool() { - crate::get_transaction_mempool::().await; + crate::get_transaction_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_balance() { - crate::get_address_balance::().await; + crate::get_address_balance::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_balance() { - crate::get_taddress_balance::().await; + crate::get_taddress_balance::().await; } } @@ -2518,7 +2474,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: mines ~110 orchard-coinbase blocks (~110 halo2 proofs) to bury a finalised spend below NON_FINALIZED_DEPTH; un-ignore for manual / dedicated CI" + ignore = "heavy: mines ~105 orchard-coinbase blocks (~105 halo2 proofs) to bury a finalised spend below the seam (FAST_TEST_MAX_NONFINALISED_DEPTH); un-ignore for manual / dedicated CI" )] async fn get_outpoint_spenders_fetch_vs_state() { crate::get_outpoint_spenders_fetch_vs_state().await; @@ -2540,61 +2496,60 @@ mod zebrad { } mod state_service { - #[allow(deprecated)] - use zaino_state::StateService; + use zaino_testutils::Direct; #[tokio::test(flavor = "multi_thread")] async fn receives_mining_reward() { - crate::receives_mining_reward::().await; + crate::receives_mining_reward::().await; } #[tokio::test(flavor = "multi_thread")] async fn connect_to_node_get_info() { - crate::connect_to_node_get_info::().await; + crate::connect_to_node_get_info::().await; } #[tokio::test(flavor = "multi_thread")] - async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + async fn send_to_ironwood() { + crate::send_to_pool::(e2e::Pool::Ironwood).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_sapling() { - crate::send_to_pool::(e2e::Pool::Sapling).await; + crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_transparent() { - crate::send_to_pool::(e2e::Pool::Transparent).await; + crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_all() { - crate::send_to_all::().await; + crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] async fn shield_for_validator() { - crate::shield_for_validator::().await; + crate::shield_for_validator::().await; } #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when cheap filler mining lands" )] async fn send_to_transparent_finalization() { - crate::send_to_transparent_finalization::().await; + crate::send_to_transparent_finalization::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mined() { - crate::get_transaction_mined::().await; + crate::get_transaction_mined::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_mempool() { - crate::get_raw_mempool::().await; + crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2608,7 +2563,7 @@ mod zebrad { ignore = "devtool WalletBalance has no unconfirmed_*/confirmed_* fields; balance asserts are commented out — restore + un-ignore when devtool surfaces unconfirmed balances" )] async fn monitor_unverified_mempool() { - crate::monitor_unverified_mempool::().await; + crate::monitor_unverified_mempool::().await; } // No get_mempool_tx here: the state backend returns mempool-tx txids diff --git a/live-tests/e2e/tests/devtool_zcashd.rs b/live-tests/e2e/tests/devtool_zcashd.rs index bcc002bd1..5950d0806 100644 --- a/live-tests/e2e/tests/devtool_zcashd.rs +++ b/live-tests/e2e/tests/devtool_zcashd.rs @@ -10,8 +10,8 @@ //! //! zcashd mines ORCHARD coinbase to `REG_O_ADDR_FROM_ABANDONART` (the abandon-art //! orchard address the devtool faucet owns), so the faucet is funded with no -//! transparent-coinbase shielding — the zcashd matrix is NOT gated on round-2 -//! P1, only on this heights alignment. +//! transparent-coinbase shielding — the zcashd matrix is NOT gated on devtool +//! transparent-coinbase shielding, only on this heights alignment. //! //! If this passes, the `json_server` tests port by swapping their zingolib //! funding for `DevtoolClients` while keeping the zaino-vs-zcashd oracle @@ -22,11 +22,9 @@ // launchers, all gated behind `zcashd_support`. Gate the whole binary so it // compiles out under `--no-default-features` (mirrors the clientless partition's json_server.rs). #![cfg(feature = "zcashd_support")] -#![allow(deprecated)] // FetchService is a deprecated re-export. - use e2e::devtool::DevtoolClients; -use zaino_state::{ChainIndex, FetchService, ZcashIndexer}; -use zaino_testutils::{TestManager, ValidatorKind, ZcashdDualFetchServices}; +use zaino_state::{ChainIndex, ZcashIndexer}; +use zaino_testutils::{Rpc, TestManager, ValidatorKind, ZcashdDualFetchServices}; use zcash_local_net::validator::zcashd::Zcashd; use zebra_chain::subtree::NoteCommitmentSubtreeIndex; use zebra_rpc::client::GetAddressBalanceRequest; @@ -38,13 +36,13 @@ use zebra_rpc::methods::GetAddressTxIdsRequest; /// faucet/recipient wallets against the resulting Zaino, without mining or /// syncing. The zcashd analogue of devtool.rs's `launch_and_build_clients`, /// concrete on zcashd (which has no StateService backend). -async fn launch_zcashd_and_build_clients() -> (TestManager, DevtoolClients) { - let test_manager = TestManager::::launch_mining_to( +async fn launch_zcashd_and_build_clients() -> (TestManager, DevtoolClients) { + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, // ORCHARD &ValidatorKind::Zcashd, None, // network -> Regtest // The heights the devtool wallet accepts (same as the zebrad path). - Some(zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), + Some(zaino_testutils::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), None, // no chain cache: build fresh at these heights true, // enable zaino false, // no json-rpc server @@ -58,6 +56,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -70,7 +69,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager /// height 2). The send/shield analogue of devtool.rs's `launch_and_fund_faucet`. async fn launch_and_fund_zcashd_faucet( orchard_notes: u32, -) -> (TestManager, DevtoolClients) { +) -> (TestManager, DevtoolClients) { let (test_manager, mut clients) = launch_zcashd_and_build_clients().await; test_manager .generate_blocks_and_wait_for_tip(orchard_notes + 1, test_manager.subscriber()) @@ -110,7 +109,7 @@ async fn faucet_receives_zcashd_orchard_reward() { /// of json_server's `create_zcashd_test_manager_and_fetch_services`. async fn create_zcashd_devtool_services() -> (ZcashdDualFetchServices, DevtoolClients) { let services = zaino_testutils::launch_zcashd_dual_fetch_services_at( - zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, + zaino_testutils::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, ) .await; let clients = e2e::devtool::build_clients( @@ -483,8 +482,9 @@ async fn shield_for_validator() { /// Devtool ports of `wallet_to_validator`'s `mod zcashd` send/shield/get-info /// column. Deferred: the heavy finalization send (`sent_to::transparent`'s -/// 99-block mine, round-3 P2), `sent_to::all`, and `monitor_unverified_mempool` -/// (round-3 P3). `send_to_transparent` here is the light send, matching the +/// seam-deep mine, waits on cheap filler mining), `sent_to::all`, and +/// `monitor_unverified_mempool` (unconfirmed mempool balances). +/// `send_to_transparent` here is the light send, matching the /// zebrad devtool port. mod wallet_to_validator { use super::*; @@ -519,13 +519,13 @@ mod wallet_to_validator { /// zcashd analogue of devtool.rs's gated `send_to_transparent_finalization`: /// a transparent send returns the same address txids from the non-finalized - /// chain and after the 99-block advance into the finalized DB. `#[ignore]`d - /// for the same reason — the advance mines orchard coinbase (~99 halo2 - /// proofs) until per-call cheap filler mining (round-3 P2) lands. + /// chain and after the seam-deep advance into the finalized DB. `#[ignore]`d + /// for the same reason — the advance mines orchard coinbase (~100 halo2 + /// proofs) until per-call cheap filler mining lands. #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when cheap filler mining lands" )] async fn send_to_transparent_finalization() { let (mut test_manager, mut clients) = launch_and_fund_zcashd_faucet(1).await; @@ -545,7 +545,9 @@ mod wallet_to_validator { test_manager .generate_blocks_bulk_and_wait_for_tips( - 99, + // Advance past the seam so the send crosses the finalised floor + // (`tip - seam`); a small margin above it keeps the boundary unambiguous. + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5, test_manager.subscriber(), test_manager.subscriber(), ) @@ -567,8 +569,8 @@ mod wallet_to_validator { } /// zcashd port of `sent_to::all` (heavy): one faucet funds a send to all - /// three pools, then a 100-block advance, and each recipient pool reports - /// 250_000. `#[ignore]`d: the 100-block advance mines orchard coinbase + /// three pools, then a seam-deep advance, and each recipient pool reports + /// 250_000. `#[ignore]`d: the seam-deep advance mines orchard coinbase /// (~100 halo2 proofs). The advance is faithful to the original but not /// load-bearing for the per-pool balance asserts (the sends confirm in one /// block), so this could be re-ported light (like the zebrad `send_to_all`) @@ -576,7 +578,7 @@ mod wallet_to_validator { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 100-block orchard advance (~100 halo2 proofs); re-port light or un-ignore with transparent filler (round-3 P2)" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); re-port light or un-ignore with transparent filler" )] async fn send_to_all() { let (mut test_manager, mut clients) = launch_and_fund_zcashd_faucet(3).await; @@ -590,7 +592,8 @@ mod wallet_to_validator { test_manager .generate_blocks_bulk_and_wait_for_tips( - 100, + // Advance past the seam so all three pool sends cross the finalised floor. + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5, test_manager.subscriber(), test_manager.subscriber(), ) diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs new file mode 100644 index 000000000..a5db94ea1 --- /dev/null +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -0,0 +1,482 @@ +//! Wallet-tier predicates across the Orchard→Ironwood activation boundary. +//! +//! Every test here runs a devtool wallet on +//! [`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`] — the hermetic replay of what +//! The Public Testnet did once at height 4,134,000: heights 2 through 5 are +//! Orchard era, [`NU6_3_TRANSITION_BOUNDARY`] (6) onward is Ironwood era. +//! The wallets derive their activation schedule from the running validator +//! (`WalletNetwork::from_validator`, infrastructure ADR 0003), so the +//! fixture heights are typed in exactly one place: the zebrad launch +//! config. Height drift between wallet, indexer, and validator is +//! unrepresentable — zainod adopts the same schedule over +//! `getblockchaininfo` (zaino#1076). +//! +//! # The predicates, and where each era's cell is covered +//! +//! | predicate (wallet-observable) | Orchard era | Ironwood era | +//! |----------------------------------------------|-------------|--------------| +//! | unified-address receipt lands in Orchard | here | false — `devtool.rs` `send_to_ironwood` asserts the Orchard pool stays empty | +//! | unified-address receipt lands in Ironwood | false (pool inactive) | `devtool.rs` `send_to_ironwood` | +//! | shielded-receiver coinbase pays the era pool | here (wallet view); wire tier in `compact_block_wire.rs` | `devtool.rs` `receives_mining_reward`; wire tier in `compact_block_wire.rs` | +//! | an Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a (nothing to exit) | here | +//! | `shield` deposits into the era pool | here | `devtool.rs` `shield_for_validator` | +//! | receipt pool flips between the last Orchard block and the activation block | here (boundary − 1) | here (exactly at the boundary) | +//! | Orchard pool value grows only below the boundary; Ironwood holds value only from it | here (per-height value pools) | here | +//! +//! Era composition of the *served* chain (coinbase routing, compact-block +//! action fields) is covered clientless in +//! `clientless/tests/compact_block_consistency.rs` and over the real gRPC +//! wire in `compact_block_wire.rs`; this file owns the cells that need a +//! wallet on both sides of the boundary. +//! +//! The Public Testnet cannot host the migration cell for us: its pre-NU6.3 +//! epoch closed at height 4,134,000, no new value may enter Orchard from +//! there (post-activation Orchard actions permit only same-receiver change +//! or withdrawal — the cross-address restriction, +//! ), +//! and we hold no pre-activation Orchard TAZ — so this hermetic fixture is +//! the only controlled venue for it. +//! +//! Deferred cells the cross-address restriction implies (chain-walk tier, +//! not wallet tier): the Orchard pool value is non-increasing from the +//! boundary, and post-activation Orchard commitments exist only as +//! same-receiver change — note the Orchard note-commitment tree therefore +//! still grows after activation; do not encode a frozen-finalRoot predicate. +//! +//! Requires a `zcash-devtool` binary built with `--features regtest_support` +//! in `TEST_BINARIES_DIR`/`PATH`, alongside the usual validator binaries. + +use e2e::devtool::DevtoolClients; +use zaino_state::ZcashIndexer; +use zaino_testutils::{ + all_pools_i32, collect_block_range, PollableTip, TestManager, ValidatorKind, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, +}; +use zainodlib::error::IndexerError; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Launch an orchard-receiver-mining zebrad + Zaino on the transition +/// heights, build devtool faucet/recipient wallets (their schedule derived +/// from the launched validator), mine one block, and sync the faucet. The +/// launch itself already leaves the tip at 2 (`TestManager` mines an +/// NU-activation block after block 1), so the helper returns at tip 3 with +/// the faucet holding the Orchard coinbase notes of heights 2 and 3. Tests +/// that need exact boundary positioning mine to absolute heights from the +/// observed tip rather than counting from here. The transition-fixture +/// analogue of `devtool.rs::launch_and_fund_faucet`. +async fn launch_transition_chain_and_fund_faucet( +) -> (TestManager, DevtoolClients) +where + Conn: zaino_testutils::ValidatorConnectionMarker, +{ + let test_manager = TestManager::::launch_mining_to( + zaino_testutils::SHIELDED_FUNDING_POOL, + &ValidatorKind::Zebrad, + None, + Some(ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS), + None, + true, + false, + false, + ) + .await + .expect("launch TestManager"); + + let mut clients = e2e::devtool::build_clients( + test_manager + .zaino_grpc_listen_address + .expect("zaino enabled") + .port(), + &test_manager.local_net, + ) + .await; + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + + (test_manager, clients) +} + +/// The chain value (in zatoshis) of the pool named `pool_id` as of +/// `height`, read from the served verbosity-1 block object — the same +/// per-height `valuePools` a zcashd `getblock` reports. Verbosity 1, not 2: +/// the fetch backend cannot deserialize a verbosity-2 block object (its +/// `tx` entries are maps where zaino-fetch expects txid strings), and the +/// value pools ride along at verbosity 1. +async fn pool_zats_at_height(subscriber: &S, height: u32, pool_id: &str) -> i64 +where + S: ZcashIndexer, + IndexerError: From, +{ + let response = subscriber + .z_get_block(height.to_string(), Some(1)) + .await + .map_err(IndexerError::from) + .expect("z_get_block verbosity 1"); + let zebra_rpc::methods::GetBlock::Object(block) = response else { + panic!("verbosity-1 getblock must return a block object"); + }; + let pools = block + .value_pools() + .as_ref() + .expect("verbosity-1 block object carries value pools"); + pools + .iter() + .find(|pool| pool.id() == pool_id) + .unwrap_or_else(|| panic!("value pools must include {pool_id}")) + .chain_value_zat() + .zatoshis() +} + +/// Orchard-era receipt: with the tip still below the boundary, the faucet's +/// coinbase note is an Orchard note (not Ironwood), and a unified-address +/// send received before the boundary lands in the recipient's Orchard pool +/// with the Ironwood pool exactly empty — the era-mirror of +/// `devtool.rs::send_to_pool(Ironwood)`. +async fn unified_receipt_lands_in_orchard_before_boundary() +where + Conn: zaino_testutils::ValidatorConnectionMarker, +{ + let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; + + // Tip is 3: inside the Orchard era, with room to confirm the send at + // height 4 while staying below the boundary at 6. + let faucet_balance = clients.faucet_balance().await; + assert!( + faucet_balance.orchard_spendable > 0, + "pre-boundary coinbase should be an orchard note, got {faucet_balance:?}" + ); + assert_eq!( + faucet_balance.ironwood_spendable, 0, + "no ironwood note can exist below the boundary, got {faucet_balance:?}" + ); + + let recipient = clients.get_recipient_address("unified").await; + let txid = clients.send_from_faucet(&recipient, 250_000).await; + dbg!(txid); + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + + test_manager.close().await; +} + +/// The ZIP 318 migration shape: an Orchard note minted before the boundary +/// is spent after it, to a unified address generated after activation, and +/// the receipt lands in the Ironwood pool with the recipient's Orchard pool +/// exactly empty. The faucet's Orchard balance must shrink: from the +/// boundary, the cross-address restriction limits each Orchard action to +/// same-receiver change or withdrawal +/// (), +/// so a genuine Orchard spend nets sent-amount-plus-fee out of the pool even +/// when change returns to the spent note's address. +async fn orchard_note_spends_to_ironwood_across_boundary() +where + Conn: zaino_testutils::ValidatorConnectionMarker, +{ + let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; + + let pre_boundary_balance = clients.faucet_balance().await; + assert!( + pre_boundary_balance.orchard_spendable > 0, + "pre-boundary coinbase should be an orchard note, got {pre_boundary_balance:?}" + ); + + // Mine to the boundary itself, from the observed tip rather than a + // hand-count. The blocks below the boundary add more Orchard coinbase + // notes; the boundary block is the first Ironwood-era block, and its + // coinbase is the faucet's first Ironwood note. + let tip = u32::try_from(test_manager.subscriber().tip_height().await) + .expect("regtest tips fit in u32"); + test_manager + .generate_blocks_and_wait_for_tip( + NU6_3_TRANSITION_BOUNDARY + .checked_sub(tip) + .expect("the launch preamble must leave room below the boundary"), + test_manager.subscriber(), + ) + .await; + clients.sync_faucet().await; + let crossed_balance = clients.faucet_balance().await; + let orchard_before_send = crossed_balance.orchard_spendable; + assert!( + crossed_balance.ironwood_spendable > 0, + "the boundary coinbase should be an ironwood note, got {crossed_balance:?}" + ); + + // Generated only now — after activation — per the migration shape under + // test: old-pool note, new-era address. + let recipient = clients.get_recipient_address("unified").await; + let txid = clients.send_from_faucet(&recipient, 250_000).await; + dbg!(txid); + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 0); + + // Pins that the send actually exited the Orchard pool rather than + // spending the boundary-height Ironwood coinbase — the note-selection + // question the first live runs of this suite exist to answer. + assert!( + clients.faucet_balance().await.orchard_spendable < orchard_before_send, + "the migration send must spend an orchard note" + ); + + // Chain-tier consequences, read from the served per-height value pools. + // The Ironwood pool holds no value below the activation height; the + // Orchard pool grows only below it (its coinbases), holds exactly + // steady across the boundary edge (the activation block's coinbase pays + // Ironwood, and no new value may enter Orchard), and shrinks at the + // migration block by the withdrawn amount plus fee. + let boundary = NU6_3_TRANSITION_BOUNDARY; + let migration_height = boundary + 1; + let subscriber = test_manager.subscriber(); + let mut orchard_at = Vec::new(); + for height in 1..=migration_height { + let ironwood = pool_zats_at_height(subscriber, height, "ironwood").await; + let orchard = pool_zats_at_height(subscriber, height, "orchard").await; + if height < boundary { + assert_eq!( + ironwood, 0, + "the ironwood pool must hold no value at height {height}, below the boundary" + ); + } + orchard_at.push(orchard); + } + for (index, orchard) in orchard_at.iter().enumerate() { + let height = index + 1; + let ironwood = pool_zats_at_height(subscriber, height as u32, "ironwood").await; + eprintln!("pool values at height {height}: orchard={orchard} ironwood={ironwood}"); + } + let orchard = |height: u32| orchard_at[height as usize - 1]; + for height in 2..boundary { + assert!( + orchard(height) > orchard(height - 1), + "each pre-boundary coinbase must grow the orchard pool (height {height}: {} after {})", + orchard(height), + orchard(height - 1) + ); + } + assert_eq!( + orchard(boundary), + orchard(boundary - 1), + "the orchard pool must hold exactly steady across the boundary edge" + ); + assert!( + pool_zats_at_height(subscriber, boundary, "ironwood").await > 0, + "the activation block's coinbase must give the ironwood pool its first value" + ); + assert!( + orchard(migration_height) < orchard(boundary), + "the migration block must shrink the orchard pool ({} at the boundary, {} at the migration block)", + orchard(boundary), + orchard(migration_height) + ); + + test_manager.close().await; +} + +/// The receipt pool flips between adjacent blocks: a send confirmed in the +/// last Orchard-era block (boundary − 1) lands in Orchard, and a send built +/// one block below the boundary but confirmed in the activation block +/// itself lands in Ironwood. The second send also pins the wallet's era +/// anticipation at the edge: it is constructed while the tip is still +/// Orchard-era, targeting the first Ironwood-era height, and it spends an +/// Orchard note (the faucet holds no Ironwood note until the activation +/// block is mined) — a migration transaction in the activation block. +async fn receipts_flip_pools_exactly_at_the_boundary() +where + Conn: zaino_testutils::ValidatorConnectionMarker, +{ + let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; + + // Position the tip at exactly boundary − 2, from the observed tip + // rather than a hand-count, so the two sends below confirm at exactly + // boundary − 1 and the boundary. + let tip = u32::try_from(test_manager.subscriber().tip_height().await) + .expect("regtest tips fit in u32"); + test_manager + .generate_blocks_and_wait_for_tip( + NU6_3_TRANSITION_BOUNDARY + .checked_sub(2 + tip) + .expect("the launch preamble must leave room below the boundary"), + test_manager.subscriber(), + ) + .await; + clients.sync_faucet().await; + + let recipient = clients.get_recipient_address("unified").await; + + // Confirmed in block 5: the last Orchard-era block. + let last_orchard_txid = clients.send_from_faucet(&recipient, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + clients.sync_recipient().await; + let balance = clients.recipient_balance().await; + assert_eq!( + e2e::Pool::Orchard.spendable_balance(&balance), + 250_000, + "receipt confirmed at boundary - 1 must be orchard, got {balance:?}" + ); + assert_eq!( + e2e::Pool::Ironwood.spendable_balance(&balance), + 0, + "no ironwood receipt below the boundary, got {balance:?}" + ); + + // Built at tip boundary − 1, confirmed in block 6: the activation block. + let first_ironwood_txid = clients.send_from_faucet(&recipient, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!( + e2e::Pool::Orchard.spendable_balance(&balance), + 250_000, + "the pre-boundary receipt must survive the flip unchanged" + ); + + // Era composition of the two served edge blocks. + let subscriber = test_manager.subscriber(); + let boundary = u64::from(NU6_3_TRANSITION_BOUNDARY); + let blocks = collect_block_range(subscriber, boundary - 1, boundary, all_pools_i32()).await; + let [last_orchard_block, activation_block] = blocks.as_slice() else { + panic!("expected exactly the two edge blocks, got {}", blocks.len()); + }; + assert_eq!(last_orchard_block.height, boundary - 1); + assert_eq!(activation_block.height, boundary); + let last_orchard_txid = e2e::devtool::txid_from_devtool(&last_orchard_txid); + e2e::assert_pool_present(last_orchard_block, &last_orchard_txid, e2e::Pool::Orchard); + e2e::assert_pool_absent(last_orchard_block, &last_orchard_txid, e2e::Pool::Ironwood); + let first_ironwood_txid = e2e::devtool::txid_from_devtool(&first_ironwood_txid); + e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Ironwood); + // The second send is itself migration-shaped: built one block below the + // boundary, it spends an Orchard note (the faucet's only spendable kind + // at that tip), so its compact form carries the Orchard spend's data + // alongside the Ironwood receipt. The receipt's pool routing is what + // flips at the boundary, and that is asserted at the wallet tier above. + e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Orchard); + + test_manager.close().await; +} + +/// Below the boundary, `shield` deposits into the Orchard pool: the faucet +/// funds the recipient's transparent address, the recipient shields, and +/// the shielded balance (net of the ZIP-317 fee, mirroring +/// `devtool.rs::shield_for_validator`) lands in Orchard with the Ironwood +/// pool exactly empty — the era-mirror of the Ironwood-era shield cell. +async fn shield_deposits_to_orchard_before_boundary() +where + Conn: zaino_testutils::ValidatorConnectionMarker, +{ + let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; + + // Tip is 3; the transparent receipt confirms at 4 and the shield at 5, + // all below the boundary at 6. + let recipient_t = clients.get_recipient_address("transparent").await; + clients.send_from_faucet(&recipient_t, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + assert_eq!( + e2e::Pool::Transparent.spendable_balance(&clients.recipient_balance().await), + 250_000 + ); + + clients.shield_recipient().await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 235_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + + test_manager.close().await; +} + +mod zebrad { + mod fetch_service { + use zaino_testutils::Rpc; + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn unified_receipt_lands_in_orchard_before_boundary() { + crate::unified_receipt_lands_in_orchard_before_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn orchard_note_spends_to_ironwood_across_boundary() { + crate::orchard_note_spends_to_ironwood_across_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn receipts_flip_pools_exactly_at_the_boundary() { + crate::receipts_flip_pools_exactly_at_the_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn shield_deposits_to_orchard_before_boundary() { + crate::shield_deposits_to_orchard_before_boundary::().await; + } + } + + mod state_service { + use zaino_testutils::Direct; + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn unified_receipt_lands_in_orchard_before_boundary() { + crate::unified_receipt_lands_in_orchard_before_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn orchard_note_spends_to_ironwood_across_boundary() { + crate::orchard_note_spends_to_ironwood_across_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn receipts_flip_pools_exactly_at_the_boundary() { + crate::receipts_flip_pools_exactly_at_the_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn shield_deposits_to_orchard_before_boundary() { + crate::shield_deposits_to_orchard_before_boundary::().await; + } + } +} diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index 83dc4ac47..ec18a2ad3 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -14,11 +14,9 @@ use zaino_state::read_u64_le; use zaino_state::write_u32_le; use zaino_state::write_u64_le; use zaino_state::CompactSize; -#[allow(deprecated)] -use zaino_state::StateService; use zaino_state::ZcashIndexer; use zaino_state::{ChainWork, IndexedBlock}; -use zaino_testutils::{TestManager, ValidatorKind}; +use zaino_testutils::{Direct, TestManager, ValidatorKind}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::serialization::{ZcashDeserialize, ZcashSerialize}; use zebra_rpc::methods::GetAddressUtxos; @@ -40,15 +38,15 @@ macro_rules! expected_read_response { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "Not a test: builds test-vector data for zaino_state::chain_index unit tests. Also funds via transparent-coinbase shielding (round-2 P1) — un-ignore to regenerate vectors once devtool can shield its own transparent coinbase (tracked by tests/devtool.rs's address_deltas)." + ignore = "Not a test: builds test-vector data for zaino_state::chain_index unit tests. Also funds via transparent-coinbase shielding — un-ignore to regenerate vectors once devtool can shield its own transparent coinbase (tracked by tests/devtool.rs's address_deltas)." )] #[allow(deprecated)] async fn create_200_block_regtest_chain_vectors() { // The committed unit-test vectors encode a mixed-pool chain built by // repeatedly shielding transparent coinbase — mining must stay transparent // so regenerated vectors keep that shape. - let mut test_manager = TestManager::::launch_mining_to( - zaino_testutils::PoolType::Transparent, + let mut test_manager = TestManager::::launch_mining_to( + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, None, @@ -67,6 +65,7 @@ async fn create_200_block_regtest_chain_vectors() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -81,7 +80,7 @@ async fn create_200_block_regtest_chain_vectors() { // *** Mine past coinbase maturity, shield the first reward, and mine it in *** // Mature the faucet's transparent coinbase (100-block maturity) and shield // it. Devtool analogue of `shield_faucet_rounds`; requires the devtool wallet - // to spend its own transparent coinbase (round-2 P1, see #[ignore]). Mine + // to spend its own transparent coinbase (see #[ignore]). Mine // generously (150) so the earliest coinbase — a few blocks past genesis — is // comfortably mature before the shield, regardless of startup height. test_manager @@ -233,6 +232,7 @@ async fn create_200_block_regtest_chain_vectors() { let mut parent_chain_work: Option = None; let mut parent_block_sapling_tree_size: u32 = 0; let mut parent_block_orchard_tree_size: u32 = 0; + let mut parent_block_ironwood_tree_size: u32 = 0; for height in 0..=chain_height.0 { let (chain_block, zebra_block, block_roots, block_treestate) = { @@ -242,7 +242,7 @@ async fn create_200_block_regtest_chain_vectors() { .await .and_then(|response| match response { zebra_rpc::methods::GetBlock::Raw(_) => { - Err(zaino_state::StateServiceError::Custom( + Err(zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Raw` type, expected only `Object` types." .to_string(), )) @@ -254,7 +254,7 @@ async fn create_200_block_regtest_chain_vectors() { match item { GetBlockTransaction::Hash(h) => Ok(h.0.to_vec()), GetBlockTransaction::Object(_) => Err( - zaino_state::StateServiceError::Custom( + zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Object` type, expected only `Hash` types." .to_string(), ), @@ -273,7 +273,7 @@ async fn create_200_block_regtest_chain_vectors() { .await .and_then(|response| match response { zebra_rpc::methods::GetBlock::Object { .. } => { - Err(zaino_state::StateServiceError::Custom( + Err(zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Object` type, expected only `Raw` types." .to_string(), )) @@ -282,7 +282,7 @@ async fn create_200_block_regtest_chain_vectors() { }) .unwrap(); - let mut state = state_service_subscriber.read_state_service.clone(); + let mut state = state_service_subscriber.read_state_service(); let (sapling_root, orchard_root) = { let (sapling_tree_response, orchard_tree_response) = futures::future::join( state.clone().call(zebra_state::ReadRequest::SaplingTree( @@ -318,7 +318,7 @@ async fn create_200_block_regtest_chain_vectors() { }; let sapling_treestate = match zebra_chain::parameters::NetworkUpgrade::Sapling - .activation_height(&state_service_subscriber.network().to_zebra_network()) + .activation_height(&state_service_subscriber.network()) { Some(activation_height) if height >= activation_height.0 => Some( state @@ -341,7 +341,7 @@ async fn create_200_block_regtest_chain_vectors() { }) .unwrap(); let orchard_treestate = match zebra_chain::parameters::NetworkUpgrade::Nu5 - .activation_height(&state_service_subscriber.network().to_zebra_network()) + .activation_height(&state_service_subscriber.network()) { Some(activation_height) if height >= activation_height.0 => Some( state @@ -376,8 +376,10 @@ async fn create_200_block_regtest_chain_vectors() { parent_chain_work, sapling_root.0.into(), orchard_root.0.into(), + None, parent_block_sapling_tree_size, parent_block_orchard_tree_size, + parent_block_ironwood_tree_size, )) .unwrap(); @@ -399,6 +401,7 @@ async fn create_200_block_regtest_chain_vectors() { // Update parent block parent_block_sapling_tree_size = chain_block.commitment_tree_data().sizes().sapling(); parent_block_orchard_tree_size = chain_block.commitment_tree_data().sizes().orchard(); + parent_block_ironwood_tree_size = chain_block.commitment_tree_data().sizes().ironwood(); parent_chain_work = Some(*chain_block.chainwork()); data.push((height, zebra_block, block_roots, block_treestate)); diff --git a/live-tests/test_environment/Containerfile b/live-tests/test_environment/Containerfile index 278d71356..a6a19c7fd 100644 --- a/live-tests/test_environment/Containerfile +++ b/live-tests/test_environment/Containerfile @@ -29,6 +29,13 @@ FROM docker.io/zodlinc/zcash:v${ZCASH_VERSION} AS zcashd-downloader FROM docker.io/library/rust:${RUST_VERSION}-trixie AS zebra-builder ARG ZEBRA_VERSION +# Git ref resolved from ZEBRA_VERSION by the workbench `get-zebra-git-ref` +# bin: ZEBRA_VERSION is the Docker Hub tag (bare, e.g. "6.0.0-rc.0") while +# zebra's git release tags carry a `v` prefix. Both build entry points +# (build-image.sh, build-n-push-ci-image.yaml) resolve and pass it; the +# bare-ZEBRA_VERSION fallback only serves hand-run builds that pin a branch +# or SHA. +ARG ZEBRA_GIT_REF # Versions pinned (DL3008) to the candidates in rust:1.95.0-trixie; bump # together with the base image (query with `apt-cache policy `). # --no-install-recommends (DL3015): ca-certificates is already present in the @@ -39,12 +46,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ clang=1:19.0-63 \ cmake=3.31.6-2 \ pkg-config=1.8.1-4 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ libstdc++6=14.2.0-19 WORKDIR /zebra # Single RUN (DL3059): clone, checkout, build, and stage the binary. RUN git clone https://github.com/ZcashFoundation/zebra . \ - && git checkout "${ZEBRA_VERSION}" \ + && git checkout "${ZEBRA_GIT_REF:-$ZEBRA_VERSION}" \ && cargo build --release --bin zebrad \ && cp target/release/zebrad /tmp/zebrad @@ -64,7 +71,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ autoconf=2.72-3.1 \ libtool=2.5.4-4 \ bsdmainutils=12.1.8 \ - curl=8.14.1-2+deb13u3 \ + curl=8.14.1-2+deb13u4 \ ca-certificates=20250419 \ netbase=6.5 @@ -92,7 +99,7 @@ ARG DEVTOOL_REV RUN apt-get update && apt-get install -y --no-install-recommends \ git=1:2.47.3-0+deb13u1 \ pkg-config=1.8.1-4 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ libsqlite3-dev=3.46.1-7+deb13u1 WORKDIR /devtool # Single RUN (DL3059): clone, checkout, build, and stage the binary. The @@ -133,12 +140,12 @@ LABEL org.zaino.test-artifacts.versions.rust="${RUST_VERSION}" # Versions pinned (DL3008) to the candidates in rust:1.95.0-trixie; bump # together with the base image (query with `apt-cache policy `). RUN apt-get update && apt-get install -y --no-install-recommends \ - curl=8.14.1-2+deb13u3 \ + curl=8.14.1-2+deb13u4 \ libclang-dev=1:19.0-63 \ build-essential=12.12 \ cmake=3.31.6-2 \ librocksdb-dev=9.10.0-1+b1 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ && rm -rf /var/lib/apt/lists/* # Create container_user. The GID may already belong to a base-image group diff --git a/live-tests/zaino-testutils/Cargo.toml b/live-tests/zaino-testutils/Cargo.toml index 38f2ef883..09b132e9f 100644 --- a/live-tests/zaino-testutils/Cargo.toml +++ b/live-tests/zaino-testutils/Cargo.toml @@ -65,5 +65,7 @@ futures = { workspace = true } http = { workspace = true } once_cell = { workspace = true } tokio = { workspace = true } -tonic.workspace = true tracing.workspace = true + +[dev-dependencies] +tonic.workspace = true diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 6ebb6298f..78ecf274c 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -7,6 +7,7 @@ use futures::StreamExt as _; use once_cell::sync::Lazy; use std::{ future::Future, + marker::PhantomData, net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, }; @@ -14,7 +15,7 @@ use std::{ use tonic::transport::Channel; use tracing::{debug, info, instrument}; use zaino_common::{ - network::{ActivationHeights, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}, + network::ActivationHeights, probing::{Liveness, Readiness}, status::Status, validator::ValidatorConfig, @@ -24,15 +25,12 @@ use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConne use zaino_proto::proto::compact_formats::CompactBlock; use zaino_proto::proto::service::{BlockId, BlockRange}; use zaino_serve::server::config::{GrpcServerConfig, JsonRpcServerConfig}; -#[cfg(feature = "zcashd_support")] -use zaino_state::BackendType; use zaino_state::{ - BlockchainSource, ChainIndex, FetchServiceSubscriber, LightWalletIndexer, LightWalletService, - NodeBackedChainIndexSubscriber, StateServiceSubscriber, ZcashIndexer, ZcashService, + BlockchainSource, ChainIndex, LightWalletIndexer, NodeBackedChainIndexSubscriber, + NodeBackedIndexerService, NodeBackedIndexerServiceConfig, NodeBackedIndexerServiceSubscriber, + ZcashService, }; -#[allow(deprecated)] -use zaino_state::{FetchService, FetchServiceConfig, StateService, StateServiceConfig}; -use zainodlib::{config::ZainodConfig, error::IndexerError, indexer::Indexer}; +use zainodlib::{config::BackendType, error::IndexerError, indexer::Indexer}; pub use zcash_local_net as services; use zcash_local_net::error::LaunchError; #[cfg(feature = "zcashd_support")] @@ -40,7 +38,7 @@ use zcash_local_net::validator::zcashd::{Zcashd, ZcashdConfig}; use zcash_local_net::validator::zebrad::{Zebrad, ZebradConfig}; pub use zcash_local_net::validator::Validator; use zcash_local_net::validator::ValidatorConfig as _; -pub use zcash_local_net::PoolType; +pub use zcash_local_net::MinerPool; use zcash_local_net::{logs::LogsToStdoutAndStderr, process::Process}; use zebra_chain::parameters::NetworkKind; use zebra_rpc::methods::GetInfo; @@ -67,7 +65,88 @@ macro_rules! validator_tests { }; } -/// All three value pools as the `i32`s a `get_block_range` request carries — +/// The canonical regtest activation heights the harness *launches* validators +/// with when a test names no others (everything through NU6.3 active from +/// height 2). This is validator-launch vocabulary — the configuring side of +/// the truth source — and is never zainod's own schedule: zainod's config +/// carries only a network kind, and both backends adopt the runtime schedule +/// from the running validator's `getblockchaininfo.upgrades` (zaino#1076). +/// It matches zcash_local_net's `supported_regtest_activation_heights`, which +/// the devtool wallet client currently requires (zaino#1368). +pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(2), + nu7: None, +}; + +/// Orchard-era-only fixture: every upgrade through NU6.2 active from height 2 and +/// NU6.3 never activating, so coinbases stay Orchard for the whole chain. For +/// client-free launches only — the zcash-devtool wallet hardcodes the canonical +/// regtest set (NU6.3 at 2), so wallet-built transactions on this fixture would fail +/// consensus branch-id validation. +pub const ORCHARD_ONLY_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: None, + nu7: None, +}; + +/// Zebrad regtest heights with every upgrade through NU6.3 active from height 2, so +/// generated blocks carry V6 coinbases from the first post-genesis era. +pub const IRONWOOD_ONLY_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(2), + nu7: None, +}; + +/// Keep in sync with [`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`]. +pub const NU6_3_TRANSITION_BOUNDARY: u32 = 6; + +/// NU5 era from height 2, NU6.3 from [`NU6_3_TRANSITION_BOUNDARY`]: the same +/// orchard-receiver miner first produces Orchard coinbases, then — at the activation +/// boundary — Ironwood ones. +pub const ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(NU6_3_TRANSITION_BOUNDARY), + nu7: None, +}; + +/// All four value pools as the `i32`s a `get_block_range` request carries — /// the "request everything" pool filter. pub fn all_pools_i32() -> Vec { use zaino_proto::proto::service::PoolType; @@ -75,14 +154,20 @@ pub fn all_pools_i32() -> Vec { PoolType::Transparent as i32, PoolType::Sapling as i32, PoolType::Orchard as i32, + PoolType::Ironwood as i32, ] } /// The shielded pools as request `i32`s — what `get_block_range` defaults to -/// when a request names no pools. +/// when a request names no pools (`PoolTypeFilter::default()`: every shielded +/// pool including Ironwood, transparent excluded). pub fn shielded_pools_i32() -> Vec { use zaino_proto::proto::service::PoolType; - vec![PoolType::Sapling as i32, PoolType::Orchard as i32] + vec![ + PoolType::Sapling as i32, + PoolType::Orchard as i32, + PoolType::Ironwood as i32, + ] } /// Collect a `get_block_range` query over heights `[start, end]` for the given @@ -221,20 +306,11 @@ pub trait PollableTip: Status + Sync { fn tip_height(&self) -> impl std::future::Future; } -impl PollableTip for FetchServiceSubscriber { - async fn tip_height(&self) -> u64 { - self.get_latest_block() - .await - .expect("PollableTip: FetchServiceSubscriber::get_latest_block failed") - .height - } -} - -impl PollableTip for StateServiceSubscriber { +impl PollableTip for NodeBackedIndexerServiceSubscriber { async fn tip_height(&self) -> u64 { self.get_latest_block() .await - .expect("PollableTip: StateServiceSubscriber::get_latest_block failed") + .expect("PollableTip: NodeBackedIndexerServiceSubscriber::get_latest_block failed") .height } } @@ -254,39 +330,32 @@ impl PollableTip for NodeBackedChainIndexSubscriber` -/// - `Service::Subscriber: PollableTip` -/// -/// **Not** bundled: the reverse bound -/// `IndexerError: From`. Rust does not -/// propagate non-`Self` bounds declared in a trait's `where`-clause -/// through a `T: TestService` constraint, so call sites that touch -/// `TestManager::launch` (or anything else exercising -/// `Indexer::launch_inner`'s `?` propagation) must still restate that -/// one bound explicitly. Everything else collapses to `TestService`. -pub trait TestService: - LightWalletService, Subscriber: PollableTip> - + Send - + Sync - + 'static -{ +/// Replaces the former `FetchService` / `StateService` `Service` type parameters: +/// there is now one service, parameterized by connection. Implementors are the +/// zero-sized markers [`Rpc`] and [`Direct`]. +pub trait ValidatorConnectionMarker: Send + Sync + 'static { + /// The on-disk backend selector this connection maps to. + const BACKEND: BackendType; } -impl TestService for T where - T: LightWalletService< - Config: TryFrom, - Subscriber: PollableTip, - > + Send - + Sync - + 'static -{ +/// JSON-RPC validator connection (formerly `FetchService`). +#[derive(Debug, Clone, Copy)] +pub struct Rpc; + +impl ValidatorConnectionMarker for Rpc { + const BACKEND: BackendType = BackendType::Rpc; +} + +/// Direct Zebra `ReadStateService` validator connection (formerly `StateService`). +#[derive(Debug, Clone, Copy)] +pub struct Direct; + +impl ValidatorConnectionMarker for Direct { + const BACKEND: BackendType = BackendType::Direct; } // temporary until activation heights are unified to zebra-chain type. @@ -323,6 +392,52 @@ pub fn local_network_from_activation_heights( nu6_2: activation_heights .nu6_2 .map(zcash_protocol::consensus::BlockHeight::from), + nu6_3: activation_heights + .nu6_3 + .map(zcash_protocol::consensus::BlockHeight::from), + } +} + +// Conversions at the zcash_local_net boundary. Named functions rather than +// From impls: the orphan rule forbids the impls here, and zaino-common must +// not depend on the harness vocabulary (it reaches us only through +// zcash_local_net's re-exports). +/// Convert zaino activation heights into the `zcash_local_net` activation heights type. +pub fn to_local_net_activation_heights( + activation_heights: &ActivationHeights, +) -> zcash_local_net::protocol::ActivationHeights { + zcash_local_net::protocol::ActivationHeights::builder() + .set_overwinter(activation_heights.overwinter) + .set_sapling(activation_heights.sapling) + .set_blossom(activation_heights.blossom) + .set_heartwood(activation_heights.heartwood) + .set_canopy(activation_heights.canopy) + .set_nu5(activation_heights.nu5) + .set_nu6(activation_heights.nu6) + .set_nu6_1(activation_heights.nu6_1) + .set_nu6_2(activation_heights.nu6_2) + .set_nu6_3(activation_heights.nu6_3) + .set_nu7(activation_heights.nu7) + .build() +} + +/// Convert the `zcash_local_net` activation heights type into zaino activation heights. +pub fn from_local_net_activation_heights( + activation_heights: &zcash_local_net::protocol::ActivationHeights, +) -> ActivationHeights { + ActivationHeights { + before_overwinter: activation_heights.overwinter(), + overwinter: activation_heights.overwinter(), + sapling: activation_heights.sapling(), + blossom: activation_heights.blossom(), + heartwood: activation_heights.heartwood(), + canopy: activation_heights.canopy(), + nu5: activation_heights.nu5(), + nu6: activation_heights.nu6(), + nu6_1: activation_heights.nu6_1(), + nu6_2: activation_heights.nu6_2(), + nu6_3: activation_heights.nu6_3(), + nu7: activation_heights.nu7(), } } @@ -358,8 +473,8 @@ pub static ZEBRAD_CHAIN_CACHE_DIR: Lazy> = Lazy::new(|| { Some(workspace_root_path.join("live-tests/chain_cache/client_rpc_tests_large")) }); -/// Path for the Zebra chain cache in the user's home directory. -pub static ZEBRAD_TESTNET_CACHE_DIR: Lazy> = Lazy::new(|| { +/// Path for the Zebra chain cache of The Public Testnet in the user's home directory. +pub static ZEBRAD_THE_PUB_TESTNET_CACHE_DIR: Lazy> = Lazy::new(|| { let home_path = PathBuf::from(std::env::var("HOME").unwrap()); Some(home_path.join(".cache/zebra")) }); @@ -399,7 +514,11 @@ pub enum ValidatorTestConfig { } /// Configuration data for Zaino Tests. -pub struct TestManager { +/// +/// Generic over the validator control plane `C` and the [`ValidatorConnectionMarker`] +/// `Conn` — the validator connection ([`Rpc`] or [`Direct`]) the single +/// [`NodeBackedIndexerService`] uses. +pub struct TestManager { /// Control plane for a validator pub local_net: C, /// Data directory for the validator. @@ -417,9 +536,11 @@ pub struct TestManager, /// Service subscriber. - pub service_subscriber: Option, + pub service_subscriber: Option, /// JsonRPC server cookie dir. pub json_server_cookie_dir: Option, + /// Selected validator connection marker. + _connection: PhantomData, } /// Needed validator functionality that is not implemented in infrastructure @@ -488,34 +609,32 @@ impl ValidatorExt for Zcashd { /// /// This constant is the single upgrade point for shielded funding: when the /// next shielded pool (ironwood) becomes minable, only this value changes. -pub const SHIELDED_FUNDING_POOL: PoolType = PoolType::ORCHARD; +pub const SHIELDED_FUNDING_POOL: MinerPool = MinerPool::Orchard; /// The pool a validator mines to when the session doesn't opt into /// [`SHIELDED_FUNDING_POOL`]: transparent for zebrad (cheapest block /// templates — a shielded miner address would cost a halo2 proof per block), -/// ORCHARD for zcashd (its historical setting; the cached launch reward +/// Orchard for zcashd (its historical setting; the cached launch reward /// funds wallets without extra mining). -pub fn default_mining_pool(validator: &ValidatorKind) -> PoolType { +pub fn default_mining_pool(validator: &ValidatorKind) -> MinerPool { if validator == &ValidatorKind::Zebrad { - PoolType::Transparent + MinerPool::Transparent } else { - PoolType::ORCHARD + MinerPool::Orchard } } -impl TestManager +impl TestManager where C: ValidatorExt, - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: ValidatorConnectionMarker, { /// Returns the service subscriber, panicking if zaino wasn't enabled at launch. /// /// Convenience for tests that always pass `enable_zaino: true` and want to /// hand the subscriber to [`Self::generate_blocks_and_wait_for_tip`] without /// repeating `.service_subscriber.as_ref().unwrap()` at every call site. - pub fn subscriber(&self) -> &Service::Subscriber { + pub fn subscriber(&self) -> &NodeBackedIndexerServiceSubscriber { self.service_subscriber .as_ref() .expect("TestManager::subscriber called but service_subscriber is None (zaino disabled at launch?)") @@ -589,7 +708,7 @@ where fields(validator = ?validator, network = ?network, mine_to_pool = ?mine_to_pool, enable_zaino, enable_clients) )] pub async fn launch_mining_to( - mine_to_pool: PoolType, + mine_to_pool: MinerPool, validator: &ValidatorKind, network: Option, activation_heights: Option, @@ -599,9 +718,9 @@ where enable_clients: bool, ) -> Result { #[cfg(feature = "zcashd_support")] - if (validator == &ValidatorKind::Zcashd) && (Service::BACKEND_TYPE == BackendType::State) { + if (validator == &ValidatorKind::Zcashd) && (Conn::BACKEND == BackendType::Direct) { return Err(std::io::Error::other( - "Cannot use state backend with zcashd.", + "Cannot use the direct (ReadStateService) connection with zcashd.", )); } zaino_common::logging::try_init(); @@ -609,8 +728,17 @@ where let activation_heights = activation_heights.unwrap_or_else(|| validator.default_activation_heights()); let network_kind = network.unwrap_or(NetworkKind::Regtest); - let zaino_network_kind = - Network::from_network_kind_and_activation_heights(&network_kind, &activation_heights); + // The validator is the single source of truth for activation heights + // (zaino#1076). zainod's config is a payload-free network kind: the + // fixture heights configure the validator alone, and zainod adopts + // the real schedule from the validator at spawn. Every + // custom-heights launch is therefore a standing regression test of + // that adoption. + let zaino_network_kind = match network_kind { + NetworkKind::Mainnet => Network::Mainnet, + NetworkKind::Testnet => Network::PubTestnet, + NetworkKind::Regtest => Network::Regtest, + }; if enable_clients && !enable_zaino { return Err(std::io::Error::other( @@ -621,7 +749,11 @@ where // Launch LocalNet: let mut config = C::Config::default(); - config.set_test_parameters(mine_to_pool, activation_heights.into(), chain_cache.clone()); + config.set_test_parameters( + mine_to_pool, + to_local_net_activation_heights(&activation_heights), + chain_cache.clone(), + ); debug!("[TEST] Launching validator"); let (local_net, validator_settings) = C::launch_validator_and_return_config(config) @@ -677,8 +809,7 @@ where "[TEST] Launching Zaino indexer" ); let indexer_config = zainodlib::config::ZainodConfig { - // TODO: Make configurable. - backend: Service::BACKEND_TYPE, + backend: Conn::BACKEND, json_server_settings: if enable_zaino_jsonrpc_server { Some(JsonRpcServerConfig { json_rpc_listen_address: zaino_json_listen_address, @@ -707,15 +838,16 @@ where metrics_endpoint: None, }; - let (handle, service_subscriber) = Indexer::::launch_inner_with_listeners( - Service::Config::try_from(indexer_config.clone()) - .expect("Failed to convert ZainodConfig to service config"), - indexer_config, - zaino_grpc_listener, - zaino_json_listener, - ) - .await - .unwrap(); + let (handle, service_subscriber) = + Indexer::::launch_inner_with_listeners( + NodeBackedIndexerServiceConfig::try_from(indexer_config.clone()) + .expect("Failed to convert ZainodConfig to service config"), + indexer_config, + zaino_grpc_listener, + zaino_json_listener, + ) + .await + .unwrap(); ( Some(handle), @@ -752,6 +884,7 @@ where zaino_grpc_listen_address, service_subscriber: zaino_service_subscriber, json_server_cookie_dir: zaino_json_server_cookie_dir, + _connection: PhantomData, }; test_manager.activate_nu5_nu6(enable_zaino).await; @@ -938,25 +1071,27 @@ where } } -/// Handles from [`launch_state_and_fetch_services`]: a state-backend -/// [`TestManager`] plus standalone fetch and state services pointed at the -/// same validator. The owned services exist only to keep their subscribers -/// alive — tests interact through the manager and the subscribers. -#[allow(deprecated)] +/// Handles from [`launch_state_and_fetch_services`]: a direct-connection +/// [`TestManager`] plus standalone `Rpc`- and `Direct`-connection +/// [`NodeBackedIndexerService`]s pointed at the same validator. The owned services +/// exist only to keep their subscribers alive — tests interact through the manager and +/// the subscribers. +/// +/// The `fetch_*` fields carry the `Rpc` (JSON-RPC) connection; the `state_*` fields carry +/// the `Direct` (`ReadStateService`) connection. pub struct StateAndFetchServices { - /// The launched validator + Zaino test manager. - pub test_manager: TestManager, - /// Owned fetch service; keeps `fetch_subscriber` alive. - pub fetch_service: FetchService, - /// Subscriber to the standalone fetch service's chain index. - pub fetch_subscriber: FetchServiceSubscriber, - /// Owned state service; keeps `state_subscriber` alive. - pub state_service: StateService, - /// Subscriber to the standalone state service's chain index. - pub state_subscriber: StateServiceSubscriber, + /// The launched validator + Zaino test manager (direct connection). + pub test_manager: TestManager, + /// Owned `Rpc`-connection service; keeps `fetch_subscriber` alive. + pub fetch_service: NodeBackedIndexerService, + /// Subscriber to the standalone `Rpc`-connection service's chain index. + pub fetch_subscriber: NodeBackedIndexerServiceSubscriber, + /// Owned `Direct`-connection service; keeps `state_subscriber` alive. + pub state_service: NodeBackedIndexerService, + /// Subscriber to the standalone `Direct`-connection service's chain index. + pub state_subscriber: NodeBackedIndexerServiceSubscriber, } -#[allow(deprecated)] impl StateAndFetchServices { /// Mine `n` blocks and wait for both the fetch and state subscribers to /// observe the new tip. @@ -975,7 +1110,6 @@ impl StateAndFetchServices { /// harness used by both the clientless and e2e live-test partitions. /// Wallet callers wrap this and additionally build lightclients from the /// returned manager's gRPC address. -#[allow(deprecated)] pub async fn launch_state_and_fetch_services( validator: &ValidatorKind, chain_cache: Option, @@ -996,15 +1130,14 @@ pub async fn launch_state_and_fetch_services( /// caller instead of [`default_mining_pool`]: [`SHIELDED_FUNDING_POOL`] for /// sessions funding wallets from coinbase, or a pinned pool for tests whose /// subject is the miner's coinbase footprint. -#[allow(deprecated)] pub async fn launch_state_and_fetch_services_mining_to( - mine_to_pool: PoolType, + mine_to_pool: MinerPool, validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, network: Option, ) -> StateAndFetchServices { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( mine_to_pool, validator, network, @@ -1026,24 +1159,11 @@ pub async fn launch_state_and_fetch_services_mining_to( Some(NetworkKind::Testnet) => { println!("Waiting for validator to spawn.."); tokio::time::sleep(std::time::Duration::from_millis(5000)).await; - Network::Testnet + Network::PubTestnet } - _ => Network::Regtest({ - let activation_heights = test_manager.local_net.get_activation_heights().await; - ActivationHeights { - before_overwinter: activation_heights.overwinter(), - overwinter: activation_heights.overwinter(), - sapling: activation_heights.sapling(), - blossom: activation_heights.blossom(), - heartwood: activation_heights.heartwood(), - canopy: activation_heights.canopy(), - nu5: activation_heights.nu5(), - nu6: activation_heights.nu6(), - nu6_1: activation_heights.nu6_1(), - nu6_2: activation_heights.nu6_2(), - nu7: activation_heights.nu7(), - } - }), + // The kind suffices: zainod adopts the schedule from the validator + // at spawn (zaino#1076). + _ => Network::Regtest, }; test_manager.local_net.print_stdout(); @@ -1067,41 +1187,42 @@ pub async fn launch_state_and_fetch_services_mining_to( None => test_manager.data_dir.clone(), }; - let state_service = StateService::spawn(StateServiceConfig::new( - zebra_state::Config { - cache_dir: state_chain_cache_dir, - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - should_backup_non_finalized_state: false, - debug_skip_non_finalized_state_backup_task: false, - }, - test_manager.full_node_rpc_listen_address.to_string(), - test_manager.full_node_grpc_listen_address, - false, - None, - None, - None, - ServiceConfig::default(), - StorageConfig { - database: DatabaseConfig { - path: test_manager - .local_net - .data_dir() - .path() - .to_path_buf() - .join("state-srvice-zaino"), + let state_service = + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_direct( + zebra_state::Config { + cache_dir: state_chain_cache_dir, + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + should_backup_non_finalized_state: false, + debug_skip_non_finalized_state_backup_task: false, + }, + test_manager.full_node_rpc_listen_address.to_string(), + test_manager.full_node_grpc_listen_address, + false, + None, + None, + None, + ServiceConfig::default(), + StorageConfig { + database: DatabaseConfig { + path: test_manager + .local_net + .data_dir() + .path() + .to_path_buf() + .join("state-srvice-zaino"), + ..Default::default() + }, ..Default::default() }, - ..Default::default() - }, - false, // ephemeral_finalised_state: tests use a persistent finalised DB - network_type, - None, - )) - .await - .unwrap(); + false, // ephemeral_finalised_state: tests use a persistent finalised DB + network_type, + None, + )) + .await + .unwrap(); let state_subscriber = state_service.get_subscriber().inner(); @@ -1116,16 +1237,15 @@ pub async fn launch_state_and_fetch_services_mining_to( } } -/// Spawn a standalone [`FetchService`] pointed at `rpc_url`, storing its index -/// under `db_path`. Shared by the launch helpers below. -#[allow(deprecated)] +/// Spawn a standalone `Rpc`-connection [`NodeBackedIndexerService`] pointed at `rpc_url`, +/// storing its index under `db_path`. Shared by the launch helpers below. async fn spawn_fetch_service( rpc_url: String, cookie_dir: Option, db_path: PathBuf, network: Network, -) -> FetchService { - FetchService::spawn(FetchServiceConfig::new( +) -> NodeBackedIndexerService { + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_rpc( rpc_url, cookie_dir, None, @@ -1152,18 +1272,17 @@ async fn spawn_fetch_service( /// server — for tests that compare the two. The owned services exist only to /// keep their subscribers alive. #[cfg(feature = "zcashd_support")] -#[allow(deprecated)] pub struct ZcashdDualFetchServices { /// The launched zcashd + Zaino test manager. - pub test_manager: TestManager, - /// Owned zcashd-direct fetch service; keeps `zcashd_subscriber` alive. - pub zcashd_fetch_service: FetchService, + pub test_manager: TestManager, + /// Owned zcashd-direct `Rpc` service; keeps `zcashd_subscriber` alive. + pub zcashd_fetch_service: NodeBackedIndexerService, /// Subscriber whose answers come from zcashd directly. - pub zcashd_subscriber: FetchServiceSubscriber, - /// Owned Zaino-pointed fetch service; keeps `zaino_subscriber` alive. - pub zaino_fetch_service: FetchService, + pub zcashd_subscriber: NodeBackedIndexerServiceSubscriber, + /// Owned Zaino-pointed `Rpc` service; keeps `zaino_subscriber` alive. + pub zaino_fetch_service: NodeBackedIndexerService, /// Subscriber whose answers come through Zaino's JSON-RPC server. - pub zaino_subscriber: FetchServiceSubscriber, + pub zaino_subscriber: NodeBackedIndexerServiceSubscriber, } #[cfg(feature = "zcashd_support")] @@ -1202,7 +1321,7 @@ pub async fn launch_zcashd_dual_fetch_services() -> ZcashdDualFetchServices { pub async fn launch_zcashd_dual_fetch_services_at( activation_heights: ActivationHeights, ) -> ZcashdDualFetchServices { - let test_manager = TestManager::::launch( + let test_manager = TestManager::::launch( &ValidatorKind::Zcashd, None, Some(activation_heights), @@ -1224,7 +1343,7 @@ pub async fn launch_zcashd_dual_fetch_services_at( .data_dir() .path() .join("zcashd-fetch-service-zaino"), - Network::Regtest(activation_heights), + Network::Regtest, ) .await; let zcashd_subscriber = zcashd_fetch_service.get_subscriber().inner(); @@ -1242,7 +1361,7 @@ pub async fn launch_zcashd_dual_fetch_services_at( .data_dir() .path() .join("zaino-fetch-service-zaino"), - Network::Regtest(activation_heights), + Network::Regtest, ) .await; let zaino_subscriber = zaino_fetch_service.get_subscriber().inner(); @@ -1299,11 +1418,10 @@ pub fn get_info_with_zeroed_timestamp(info: GetInfo) -> GetInfo { /// harness in both live-test partitions (`clientless` and `e2e`). The `e2e` /// partition wraps this and builds lightclients from the returned manager's /// gRPC address. -#[allow(deprecated)] pub async fn launch_with_fetch_subscriber( validator: &ValidatorKind, chain_cache: Option, -) -> (TestManager, FetchServiceSubscriber) { +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { launch_with_fetch_subscriber_mining_to::( default_mining_pool(validator), validator, @@ -1316,13 +1434,12 @@ pub async fn launch_with_fetch_subscriber( /// caller instead of [`default_mining_pool`]: [`SHIELDED_FUNDING_POOL`] for /// sessions funding wallets from coinbase, or a pinned pool for tests whose /// subject is the miner's coinbase footprint. -#[allow(deprecated)] pub async fn launch_with_fetch_subscriber_mining_to( - mine_to_pool: PoolType, + mine_to_pool: MinerPool, validator: &ValidatorKind, chain_cache: Option, -) -> (TestManager, FetchServiceSubscriber) { - let mut test_manager = TestManager::::launch_mining_to( +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { + let mut test_manager = TestManager::::launch_mining_to( mine_to_pool, validator, None, @@ -1338,9 +1455,7 @@ pub async fn launch_with_fetch_subscriber_mining_to( (test_manager, fetch_service_subscriber) } -impl Drop - for TestManager -{ +impl Drop for TestManager { fn drop(&mut self) { debug!("[TEST] Shutting down test environment"); if let Some(handle) = &self.zaino_handle { @@ -1362,24 +1477,20 @@ async fn build_client( #[cfg(test)] mod launch_testmanager { use super::*; - #[allow(deprecated)] - use zaino_state::FetchService; /// Launch with no network/heights overrides and the optional servers off — /// the smoke tests' shared launch shape; only the chain cache and the /// zaino toggle vary. - async fn launch_minimal( + async fn launch_minimal( validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, - ) -> TestManager + ) -> TestManager where C: ValidatorExt, - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: ValidatorConnectionMarker, { - TestManager::::launch( + TestManager::::launch( validator, None, None, @@ -1402,7 +1513,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, false).await; + launch_minimal::(&ValidatorKind::Zcashd, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1411,7 +1522,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, false).await; + launch_minimal::(&ValidatorKind::Zcashd, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1422,7 +1533,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zcashd, ZCASHD_CHAIN_CACHE_DIR.clone(), false, @@ -1436,7 +1547,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, true).await; + launch_minimal::(&ValidatorKind::Zcashd, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await @@ -1458,8 +1569,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1468,8 +1578,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1480,7 +1589,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zebrad, ZEBRAD_CHAIN_CACHE_DIR.clone(), false, @@ -1494,8 +1603,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, true) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await .unwrap(); @@ -1505,15 +1613,12 @@ mod launch_testmanager { mod state_service { use super::*; - #[allow(deprecated)] - use zaino_state::StateService; #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1522,8 +1627,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1534,7 +1638,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zebrad, ZEBRAD_CHAIN_CACHE_DIR.clone(), false, @@ -1548,8 +1652,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, true) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await .unwrap(); diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index ed7c022f5..c126067a0 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -8,6 +8,9 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- `ActivationHeights.nu6_3` (serde key `"NU6.3"`) for the NU6.3 network + upgrade. `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` currently leaves it `None` + (inactive); a chain with NU6.3 active needs the height stated explicitly. - `StorageConfig::database.sync_checkpoint_interval` (seconds, default 120) — max wall-clock time spent buffering a bulk-sync write batch before flushing. Under the env's `NO_SYNC` mode this also bounds the window of unflushed writes at risk @@ -18,6 +21,10 @@ and this library adheres to Rust's notion of `sync_write_batch_size` so the two operations cannot inflate each other's peak memory. ### Changed +- `crypto::ensure_default_crypto_provider` now installs rustls's + **aws-lc-rs** provider (was ring) as the process-level default, and the + crate's rustls features become `aws_lc_rs` + `prefer-post-quantum` + (ADR-0006). First-install-wins semantics are unchanged. - **Breaking** — `StorageConfig::database.sync_write_batch_bytes` (raw bytes) is renamed to `sync_write_batch_size` and now expressed in **GiB** (new `SyncWriteBatchSize` newtype, mirroring `DatabaseSize`); the default is 8 GiB. @@ -28,8 +35,27 @@ and this library adheres to Rust's notion of unrecognized key under `[storage.database]` (e.g. a stale `sync_write_batch_bytes`) is a hard parse error instead of being silently ignored and falling back to the default budget — the silent fallback previously OOM-killed nodes. +- The `ActivationHeights` ↔ `ConfiguredActivationHeights` conversions and the + zebra-network → `ActivationHeights` extraction are now generated from a single + variant/field pair list (internal refactor; conversion behavior unchanged). +- `logging::init` / `logging::try_init` build their subscriber through one shared + installer (internal refactor). The `ZAINOLOG_FORMAT` / `ZAINOLOG_COLOR` / + `RUST_LOG` runtime interface and output formats are unchanged. ### Deprecated ### Removed +- Unused dependencies `thiserror`, `nu-ansi-term`, and `hex` (`hex`'s last + consumers were the display wrappers removed below). Verified empirically: + each dependency was deleted in turn and the crate re-checked with + `cargo check --all-targets`. +- **Breaking** — `Network::zaino_regtest_heights` (unused; regtest heights come + from `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or an explicit `ActivationHeights`). +- **Breaking** — `logging::DisplayHash` and `logging::DisplayHexStr` (unused). +- **Breaking** — the unused programmatic logging-configuration surface: + `logging::init_with_config` / `logging::try_init_with_config` and the `LogConfig` + / `LogFormat` types (including the `LogConfig` builder methods and + `show_span_events`, which no caller could reach). Logging is configured via the + `ZAINOLOG_FORMAT` / `ZAINOLOG_COLOR` / `RUST_LOG` environment variables, whose + behavior is unchanged. ### Fixed ## [0.2.0] - 2026-06-17 diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 51cfc662e..2ac216842 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.3.0" +version = "0.4.0" [dependencies] # Zebra @@ -15,16 +15,12 @@ zebra-chain = { workspace = true } # Serialization serde = { workspace = true, features = ["derive"] } -# Error handling -thiserror = { workspace = true } - # Logging tracing = { workspace = true } tracing-subscriber = { workspace = true } tracing-tree = { workspace = true } -nu-ansi-term = { workspace = true } -hex = { workspace = true } time = { workspace = true } - -# Zingo -zingo_common_components = { workspace = true } +rustls = { version = "0.23", default-features = false, features = [ + "aws_lc_rs", + "prefer-post-quantum", +] } diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index 3a0f57829..61a3dd294 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -5,74 +5,38 @@ use std::fmt; use serde::{Deserialize, Serialize}; use zebra_chain::parameters::testnet::ConfiguredActivationHeights; -pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { - overwinter: Some(1), - before_overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(2), - nu6: Some(2), - nu6_1: Some(2), - nu6_2: Some(2), - nu7: None, -}; - -/// Network type for Zaino configuration. +/// The network *kind* zaino is configured for. Deliberately payload-free: +/// activation heights are chain facts the validator owns, so a config value +/// cannot carry them — the backends adopt the runtime schedule from the +/// validator's `getblockchaininfo.upgrades` at spawn and hold it as a +/// `zebra_chain::parameters::Network` +/// (). A pre-adoption +/// height read is unrepresentable: this type has no heights to read. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(from = "NetworkSerde", into = "NetworkSerde")] pub enum Network { /// Mainnet network Mainnet, - /// Testnet network - Testnet, + /// The Public Testnet: the shared-consensus test network, whose + /// activation heights come only from checked-in zebra parameters — + /// never from configuration. Accepts the legacy config spelling + /// `"Testnet"`. + #[serde(alias = "Testnet")] + PubTestnet, /// Regtest network (for local testing) - Regtest(ActivationHeights), + Regtest, } impl fmt::Display for Network { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Network::Mainnet => write!(f, "Mainnet"), - Network::Testnet => write!(f, "Testnet"), - Network::Regtest(_) => write!(f, "Regtest"), - } - } -} - -/// Helper type for Network serialization/deserialization. -/// -/// This allows Network to serialize as simple strings ("Mainnet", "Testnet", "Regtest") -/// while the actual Network::Regtest variant carries activation heights internally. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -enum NetworkSerde { - Mainnet, - Testnet, - Regtest, -} - -impl From for Network { - fn from(value: NetworkSerde) -> Self { - match value { - NetworkSerde::Mainnet => Network::Mainnet, - NetworkSerde::Testnet => Network::Testnet, - NetworkSerde::Regtest => Network::Regtest(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), - } - } -} - -impl From for NetworkSerde { - fn from(value: Network) -> Self { - match value { - Network::Mainnet => NetworkSerde::Mainnet, - Network::Testnet => NetworkSerde::Testnet, - Network::Regtest(_) => NetworkSerde::Regtest, + Network::PubTestnet => write!(f, "PubTestnet"), + Network::Regtest => write!(f, "Regtest"), } } } -/// Configurable activation heights for Regtest and configured Testnets. +/// Configurable activation heights for a Regtest validator launch. /// /// We use our own type instead of the zebra type /// as the zebra type is missing a number of useful @@ -108,6 +72,9 @@ pub struct ActivationHeights { /// Activation height for `NU6.2` network upgrade. #[serde(rename = "NU6.2")] pub nu6_2: Option, + /// Activation height for `NU6.3` network upgrade. + #[serde(rename = "NU6.3")] + pub nu6_3: Option, /// Activation height for `NU7` network upgrade. #[serde(rename = "NU7")] pub nu7: Option, @@ -126,151 +93,96 @@ impl Default for ActivationHeights { nu6: Some(2), nu6_1: Some(2), nu6_2: Some(2), + nu6_3: None, nu7: None, } } } -impl From for zingo_common_components::protocol::ActivationHeights { - fn from(val: ActivationHeights) -> Self { - zingo_common_components::protocol::ActivationHeightsBuilder::new() - .set_overwinter(val.overwinter) - .set_sapling(val.sapling) - .set_blossom(val.blossom) - .set_heartwood(val.heartwood) - .set_canopy(val.canopy) - .set_nu5(val.nu5) - .set_nu6(val.nu6) - .set_nu6_1(val.nu6_1) - .set_nu6_2(val.nu6_2) - .set_nu7(val.nu7) - .build() - } -} - -impl From for ActivationHeights { - fn from( - ConfiguredActivationHeights { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu7, - }: ConfiguredActivationHeights, - ) -> Self { - Self { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu7, - } - } -} -impl From for ConfiguredActivationHeights { - fn from( - ActivationHeights { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu7, - }: ActivationHeights, - ) -> Self { - Self { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu7, +/// Records the `NetworkUpgrade`-variant ↔ `ActivationHeights`-field correspondence +/// exactly once, generating everything derived from it: the two field-by-field +/// `From` conversions between [`ActivationHeights`] and zebra's +/// [`ConfiguredActivationHeights`] (the structs share field names), the +/// all-`None` [`ActivationHeights::NEVER_ACTIVATED`] schedule, and the +/// per-upgrade [`ActivationHeights::slot_mut`] accessor. +/// +/// A declarative macro rather than functions because plain `fn`s cannot abstract +/// over struct fields, and the variant/field spellings (`Nu5`/`nu5`) differ only +/// by casing, which `macro_rules!` cannot derive — hence explicit pairs. +/// +/// Zebra's side of these conversions is structurally stable; the recurring edit +/// here is a new network upgrade, which lands as a single `(Variant, field)` +/// entry in the invocation below (after adding the struct field). The exhaustive +/// destructures and match keep full compile-time drift detection: a new zebra +/// field or variant fails the build until its pair is added. +macro_rules! activation_heights_mirror { + ($(($variant:ident, $field:ident)),* $(,)?) => { + impl From for ActivationHeights { + fn from( + ConfiguredActivationHeights { $($field),* }: ConfiguredActivationHeights, + ) -> Self { + Self { $($field),* } + } } - } -} -impl From for ActivationHeights { - fn from(activation_heights: zingo_common_components::protocol::ActivationHeights) -> Self { - ActivationHeights { - before_overwinter: activation_heights.overwinter(), - overwinter: activation_heights.overwinter(), - sapling: activation_heights.sapling(), - blossom: activation_heights.blossom(), - heartwood: activation_heights.heartwood(), - canopy: activation_heights.canopy(), - nu5: activation_heights.nu5(), - nu6: activation_heights.nu6(), - nu6_1: activation_heights.nu6_1(), - nu6_2: activation_heights.nu6_2(), - nu7: activation_heights.nu7(), + impl From for ConfiguredActivationHeights { + fn from(ActivationHeights { $($field),* }: ActivationHeights) -> Self { + Self { $($field),* } + } } - } -} -impl Network { - /// Convert to Zebra's network type for internal use (alias for to_zebra_default). - pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { - self.into() - } - - /// Get the standard regtest activation heights used by Zaino. - pub fn zaino_regtest_heights() -> ConfiguredActivationHeights { - ConfiguredActivationHeights { - before_overwinter: Some(1), - overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(1), - nu6: Some(1), - nu6_1: None, - nu6_2: None, - nu7: None, + impl ActivationHeights { + /// The all-`None` schedule: every upgrade never-activated. The + /// starting point for building heights from an external report, + /// where an upgrade absent from the report must stay unset. + pub const NEVER_ACTIVATED: Self = Self { $($field: None),* }; + + /// Mutable slot for `upgrade`'s activation height, or `None` for + /// `Genesis` (height 0 by definition; it has no slot). + pub fn slot_mut( + &mut self, + upgrade: zebra_chain::parameters::NetworkUpgrade, + ) -> Option<&mut Option> { + match upgrade { + zebra_chain::parameters::NetworkUpgrade::Genesis => None, + $( + zebra_chain::parameters::NetworkUpgrade::$variant => { + Some(&mut self.$field) + } + )* + } + } } - } + }; +} + +activation_heights_mirror!( + (BeforeOverwinter, before_overwinter), + (Overwinter, overwinter), + (Sapling, sapling), + (Blossom, blossom), + (Heartwood, heartwood), + (Canopy, canopy), + (Nu5, nu5), + (Nu6, nu6), + (Nu6_1, nu6_1), + (Nu6_2, nu6_2), + (Nu6_3, nu6_3), + (Nu7, nu7), +); +impl Network { /// Determines if we should wait for the server to fully sync. Used for testing /// - /// - Mainnet/Testnet: Skip sync (false) because we don't want to sync real chains in tests + /// - Mainnet / The Public Testnet: Skip sync (false) because we don't want + /// to sync real chains in tests /// - Regtest: Enable sync (true) because regtest is local and fast to sync pub fn wait_on_server_sync(&self) -> bool { match self { - Network::Mainnet | Network::Testnet => false, // Real networks - don't try to sync the whole chain - Network::Regtest(_) => true, // Local network - safe and fast to sync - } - } - - pub fn from_network_kind_and_activation_heights( - network: &zebra_chain::parameters::NetworkKind, - activation_heights: &ActivationHeights, - ) -> Self { - match network { - zebra_chain::parameters::NetworkKind::Mainnet => Network::Mainnet, - zebra_chain::parameters::NetworkKind::Testnet => Network::Testnet, - zebra_chain::parameters::NetworkKind::Regtest => Network::Regtest(*activation_heights), + // Real networks - don't try to sync the whole chain + Network::Mainnet | Network::PubTestnet => false, + // Local network - safe and fast to sync + Network::Regtest => true, } } } @@ -281,81 +193,26 @@ impl From for Network { zebra_chain::parameters::Network::Mainnet => Network::Mainnet, zebra_chain::parameters::Network::Testnet(parameters) => { if parameters.is_regtest() { - let mut activation_heights = ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu7: None, - }; - for (height, upgrade) in parameters.activation_heights().iter() { - match upgrade { - zebra_chain::parameters::NetworkUpgrade::Genesis => (), - zebra_chain::parameters::NetworkUpgrade::BeforeOverwinter => { - activation_heights.before_overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Overwinter => { - activation_heights.overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Sapling => { - activation_heights.sapling = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Blossom => { - activation_heights.blossom = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Heartwood => { - activation_heights.heartwood = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Canopy => { - activation_heights.canopy = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu5 => { - activation_heights.nu5 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6 => { - activation_heights.nu6 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_1 => { - activation_heights.nu6_1 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_2 => { - activation_heights.nu6_2 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu7 => { - activation_heights.nu7 = Some(height.0) - } - } - } - Network::Regtest(activation_heights) + Network::Regtest } else { - Network::Testnet + Network::PubTestnet } } } } } -impl From for zebra_chain::parameters::Network { - fn from(val: Network) -> Self { - match val { - Network::Regtest(activation_heights) => zebra_chain::parameters::Network::new_regtest( - Into::::into(activation_heights).into(), - ), - Network::Testnet => zebra_chain::parameters::Network::new_default_testnet(), - Network::Mainnet => zebra_chain::parameters::Network::Mainnet, - } - } -} - -impl From<&Network> for zebra_chain::parameters::Network { - fn from(val: &Network) -> Self { - (*val).into() +impl ActivationHeights { + /// Builds the runtime regtest network for a chain whose schedule is + /// known first-hand: a validator being *launched* with these heights, or + /// a test fixture that is its own chain (mockchain sources, proptest + /// block generators). Production indexer code never calls this with + /// configured values — it adopts the runtime network from the + /// validator's reported schedule instead (zaino#1076). + pub fn to_regtest_network(&self) -> zebra_chain::parameters::Network { + zebra_chain::parameters::Network::new_regtest( + Into::::into(*self).into(), + ) } } @@ -376,15 +233,12 @@ mod tests { nu6: Some(1), nu6_1: Some(1), nu6_2: Some(2), + nu6_3: Some(500), nu7: Some(1000), }; let zebra_heights: zebra_chain::parameters::testnet::ConfiguredActivationHeights = heights.into(); assert_eq!(zebra_heights.nu6_2, Some(2)); - - let zingo_heights: zingo_common_components::protocol::ActivationHeights = heights.into(); - let heights = ActivationHeights::from(zingo_heights); - assert_eq!(heights.nu6_2, Some(2)); } } diff --git a/packages/zaino-common/src/consensus.rs b/packages/zaino-common/src/consensus.rs new file mode 100644 index 000000000..1f02dcab1 --- /dev/null +++ b/packages/zaino-common/src/consensus.rs @@ -0,0 +1,25 @@ +//! Consensus-derived constants with a single source of truth. +//! +//! Each value derives from zebra's authoritative upstream constant; nothing else in +//! the workspace should hard-code these — reference this module instead. + +/// Number of confirmations before a coinbase output becomes spendable. +/// +/// Single source of truth, from zebra's transparent-coinbase maturity rule. +pub const COINBASE_MATURITY: u32 = zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY; + +/// Distance below the best-chain tip of the finalised / non-finalised seam: a block +/// buried deeper than this is finalised (reorg-stable). +/// +/// Derived from zebra's protocol reorg limit (`MAX_BLOCK_REORG_HEIGHT`). The `+ 1` +/// accounts for the tip block itself, preserving the historical seam semantics. +pub const MAX_NONFINALISED_DEPTH: u32 = + zebra_chain::parameters::constants::MAX_BLOCK_REORG_HEIGHT + 1; + +/// A tractable one-tenth of [`MAX_NONFINALISED_DEPTH`], for fast tests that need a +/// finalised seam without building a full ~[`MAX_NONFINALISED_DEPTH`]-block chain. +/// +/// Integer division, so this is `100` when the real depth is `1001`. Test-only: it +/// lets in-crate tests select a shallow seam that still derives from the single +/// source of truth rather than a hard-coded literal. +pub const FAST_TEST_MAX_NONFINALISED_DEPTH: u32 = MAX_NONFINALISED_DEPTH / 10; diff --git a/packages/zaino-common/src/crypto.rs b/packages/zaino-common/src/crypto.rs new file mode 100644 index 000000000..c24dc59d9 --- /dev/null +++ b/packages/zaino-common/src/crypto.rs @@ -0,0 +1,36 @@ +//! Process-level rustls `CryptoProvider` management. + +/// Installs rustls's `aws-lc-rs` provider as the process-level default +/// if no provider is installed yet. +/// +/// Call this before constructing anything that builds a rustls config — +/// a `reqwest` client (the workspace enables reqwest's +/// `rustls-no-provider` feature, which never auto-selects a provider) or +/// a TLS-enabled tonic server. rustls's own auto-selection is also +/// unreliable here: it works only when exactly one of rustls's `ring` / +/// `aws-lc-rs` features is enabled, and dependency feature-unification +/// can silently enable both, which panicked at runtime on TLS-enabled +/// deployments (zingolabs/zaino#1360). Installing explicitly removes +/// that dependence on the feature graph. +/// +/// aws-lc-rs is the preferred provider (ADR-0006): it is the only rustls +/// provider with post-quantum key exchange, and the workspace enables +/// rustls's `prefer-post-quantum` feature so the X25519MLKEM768 hybrid +/// group leads this provider's defaults. That ordering governs zaino's +/// outbound (client-role) handshakes; rustls servers follow the client's +/// group preference, so inbound hybrid uptake is decided by the +/// connecting wallet's TLS stack. Classical key-exchange groups remain +/// offered and accepted, but are deprecated (see the ADR). +/// +/// The process default is first-install-wins and is not constrained by +/// our crates' rustls features, so an embedder (e.g. zallet) that has +/// already installed a provider keeps it: zaino then handshakes through +/// that provider instead of aws-lc-rs, which is fine for any provider +/// implementing the standard TLS suites (ring and aws-lc-rs both do). +pub fn ensure_default_crypto_provider() { + if rustls::crypto::CryptoProvider::get_default().is_none() { + // A racing concurrent install is the only error case; either + // provider serves. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + } +} diff --git a/packages/zaino-common/src/lib.rs b/packages/zaino-common/src/lib.rs index 3e46a792c..818f23935 100644 --- a/packages/zaino-common/src/lib.rs +++ b/packages/zaino-common/src/lib.rs @@ -4,6 +4,8 @@ //! and common utilities used across the Zaino blockchain indexer ecosystem. pub mod config; +pub mod consensus; +pub mod crypto; pub mod logging; pub mod net; pub mod probing; @@ -15,7 +17,7 @@ pub use net::{resolve_socket_addr, try_resolve_address, AddressResolution}; // Re-export commonly used config types at crate root for backward compatibility. // This allows existing code using `use zaino_common::Network` to continue working. -pub use config::network::{ActivationHeights, Network, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}; +pub use config::network::{ActivationHeights, Network}; pub use config::service::ServiceConfig; pub use config::storage::{ AccumulatorRebuildMemorySize, CacheConfig, DatabaseConfig, DatabaseSize, StorageConfig, diff --git a/packages/zaino-common/src/logging.rs b/packages/zaino-common/src/logging.rs index e97703bd3..7bf812671 100644 --- a/packages/zaino-common/src/logging.rs +++ b/packages/zaino-common/src/logging.rs @@ -18,23 +18,19 @@ //! ```no_run //! use zaino_common::logging; //! -//! // Initialize with defaults (tree format, info level) +//! // Initialize logging, configured via the environment variables above. //! logging::init(); -//! -//! // Or with custom configuration -//! logging::init_with_config(logging::LogConfig::default().format(logging::LogFormat::Json)); //! ``` use std::env; -use std::fmt; use std::io::IsTerminal; use time::macros::format_description; use tracing::Level; use tracing_subscriber::{ - fmt::{format::FmtSpan, time::UtcTime}, + fmt::time::UtcTime, layer::SubscriberExt, - util::SubscriberInitExt, + util::{SubscriberInitExt, TryInitError}, EnvFilter, }; use tracing_tree::HierarchicalLayer; @@ -45,7 +41,7 @@ const TIME_FORMAT: &[time::format_description::FormatItem<'static>] = /// Log output format. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum LogFormat { +enum LogFormat { /// Hierarchical tree view showing span nesting. Tree, /// Flat chronological stream (default). @@ -57,7 +53,7 @@ pub enum LogFormat { impl LogFormat { /// Parse from string (case-insensitive). - pub fn parse_str(s: &str) -> Option { + fn parse_str(s: &str) -> Option { match s.to_lowercase().as_str() { "tree" => Some(LogFormat::Tree), "stream" => Some(LogFormat::Stream), @@ -67,7 +63,7 @@ impl LogFormat { } /// Get from ZAINOLOG_FORMAT environment variable. - pub fn from_env() -> Self { + fn from_env() -> Self { env::var("ZAINOLOG_FORMAT") .ok() .and_then(|s| Self::parse_str(&s)) @@ -75,27 +71,15 @@ impl LogFormat { } } -impl fmt::Display for LogFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - LogFormat::Tree => write!(f, "tree"), - LogFormat::Stream => write!(f, "stream"), - LogFormat::Json => write!(f, "json"), - } - } -} - -/// Logging configuration. +/// Logging configuration, read from the environment. #[derive(Debug, Clone)] -pub struct LogConfig { +struct LogConfig { /// Output format (tree, stream, or json). - pub format: LogFormat, + format: LogFormat, /// Enable ANSI colors. - pub color: bool, + color: bool, /// Default log level. - pub level: Level, - /// Show span events (enter/exit). - pub show_span_events: bool, + level: Level, } impl Default for LogConfig { @@ -119,228 +103,72 @@ impl Default for LogConfig { format: LogFormat::from_env(), color, level: Level::INFO, - show_span_events: false, } } } -impl LogConfig { - /// Create a new config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the log format. - pub fn format(mut self, format: LogFormat) -> Self { - self.format = format; - self - } - - /// Enable or disable colors. - pub fn color(mut self, color: bool) -> Self { - self.color = color; - self - } - - /// Set the default log level. - pub fn level(mut self, level: Level) -> Self { - self.level = level; - self - } - - /// Show span enter/exit events. - pub fn show_span_events(mut self, show: bool) -> Self { - self.show_span_events = show; - self - } -} - -/// Initialize logging with default configuration. +/// Initialize logging, configured from the environment (see module docs). +/// +/// # Panics /// -/// Reads `ZAINOLOG_FORMAT` environment variable to determine format: -/// - "stream" (default): Flat chronological output with timestamps -/// - "tree": Hierarchical span-based output -/// - "json": Machine-parseable JSON +/// Panics if a global tracing subscriber has already been set. pub fn init() { - init_with_config(LogConfig::default()); -} - -/// Initialize logging with custom configuration. -pub fn init_with_config(config: LogConfig) { - // If RUST_LOG is set, use it directly. Otherwise, default to zaino crates only. - // Users can set RUST_LOG=info to see all crates including zebra. - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { - EnvFilter::new(format!( - "zaino={level},zainod={level},zainodlib={level}", - level = config.level.as_str() - )) - }); - - match config.format { - LogFormat::Tree => init_tree(env_filter, config), - LogFormat::Stream => init_stream(env_filter, config), - LogFormat::Json => init_json(env_filter), - } + try_install(LogConfig::default()).expect("global tracing subscriber already set"); } /// Try to initialize logging (won't fail if already initialized). /// /// Useful for tests where multiple test functions may try to initialize logging. pub fn try_init() { - try_init_with_config(LogConfig::default()); + let _ = try_install(LogConfig::default()); } -/// Try to initialize logging with custom configuration. -pub fn try_init_with_config(config: LogConfig) { +/// Build the subscriber described by `config` and install it as the global +/// default, erroring if one is already set. +fn try_install(config: LogConfig) -> Result<(), TryInitError> { + // If RUST_LOG is set, use it directly. Otherwise, default to zaino crates only. + // Users can set RUST_LOG=info to see all crates including zebra. let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { EnvFilter::new(format!( "zaino={level},zainod={level},zainodlib={level}", level = config.level.as_str() )) }); + let registry = tracing_subscriber::registry().with(env_filter); match config.format { - LogFormat::Tree => { - let _ = try_init_tree(env_filter, config); - } - LogFormat::Stream => { - let _ = try_init_stream(env_filter, config); - } - LogFormat::Json => { - let _ = try_init_json(env_filter); - } - } -} - -fn init_tree(env_filter: EnvFilter, config: LogConfig) { - let layer = HierarchicalLayer::new(2) - .with_ansi(config.color) - .with_targets(true) - .with_bracketed_fields(true) - .with_indent_lines(true) - .with_thread_ids(false) - .with_thread_names(false) - .with_deferred_spans(true) // Only show spans when they have events - .with_verbose_entry(false) // Don't repeat span info on entry - .with_verbose_exit(false); // Don't repeat span info on exit - - tracing_subscriber::registry() - .with(env_filter) - .with(layer) - .init(); -} - -fn try_init_tree( - env_filter: EnvFilter, - config: LogConfig, -) -> Result<(), tracing_subscriber::util::TryInitError> { - let layer = HierarchicalLayer::new(2) - .with_ansi(config.color) - .with_targets(true) - .with_bracketed_fields(true) - .with_indent_lines(true) - .with_thread_ids(false) - .with_thread_names(false) - .with_deferred_spans(true) // Only show spans when they have events - .with_verbose_entry(false) // Don't repeat span info on entry - .with_verbose_exit(false); // Don't repeat span info on exit - - tracing_subscriber::registry() - .with(env_filter) - .with(layer) - .try_init() -} - -fn init_stream(env_filter: EnvFilter, config: LogConfig) { - let span_events = if config.show_span_events { - FmtSpan::ENTER | FmtSpan::EXIT - } else { - FmtSpan::NONE - }; - - tracing_subscriber::fmt() - .with_env_filter(env_filter) - .with_timer(UtcTime::new(TIME_FORMAT)) - .with_target(true) - .with_ansi(config.color) - .with_span_events(span_events) - .pretty() - .init(); -} - -fn try_init_stream( - env_filter: EnvFilter, - config: LogConfig, -) -> Result<(), tracing_subscriber::util::TryInitError> { - let span_events = if config.show_span_events { - FmtSpan::ENTER | FmtSpan::EXIT - } else { - FmtSpan::NONE - }; - - let fmt_layer = tracing_subscriber::fmt::layer() - .with_timer(UtcTime::new(TIME_FORMAT)) - .with_target(true) - .with_ansi(config.color) - .with_span_events(span_events) - .pretty(); - - tracing_subscriber::registry() - .with(env_filter) - .with(fmt_layer) - .try_init() -} - -fn init_json(env_filter: EnvFilter) { - // JSON format keeps full RFC3339 timestamps for machine parsing - tracing_subscriber::fmt() - .with_env_filter(env_filter) - .json() - .with_timer(UtcTime::rfc_3339()) - .with_target(true) - .init(); -} - -fn try_init_json(env_filter: EnvFilter) -> Result<(), tracing_subscriber::util::TryInitError> { - // JSON format keeps full RFC3339 timestamps for machine parsing - let fmt_layer = tracing_subscriber::fmt::layer() - .json() - .with_timer(UtcTime::rfc_3339()) - .with_target(true); - - tracing_subscriber::registry() - .with(env_filter) - .with(fmt_layer) - .try_init() -} - -/// Wrapper for displaying hashes in a compact format. -/// -/// Shows the first 4 bytes (8 hex chars) followed by "..". -pub struct DisplayHash<'a>(pub &'a [u8]); - -impl fmt::Display for DisplayHash<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0.len() >= 4 { - write!(f, "{}..", hex::encode(&self.0[..4])) - } else { - write!(f, "{}", hex::encode(self.0)) - } - } -} - -/// Wrapper for displaying hex strings in a compact format. -/// -/// Shows the first 8 chars followed by "..". -pub struct DisplayHexStr<'a>(pub &'a str); - -impl fmt::Display for DisplayHexStr<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0.len() > 8 { - write!(f, "{}..", &self.0[..8]) - } else { - write!(f, "{}", self.0) - } + LogFormat::Tree => registry + .with( + HierarchicalLayer::new(2) + .with_ansi(config.color) + .with_targets(true) + .with_bracketed_fields(true) + .with_indent_lines(true) + .with_thread_ids(false) + .with_thread_names(false) + .with_deferred_spans(true) // Only show spans when they have events + .with_verbose_entry(false) // Don't repeat span info on entry + .with_verbose_exit(false), // Don't repeat span info on exit + ) + .try_init(), + LogFormat::Stream => registry + .with( + tracing_subscriber::fmt::layer() + .with_timer(UtcTime::new(TIME_FORMAT)) + .with_target(true) + .with_ansi(config.color) + .pretty(), + ) + .try_init(), + LogFormat::Json => registry + .with( + // JSON format keeps full RFC3339 timestamps for machine parsing + tracing_subscriber::fmt::layer() + .json() + .with_timer(UtcTime::rfc_3339()) + .with_target(true), + ) + .try_init(), } } @@ -356,31 +184,4 @@ mod tests { assert_eq!(LogFormat::parse_str("json"), Some(LogFormat::Json)); assert_eq!(LogFormat::parse_str("unknown"), None); } - - #[test] - fn test_display_hash() { - let hash = [0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0]; - assert_eq!(format!("{}", DisplayHash(&hash)), "12345678.."); - } - - #[test] - fn test_display_hex_str() { - let hex_str = "1234567890abcdef"; - assert_eq!(format!("{}", DisplayHexStr(hex_str)), "12345678.."); - - let short_hex = "1234"; - assert_eq!(format!("{}", DisplayHexStr(short_hex)), "1234"); - } - - #[test] - fn test_config_builder() { - let config = LogConfig::new() - .format(LogFormat::Json) - .color(false) - .level(Level::DEBUG); - - assert_eq!(config.format, LogFormat::Json); - assert!(!config.color); - assert_eq!(config.level, Level::DEBUG); - } } diff --git a/packages/zaino-fetch/CHANGELOG.md b/packages/zaino-fetch/CHANGELOG.md index 458372e2d..ad29e80d3 100644 --- a/packages/zaino-fetch/CHANGELOG.md +++ b/packages/zaino-fetch/CHANGELOG.md @@ -8,7 +8,14 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- Ironwood (NU6.3) / V6 transaction support: ironwood value balances are + read from parsed transactions, and compact-block construction populates + `CompactTx.ironwoodActions` and the block's `ironwoodCommitmentTreeSize`. ### Changed +- Transaction parsing delegates to + `zebra_chain::transaction::Transaction::zcash_deserialize` (zebra-chain 11), + replacing the hand-rolled parser that rejected transactions above v5 + ("Unsupported tx version 6"). ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-fetch/Cargo.toml b/packages/zaino-fetch/Cargo.toml index 8f6f0a910..e835aab94 100644 --- a/packages/zaino-fetch/Cargo.toml +++ b/packages/zaino-fetch/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.3.0" +version = "0.4.0" [features] default = [] @@ -49,8 +49,6 @@ serde = { workspace = true, features = ["derive"] } hex = { workspace = true, features = ["serde"] } indexmap = { workspace = true, features = ["serde"] } base64 = { workspace = true } -byteorder = { workspace = true } -sha2 = { workspace = true } jsonrpsee-types = { workspace = true } derive_more = { workspace = true, features = ["from"] } diff --git a/packages/zaino-fetch/src/chain/block.rs b/packages/zaino-fetch/src/chain/block.rs index e7bc978fd..1ead20241 100644 --- a/packages/zaino-fetch/src/chain/block.rs +++ b/packages/zaino-fetch/src/chain/block.rs @@ -1,247 +1,78 @@ //! Block fetching and deserialization functionality. -use crate::chain::{ - error::ParseError, - transaction::FullTransaction, - utils::{read_bytes, read_i32, read_u32, read_zcash_script_i64, CompactSize, ParseFromSlice}, +use crate::{ + chain::{error::ParseError, transaction::FullTransaction}, + utils::ParseFromSlice, }; -use sha2::{Digest, Sha256}; -use std::io::Cursor; +use std::{io::Cursor, sync::Arc}; use zaino_proto::proto::{ compact_formats::{ChainMetadata, CompactBlock}, utils::PoolTypeFilter, }; - -/// A block header, containing metadata about a block. -/// -/// How are blocks chained together? They are chained together via the -/// backwards reference (previous header hash) present in the block -/// header. Each block points backwards to its parent, all the way -/// back to the genesis block (the first block in the blockchain). -#[derive(Debug, Clone)] -struct BlockHeaderData { - /// The block's version field. This is supposed to be `4`: - /// - /// > The current and only defined block version number for Zcash is 4. - /// - /// but this was not enforced by the consensus rules, and defective mining - /// software created blocks with other versions, so instead it's effectively - /// a free field. The only constraint is that it must be at least `4` when - /// interpreted as an `i32`. - /// - /// Size \[bytes\]: 4 - version: i32, - - /// The hash of the previous block, used to create a chain of blocks back to - /// the genesis block. - /// - /// This ensures no previous block can be changed without also changing this - /// block's header. - /// - /// Size \[bytes\]: 32 - hash_prev_block: Vec, - - /// The root of the Bitcoin-inherited transaction Merkle tree, binding the - /// block header to the transactions in the block. - /// - /// Note that because of a flaw in Bitcoin's design, the `merkle_root` does - /// not always precisely bind the contents of the block (CVE-2012-2459). It - /// is sometimes possible for an attacker to create multiple distinct sets of - /// transactions with the same Merkle root, although only one set will be - /// valid. - /// - /// Size \[bytes\]: 32 - hash_merkle_root: Vec, - - /// \[Pre-Sapling\] A reserved field which should be ignored. - /// \[Sapling onward\] The root LEBS2OSP_256(rt) of the Sapling note - /// commitment tree corresponding to the final Sapling treestate of this - /// block. - /// - /// Size \[bytes\]: 32 - hash_final_sapling_root: Vec, - - /// The block timestamp is a Unix epoch time (UTC) when the miner - /// started hashing the header (according to the miner). - /// - /// Size \[bytes\]: 4 - time: u32, - - /// An encoded version of the target threshold this block's header - /// hash must be less than or equal to, in the same nBits format - /// used by Bitcoin. - /// - /// For a block at block height `height`, bits MUST be equal to - /// `ThresholdBits(height)`. - /// - /// [Bitcoin-nBits](https://bitcoin.org/en/developer-reference#target-nbits) - /// - /// Size \[bytes\]: 4 - n_bits_bytes: Vec, - - /// An arbitrary field that miners can change to modify the header - /// hash in order to produce a hash less than or equal to the - /// target threshold. - /// - /// Size \[bytes\]: 32 - nonce: Vec, - - /// The Equihash solution. - /// - /// Size \[bytes\]: CompactLength - solution: Vec, -} - -impl ParseFromSlice for BlockHeaderData { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for BlockHeaderData::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for BlockHeaderData::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let version = read_i32(&mut cursor, "Error reading BlockHeaderData::version")?; - let hash_prev_block = read_bytes( - &mut cursor, - 32, - "Error reading BlockHeaderData::hash_prev_block", - )?; - let hash_merkle_root = read_bytes( - &mut cursor, - 32, - "Error reading BlockHeaderData::hash_merkle_root", - )?; - let hash_final_sapling_root = read_bytes( - &mut cursor, - 32, - "Error reading BlockHeaderData::hash_final_sapling_root", - )?; - let time = read_u32(&mut cursor, "Error reading BlockHeaderData::time")?; - let n_bits_bytes = read_bytes( - &mut cursor, - 4, - "Error reading BlockHeaderData::n_bits_bytes", - )?; - let nonce = read_bytes(&mut cursor, 32, "Error reading BlockHeaderData::nonce")?; - - let solution = { - let compact_length = CompactSize::read(&mut cursor)?; - read_bytes( - &mut cursor, - compact_length as usize, - "Error reading BlockHeaderData::solution", - )? - }; - - Ok(( - &data[cursor.position() as usize..], - BlockHeaderData { - version, - hash_prev_block, - hash_merkle_root, - hash_final_sapling_root, - time, - n_bits_bytes, - nonce, - solution, - }, - )) - } -} - -impl BlockHeaderData { - /// Serializes the block header into a byte vector. - fn to_binary(&self) -> Result, ParseError> { - let mut buffer = Vec::new(); - - buffer.extend(&self.version.to_le_bytes()); - buffer.extend(&self.hash_prev_block); - buffer.extend(&self.hash_merkle_root); - buffer.extend(&self.hash_final_sapling_root); - buffer.extend(&self.time.to_le_bytes()); - buffer.extend(&self.n_bits_bytes); - buffer.extend(&self.nonce); - let mut solution_compact_size = Vec::new(); - CompactSize::write(&mut solution_compact_size, self.solution.len())?; - buffer.extend(solution_compact_size); - buffer.extend(&self.solution); - - Ok(buffer) - } - - /// Extracts the block hash from the block header. - fn get_hash(&self) -> Result, ParseError> { - let serialized_header = self.to_binary()?; - - let mut hasher = Sha256::new(); - hasher.update(&serialized_header); - let digest = hasher.finalize_reset(); - hasher.update(digest); - let final_digest = hasher.finalize(); - - Ok(final_digest.to_vec()) - } -} +use zebra_chain::serialization::{ZcashDeserialize as _, ZcashSerialize as _}; /// Complete block header. #[derive(Debug, Clone)] pub struct FullBlockHeader { - /// Block header data. - raw_block_header: BlockHeaderData, - - /// Hash of the current block. + header: Arc, + raw_bytes: Vec, cached_hash: Vec, } impl FullBlockHeader { + fn from_zebra(header: Arc) -> Result { + let raw_bytes = header.zcash_serialize_to_vec()?; + let cached_hash = header.hash().0.to_vec(); + + Ok(Self { + header, + raw_bytes, + cached_hash, + }) + } + /// Returns the Zcash block version. pub fn version(&self) -> i32 { - self.raw_block_header.version + i32::from_le_bytes(self.header.version.to_le_bytes()) } /// Returns The hash of the previous block. pub fn hash_prev_block(&self) -> Vec { - self.raw_block_header.hash_prev_block.clone() + self.header.previous_block_hash.0.to_vec() } /// Returns the root of the Bitcoin-inherited transaction Merkle tree. pub fn hash_merkle_root(&self) -> Vec { - self.raw_block_header.hash_merkle_root.clone() + self.header.merkle_root.0.to_vec() } /// Returns the final sapling root of the block. pub fn final_sapling_root(&self) -> Vec { - self.raw_block_header.hash_final_sapling_root.clone() + self.header.commitment_bytes.to_vec() } /// Returns the time when the miner started hashing the header (according to the miner). pub fn time(&self) -> u32 { - self.raw_block_header.time + u32::try_from(self.header.time.timestamp()) + .expect("deserialized block header timestamps fit in u32") } /// Returns an encoded version of the target threshold. pub fn n_bits_bytes(&self) -> Vec { - self.raw_block_header.n_bits_bytes.clone() + self.raw_bytes[104..108].to_vec() } /// Returns the block's nonce. pub fn nonce(&self) -> Vec { - self.raw_block_header.nonce.clone() + self.header.nonce.to_vec() } /// Returns the block's Equihash solution. pub fn solution(&self) -> Vec { - self.raw_block_header.solution.clone() + match &self.header.solution { + zebra_chain::work::equihash::Solution::Common(solution) => solution.to_vec(), + zebra_chain::work::equihash::Solution::Regtest(solution) => solution.to_vec(), + } } /// Returns the Hash of the current block. @@ -254,8 +85,6 @@ impl FullBlockHeader { #[derive(Debug, Clone)] pub struct FullBlock { /// The block header, containing block metadata. - /// - /// Size \[bytes\]: 140+CompactLength hdr: FullBlockHeader, /// The block transactions. @@ -271,74 +100,52 @@ impl ParseFromSlice for FullBlock { txid: Option>>, tx_version: Option, ) -> Result<(&[u8], Self), ParseError> { - let txid = txid.ok_or_else(|| { - ParseError::InvalidData("txid must be used for FullBlock::parse_from_slice".to_string()) - })?; if tx_version.is_some() { return Err(ParseError::InvalidData( "tx_version must be None for FullBlock::parse_from_slice".to_string(), )); } + let mut cursor = Cursor::new(data); + let block = zebra_chain::block::Block::zcash_deserialize(&mut cursor)?; + let consumed = usize::try_from(cursor.position())?; + let txids = txid.unwrap_or_default(); - let (remaining_data, block_header_data) = - BlockHeaderData::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - let tx_count = CompactSize::read(&mut cursor)?; - if txid.len() != tx_count as usize { + if !txids.is_empty() && txids.len() != block.transactions.len() { return Err(ParseError::InvalidData(format!( "number of txids ({}) does not match tx_count ({})", - txid.len(), - tx_count + txids.len(), + block.transactions.len() ))); } - let mut transactions = Vec::with_capacity(tx_count as usize); - let mut remaining_data = &data[cursor.position() as usize..]; - for txid_item in txid.iter() { - if remaining_data.is_empty() { - return Err(ParseError::InvalidData( - "parsing block transactions: not enough data for transaction.".to_string(), - )); - } - let (new_remaining_data, tx) = FullTransaction::parse_from_slice( - &data[cursor.position() as usize..], - Some(vec![txid_item.clone()]), - None, - )?; - transactions.push(tx); - remaining_data = new_remaining_data; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let block_height = Self::get_block_height(&transactions)?; - let block_hash = block_header_data.get_hash()?; - Ok(( - remaining_data, - FullBlock { - hdr: FullBlockHeader { - raw_block_header: block_header_data, - cached_hash: block_hash, - }, - vtx: transactions, - height: block_height, - }, - )) + let hdr = FullBlockHeader::from_zebra(block.header.clone())?; + let vtx = block + .transactions + .iter() + .cloned() + .enumerate() + .map(|(index, transaction)| { + FullTransaction::from_zebra(transaction, txids.get(index).cloned()) + }) + .collect::, _>>()?; + let height = block + .coinbase_height() + .map(|height| i32::try_from(height.0)) + .transpose()? + .ok_or_else(|| ParseError::InvalidData("missing block coinbase height".to_string()))?; + + Ok((&data[consumed..], Self { hdr, vtx, height })) } } -/// Genesis block special case. -/// -/// From LightWalletD: -/// see . -const GENESIS_TARGET_DIFFICULTY: u32 = 520617983; - impl FullBlock { /// Returns the full block header. pub fn header(&self) -> FullBlockHeader { self.hdr.clone() } - /// Returns the transactions held in the block. + /// Returns the transactions held in the block. pub fn transactions(&self) -> Vec { self.vtx.clone() } @@ -350,7 +157,7 @@ impl FullBlock { /// Returns the Orchard `authDataRoot` of the block, taken from the coinbase transaction's anchorOrchard field. /// - /// If the coinbase transaction is v5 and includes an Orchard bundle, this is the root of the Orchard commitment tree + /// If the coinbase transaction includes an Orchard bundle, this is the root of the Orchard commitment tree /// after applying all Orchard actions in the block. /// /// Returns `Some(Vec)` if present, else `None`. @@ -365,7 +172,9 @@ impl FullBlock { return Err(ParseError::InvalidData(format!( "Error decoding full block - {} bytes of Remaining data. Compact Block Created: ({:?})", remaining_data.len(), - full_block.into_compact_block(0, 0, PoolTypeFilter::includes_all()) + full_block + .clone() + .into_compact_block(0, 0, 0, PoolTypeFilter::includes_all()) ))); } Ok(full_block) @@ -378,21 +187,22 @@ impl FullBlock { self, sapling_commitment_tree_size: u32, orchard_commitment_tree_size: u32, + ironwood_commitment_tree_size: u32, pool_types: PoolTypeFilter, ) -> Result { let vtx = self .vtx .into_iter() .enumerate() - .filter_map(|(index, tx)| { - match tx.to_compact_tx(Some(index as u64), &pool_types) { + .filter_map( + |(index, tx)| match tx.to_compact_tx(Some(index as u64), &pool_types) { Ok(compact_tx) => { - // Omit transactions that have no elements in any requested pool type. if !compact_tx.vin.is_empty() || !compact_tx.vout.is_empty() || !compact_tx.spends.is_empty() || !compact_tx.outputs.is_empty() || !compact_tx.actions.is_empty() + || !compact_tx.ironwood_actions.is_empty() { Some(Ok(compact_tx)) } else { @@ -400,64 +210,23 @@ impl FullBlock { } } Err(parse_error) => Some(Err(parse_error)), - } - }) + }, + ) .collect::, _>>()?; - // NOTE: LightWalletD doesnt return a compact block header, however this could be used to return data if useful. - // let header = self.hdr.raw_block_header.to_binary()?; - let header = Vec::new(); - - let compact_block = CompactBlock { + Ok(CompactBlock { proto_version: 1, height: self.height as u64, hash: self.hdr.cached_hash.clone(), - prev_hash: self.hdr.raw_block_header.hash_prev_block.clone(), - time: self.hdr.raw_block_header.time, - header, + prev_hash: self.hdr.hash_prev_block(), + time: self.hdr.time(), + header: Vec::new(), vtx, chain_metadata: Some(ChainMetadata { sapling_commitment_tree_size, orchard_commitment_tree_size, + ironwood_commitment_tree_size, }), - }; - - Ok(compact_block) - } - - #[deprecated] - /// Converts a zcash full block into a **legacy** compact block. - pub fn into_compact( - self, - sapling_commitment_tree_size: u32, - orchard_commitment_tree_size: u32, - ) -> Result { - self.into_compact_block( - sapling_commitment_tree_size, - orchard_commitment_tree_size, - PoolTypeFilter::default(), - ) - } - - /// Extracts the block height from the coinbase transaction. - fn get_block_height(transactions: &[FullTransaction]) -> Result { - let transparent_inputs = transactions[0].transparent_inputs(); - let (_, _, script_sig) = transparent_inputs[0].clone(); - let coinbase_script = script_sig.as_slice(); - - let mut cursor = Cursor::new(coinbase_script); - - let height_num: i64 = read_zcash_script_i64(&mut cursor)?; - if height_num < 0 { - return Ok(-1); - } - if height_num > i64::from(u32::MAX) { - return Ok(-1); - } - if (height_num as u32) == GENESIS_TARGET_DIFFICULTY { - return Ok(0); - } - - Ok(height_num as i32) + }) } } diff --git a/packages/zaino-fetch/src/chain/error.rs b/packages/zaino-fetch/src/chain/error.rs index 2d1d6ac19..58575a85b 100644 --- a/packages/zaino-fetch/src/chain/error.rs +++ b/packages/zaino-fetch/src/chain/error.rs @@ -27,6 +27,10 @@ pub enum ParseError { #[error("Prost Decode Error: {0}")] ProstDecodeError(#[from] prost::DecodeError), + /// Zebra consensus serialization error. + #[error("Zcash serialization error: {0}")] + SerializationError(#[from] zebra_chain::serialization::SerializationError), + /// Integer conversion error. #[error("Integer conversion error: {0}")] TryFromIntError(#[from] std::num::TryFromIntError), diff --git a/packages/zaino-fetch/src/chain/transaction.rs b/packages/zaino-fetch/src/chain/transaction.rs index 81586349c..68c784466 100644 --- a/packages/zaino-fetch/src/chain/transaction.rs +++ b/packages/zaino-fetch/src/chain/transaction.rs @@ -1,10 +1,7 @@ //! Transaction fetching and deserialization functionality. -use crate::chain::{ - error::ParseError, - utils::{read_bytes, read_i64, read_u32, read_u64, skip_bytes, CompactSize, ParseFromSlice}, -}; -use std::io::Cursor; +use crate::{chain::error::ParseError, utils::ParseFromSlice}; +use std::{io::Cursor, sync::Arc}; use zaino_proto::proto::{ compact_formats::{ CompactOrchardAction, CompactSaplingOutput, CompactSaplingSpend, CompactTx, CompactTxIn, @@ -12,933 +9,22 @@ use zaino_proto::proto::{ }, utils::PoolTypeFilter, }; - -/// Txin format as described in -#[derive(Debug, Clone)] -pub struct TxIn { - // PrevTxHash - Size\[bytes\]: 32 - prev_txid: Vec, - // PrevTxOutIndex - Size\[bytes\]: 4 - prev_index: u32, - /// CompactSize-prefixed, could be a pubkey or a script - /// - /// Size\[bytes\]: CompactSize - script_sig: Vec, - // SequenceNumber \[IGNORED\] - Size\[bytes\]: 4 -} - -impl TxIn { - fn into_inner(self) -> (Vec, u32, Vec) { - (self.prev_txid, self.prev_index, self.script_sig) - } - - /// Returns `true` if this `OutPoint` is "null" in the Bitcoin sense: it has txid set to - /// all-zeroes and output index set to `u32::MAX`. - fn is_null(&self) -> bool { - self.prev_txid.as_slice() == [0u8; 32] && self.prev_index == u32::MAX - } -} - -impl ParseFromSlice for TxIn { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for TxIn::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for TxIn::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let prev_txid = read_bytes(&mut cursor, 32, "Error reading TxIn::PrevTxHash")?; - let prev_index = read_u32(&mut cursor, "Error reading TxIn::PrevTxOutIndex")?; - let script_sig = { - let compact_length = CompactSize::read(&mut cursor)?; - read_bytes( - &mut cursor, - compact_length as usize, - "Error reading TxIn::ScriptSig", - )? - }; - skip_bytes(&mut cursor, 4, "Error skipping TxIn::SequenceNumber")?; - - Ok(( - &data[cursor.position() as usize..], - TxIn { - prev_txid, - prev_index, - script_sig, - }, - )) - } -} - -/// Txout format as described in -#[derive(Debug, Clone)] -pub struct TxOut { - /// Non-negative int giving the number of zatoshis to be transferred - /// - /// Size\[bytes\]: 8 - value: u64, - // Script - Size\[bytes\]: CompactSize - script_hash: Vec, -} - -impl TxOut { - fn into_inner(self) -> (u64, Vec) { - (self.value, self.script_hash) - } -} - -impl ParseFromSlice for TxOut { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for TxOut::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for TxOut::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let value = read_u64(&mut cursor, "Error TxOut::reading Value")?; - let script_hash = { - let compact_length = CompactSize::read(&mut cursor)?; - read_bytes( - &mut cursor, - compact_length as usize, - "Error reading TxOut::ScriptHash", - )? - }; - - Ok(( - &data[cursor.position() as usize..], - TxOut { script_hash, value }, - )) - } -} - -#[allow(clippy::type_complexity)] -fn parse_transparent(data: &[u8]) -> Result<(&[u8], Vec, Vec), ParseError> { - let mut cursor = Cursor::new(data); - - let tx_in_count = CompactSize::read(&mut cursor)?; - let mut tx_ins = Vec::with_capacity(tx_in_count as usize); - for _ in 0..tx_in_count { - let (remaining_data, tx_in) = - TxIn::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - tx_ins.push(tx_in); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let tx_out_count = CompactSize::read(&mut cursor)?; - let mut tx_outs = Vec::with_capacity(tx_out_count as usize); - for _ in 0..tx_out_count { - let (remaining_data, tx_out) = - TxOut::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - tx_outs.push(tx_out); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - Ok((&data[cursor.position() as usize..], tx_ins, tx_outs)) -} - -/// Spend is a Sapling Spend Description as described in 7.3 of the Zcash -/// protocol specification. -#[derive(Debug, Clone)] -pub struct Spend { - // Cv \[IGNORED\] - Size\[bytes\]: 32 - // Anchor \[IGNORED\] - Size\[bytes\]: 32 - /// A nullifier to a sapling note. - /// - /// Size\[bytes\]: 32 - nullifier: Vec, - // Rk \[IGNORED\] - Size\[bytes\]: 32 - // Zkproof \[IGNORED\] - Size\[bytes\]: 192 - // SpendAuthSig \[IGNORED\] - Size\[bytes\]: 64 -} - -impl Spend { - fn into_inner(self) -> Vec { - self.nullifier - } -} - -impl ParseFromSlice for Spend { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for Spend::parse_from_slice".to_string(), - )); - } - let tx_version = tx_version.ok_or_else(|| { - ParseError::InvalidData( - "tx_version must be used for Spend::parse_from_slice".to_string(), - ) - })?; - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 32, "Error skipping Spend::Cv")?; - if tx_version <= 4 { - skip_bytes(&mut cursor, 32, "Error skipping Spend::Anchor")?; - } - let nullifier = read_bytes(&mut cursor, 32, "Error reading Spend::nullifier")?; - skip_bytes(&mut cursor, 32, "Error skipping Spend::Rk")?; - if tx_version <= 4 { - skip_bytes(&mut cursor, 192, "Error skipping Spend::Zkproof")?; - skip_bytes(&mut cursor, 64, "Error skipping Spend::SpendAuthSig")?; - } - - Ok((&data[cursor.position() as usize..], Spend { nullifier })) - } -} - -/// output is a Sapling Output Description as described in section 7.4 of the -/// Zcash protocol spec. -#[derive(Debug, Clone)] -pub struct Output { - // Cv \[IGNORED\] - Size\[bytes\]: 32 - /// U-coordinate of the note commitment, derived from the note's value, recipient, and a - /// random value. - /// - /// Size\[bytes\]: 32 - cmu: Vec, - /// Ephemeral public key for Diffie-Hellman key exchange. - /// - /// Size\[bytes\]: 32 - ephemeral_key: Vec, - /// Encrypted transaction details including value transferred and an optional memo. - /// - /// Size\[bytes\]: 580 - enc_ciphertext: Vec, - // OutCiphertext \[IGNORED\] - Size\[bytes\]: 80 - // Zkproof \[IGNORED\] - Size\[bytes\]: 192 -} - -impl Output { - fn into_parts(self) -> (Vec, Vec, Vec) { - (self.cmu, self.ephemeral_key, self.enc_ciphertext) - } -} - -impl ParseFromSlice for Output { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for Output::parse_from_slice".to_string(), - )); - } - let tx_version = tx_version.ok_or_else(|| { - ParseError::InvalidData( - "tx_version must be used for Output::parse_from_slice".to_string(), - ) - })?; - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 32, "Error skipping Output::Cv")?; - let cmu = read_bytes(&mut cursor, 32, "Error reading Output::cmu")?; - let ephemeral_key = read_bytes(&mut cursor, 32, "Error reading Output::ephemeral_key")?; - let enc_ciphertext = read_bytes(&mut cursor, 580, "Error reading Output::enc_ciphertext")?; - skip_bytes(&mut cursor, 80, "Error skipping Output::OutCiphertext")?; - if tx_version <= 4 { - skip_bytes(&mut cursor, 192, "Error skipping Output::Zkproof")?; - } - - Ok(( - &data[cursor.position() as usize..], - Output { - cmu, - ephemeral_key, - enc_ciphertext, - }, - )) - } -} - -/// joinSplit is a JoinSplit description as described in 7.2 of the Zcash -/// protocol spec. Its exact contents differ by transaction version and network -/// upgrade level. Only version 4 is supported, no need for proofPHGR13. -/// -/// NOTE: Legacy, no longer used but included for consistency. -#[derive(Debug, Clone)] -struct JoinSplit { - //vpubOld \[IGNORED\] - Size\[bytes\]: 8 - //vpubNew \[IGNORED\] - Size\[bytes\]: 8 - //anchor \[IGNORED\] - Size\[bytes\]: 32 - //nullifiers \[IGNORED\] - Size\[bytes\]: 64/32 - //commitments \[IGNORED\] - Size\[bytes\]: 64/32 - //ephemeralKey \[IGNORED\] - Size\[bytes\]: 32 - //randomSeed \[IGNORED\] - Size\[bytes\]: 32 - //vmacs \[IGNORED\] - Size\[bytes\]: 64/32 - //proofGroth16 \[IGNORED\] - Size\[bytes\]: 192 - //encCiphertexts \[IGNORED\] - Size\[bytes\]: 1202 -} - -impl ParseFromSlice for JoinSplit { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for JoinSplit::parse_from_slice".to_string(), - )); - } - let proof_size = match tx_version { - Some(2) | Some(3) => 296, // BCTV14 proof for v2/v3 transactions - Some(4) => 192, // Groth16 proof for v4 transactions - None => 192, // Default to Groth16 for unknown versions - _ => { - return Err(ParseError::InvalidData(format!( - "Unsupported tx_version {tx_version:?} for JoinSplit::parse_from_slice" - ))) - } - }; - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 8, "Error skipping JoinSplit::vpubOld")?; - skip_bytes(&mut cursor, 8, "Error skipping JoinSplit::vpubNew")?; - skip_bytes(&mut cursor, 32, "Error skipping JoinSplit::anchor")?; - skip_bytes(&mut cursor, 64, "Error skipping JoinSplit::nullifiers")?; - skip_bytes(&mut cursor, 64, "Error skipping JoinSplit::commitments")?; - skip_bytes(&mut cursor, 32, "Error skipping JoinSplit::ephemeralKey")?; - skip_bytes(&mut cursor, 32, "Error skipping JoinSplit::randomSeed")?; - skip_bytes(&mut cursor, 64, "Error skipping JoinSplit::vmacs")?; - skip_bytes( - &mut cursor, - proof_size, - &format!("Error skipping JoinSplit::proof (size {proof_size})"), - )?; - skip_bytes( - &mut cursor, - 1202, - "Error skipping JoinSplit::encCiphertexts", - )?; - - Ok((&data[cursor.position() as usize..], JoinSplit {})) - } -} - -/// An Orchard action. -#[derive(Debug, Clone)] -struct Action { - // Cv \[IGNORED\] - Size\[bytes\]: 32 - /// A nullifier to a orchard note. - /// - /// Size\[bytes\]: 32 - nullifier: Vec, - // Rk \[IGNORED\] - Size\[bytes\]: 32 - /// X-coordinate of the commitment to the note. - /// - /// Size\[bytes\]: 32 - cmx: Vec, - /// Ephemeral public key. - /// - /// Size\[bytes\]: 32 - ephemeral_key: Vec, - /// Encrypted details of the new note, including its value and recipient's data. - /// - /// Size\[bytes\]: 580 - enc_ciphertext: Vec, - // OutCiphertext \[IGNORED\] - Size\[bytes\]: 80 -} - -impl Action { - fn into_parts(self) -> (Vec, Vec, Vec, Vec) { - ( - self.nullifier, - self.cmx, - self.ephemeral_key, - self.enc_ciphertext, - ) - } -} - -impl ParseFromSlice for Action { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for Action::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for Action::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 32, "Error skipping Action::Cv")?; - let nullifier = read_bytes(&mut cursor, 32, "Error reading Action::nullifier")?; - skip_bytes(&mut cursor, 32, "Error skipping Action::Rk")?; - let cmx = read_bytes(&mut cursor, 32, "Error reading Action::cmx")?; - let ephemeral_key = read_bytes(&mut cursor, 32, "Error reading Action::ephemeral_key")?; - let enc_ciphertext = read_bytes(&mut cursor, 580, "Error reading Action::enc_ciphertext")?; - skip_bytes(&mut cursor, 80, "Error skipping Action::OutCiphertext")?; - - Ok(( - &data[cursor.position() as usize..], - Action { - nullifier, - cmx, - ephemeral_key, - enc_ciphertext, - }, - )) - } -} - -/// Full Zcash transaction data. -#[derive(Debug, Clone)] -struct TransactionData { - /// Indicates if the transaction is an Overwinter-enabled transaction. - /// - /// Size\[bytes\]: [in 4 byte header] - f_overwintered: bool, - /// The transaction format version. - /// - /// Size\[bytes\]: [in 4 byte header] - version: u32, - /// Version group ID, used to specify transaction type and validate its components. - /// - /// Size\[bytes\]: 4 - n_version_group_id: Option, - /// Consensus branch ID, used to identify the network upgrade that the transaction is valid for. - /// - /// Size\[bytes\]: 4 - consensus_branch_id: u32, - /// List of transparent inputs in a transaction. - /// - /// Size\[bytes\]: Vec<40+CompactSize> - transparent_inputs: Vec, - /// List of transparent outputs in a transaction. - /// - /// Size\[bytes\]: Vec<8+CompactSize> - transparent_outputs: Vec, - // NLockTime \[IGNORED\] - Size\[bytes\]: 4 - // NExpiryHeight \[IGNORED\] - Size\[bytes\]: 4 - // ValueBalanceSapling - Size\[bytes\]: 8 - /// Value balance for the Sapling pool (v4/v5). None if not present. - value_balance_sapling: Option, - /// List of shielded spends from the Sapling pool - /// - /// Size\[bytes\]: Vec<384> - shielded_spends: Vec, - /// List of shielded outputs from the Sapling pool - /// - /// Size\[bytes\]: Vec<948> - shielded_outputs: Vec, - /// List of JoinSplit descriptions in a transaction, no longer supported. - /// - /// Size\[bytes\]: Vec<1602-1698> - #[allow(dead_code)] - join_splits: Vec, - /// joinSplitPubKey \[IGNORED\] - Size\[bytes\]: 32 - /// joinSplitSig \[IGNORED\] - Size\[bytes\]: 64 - /// bindingSigSapling \[IGNORED\] - Size\[bytes\]: 64 - /// List of Orchard actions. - /// - /// Size\[bytes\]: Vec<820> - orchard_actions: Vec, - /// ValueBalanceOrchard - Size\[bytes\]: 8 - /// Value balance for the Orchard pool (v5 only). None if not present. - value_balance_orchard: Option, - /// AnchorOrchard - Size\[bytes\]: 32 - /// In non-coinbase transactions, this is the anchor (authDataRoot) of a prior block's Orchard note commitment tree. - /// In the coinbase transaction, this commits to the final Orchard tree state for the current block — i.e., it *is* the block's authDataRoot. - /// Present in v5 transactions only, if any Orchard actions exist in the block. - anchor_orchard: Option>, -} - -impl TransactionData { - /// Parses a v1 transaction. - /// - /// A v1 transaction contains the following fields: - /// - /// - header: u32 - /// - tx_in_count: usize - /// - tx_in: tx_in - /// - tx_out_count: usize - /// - tx_out: tx_out - /// - lock_time: u32 - pub(crate) fn parse_v1(data: &[u8], version: u32) -> Result<(&[u8], Self), ParseError> { - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - // let lock_time = read_u32(&mut cursor, "Error reading TransactionData::lock_time")?; - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - // lock_time: Some(lock_time), - n_version_group_id: None, - value_balance_sapling: None, - shielded_spends: Vec::new(), - shielded_outputs: Vec::new(), - join_splits: Vec::new(), - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - /// Parses a v2 transaction. - /// - /// A v2 transaction contains the following fields: - /// - /// - header: u32 - /// - tx_in_count: usize - /// - tx_in: tx_in - /// - tx_out_count: usize - /// - tx_out: tx_out - /// - lock_time: u32 - /// - nJoinSplit: compactSize <- New - /// - vJoinSplit: JSDescriptionBCTV14\[nJoinSplit\] <- New - /// - joinSplitPubKey: byte\[32\] <- New - /// - joinSplitSig: byte\[64\] <- New - pub(crate) fn parse_v2(data: &[u8], version: u32) -> Result<(&[u8], Self), ParseError> { - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - - let join_split_count = CompactSize::read(&mut cursor)?; - let mut join_splits = Vec::with_capacity(join_split_count as usize); - for _ in 0..join_split_count { - let (remaining_data, join_split) = JoinSplit::parse_from_slice( - &data[cursor.position() as usize..], - None, - Some(version), - )?; - join_splits.push(join_split); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - if join_split_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::joinSplitPubKey", - )?; - skip_bytes( - &mut cursor, - 64, - "could not skip TransactionData::joinSplitSig", - )?; - } - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - join_splits, - n_version_group_id: None, - value_balance_sapling: None, - shielded_spends: Vec::new(), - shielded_outputs: Vec::new(), - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - /// Parses a v3 transaction. - /// - /// A v3 transaction contains the following fields: - /// - /// - header: u32 - /// - nVersionGroupId: u32 = 0x03C48270 <- New - /// - tx_in_count: usize - /// - tx_in: tx_in - /// - tx_out_count: usize - /// - tx_out: tx_out - /// - lock_time: u32 - /// - nExpiryHeight: u32 <- New - /// - nJoinSplit: compactSize - /// - vJoinSplit: JSDescriptionBCTV14\[nJoinSplit\] - /// - joinSplitPubKey: byte\[32\] - /// - joinSplitSig: byte\[64\] - pub(crate) fn parse_v3( - data: &[u8], - version: u32, - n_version_group_id: u32, - ) -> Result<(&[u8], Self), ParseError> { - if n_version_group_id != 0x03C48270 { - return Err(ParseError::InvalidData( - "n_version_group_id must be 0x03C48270".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - skip_bytes( - &mut cursor, - 4, - "Error skipping TransactionData::nExpiryHeight", - )?; - - let join_split_count = CompactSize::read(&mut cursor)?; - let mut join_splits = Vec::with_capacity(join_split_count as usize); - for _ in 0..join_split_count { - let (remaining_data, join_split) = JoinSplit::parse_from_slice( - &data[cursor.position() as usize..], - None, - Some(version), - )?; - join_splits.push(join_split); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - if join_split_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::joinSplitPubKey", - )?; - skip_bytes( - &mut cursor, - 64, - "could not skip TransactionData::joinSplitSig", - )?; - } - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - join_splits, - n_version_group_id: None, - value_balance_sapling: None, - shielded_spends: Vec::new(), - shielded_outputs: Vec::new(), - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - fn parse_v4( - data: &[u8], - version: u32, - n_version_group_id: u32, - ) -> Result<(&[u8], Self), ParseError> { - if n_version_group_id != 0x892F2085 { - return Err(ParseError::InvalidData(format!( - "version group ID {n_version_group_id:x} must be 0x892F2085 for v4 transactions" - ))); - } - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - skip_bytes( - &mut cursor, - 4, - "Error skipping TransactionData::nExpiryHeight", - )?; - let value_balance_sapling = Some(read_i64( - &mut cursor, - "Error reading TransactionData::valueBalanceSapling", - )?); - - let spend_count = CompactSize::read(&mut cursor)?; - let mut shielded_spends = Vec::with_capacity(spend_count as usize); - for _ in 0..spend_count { - let (remaining_data, spend) = - Spend::parse_from_slice(&data[cursor.position() as usize..], None, Some(4))?; - shielded_spends.push(spend); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let output_count = CompactSize::read(&mut cursor)?; - let mut shielded_outputs = Vec::with_capacity(output_count as usize); - for _ in 0..output_count { - let (remaining_data, output) = - Output::parse_from_slice(&data[cursor.position() as usize..], None, Some(4))?; - shielded_outputs.push(output); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let join_split_count = CompactSize::read(&mut cursor)?; - let mut join_splits = Vec::with_capacity(join_split_count as usize); - for _ in 0..join_split_count { - let (remaining_data, join_split) = JoinSplit::parse_from_slice( - &data[cursor.position() as usize..], - None, - Some(version), - )?; - join_splits.push(join_split); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - if join_split_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::joinSplitPubKey", - )?; - skip_bytes( - &mut cursor, - 64, - "could not skip TransactionData::joinSplitSig", - )?; - } - if spend_count + output_count > 0 { - skip_bytes( - &mut cursor, - 64, - "Error skipping TransactionData::bindingSigSapling", - )?; - } - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - n_version_group_id: Some(n_version_group_id), - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - value_balance_sapling, - shielded_spends, - shielded_outputs, - join_splits, - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - fn parse_v5( - data: &[u8], - version: u32, - n_version_group_id: u32, - ) -> Result<(&[u8], Self), ParseError> { - if n_version_group_id != 0x26A7270A { - return Err(ParseError::InvalidData(format!( - "version group ID {n_version_group_id:x} must be 0x892F2085 for v5 transactions" - ))); - } - let mut cursor = Cursor::new(data); - - let consensus_branch_id = read_u32( - &mut cursor, - "Error reading TransactionData::ConsensusBranchId", - )?; - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - skip_bytes( - &mut cursor, - 4, - "Error skipping TransactionData::nExpiryHeight", - )?; - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - let spend_count = CompactSize::read(&mut cursor)?; - if spend_count >= (1 << 16) { - return Err(ParseError::InvalidData(format!( - "spendCount ({spend_count}) must be less than 2^16" - ))); - } - let mut shielded_spends = Vec::with_capacity(spend_count as usize); - for _ in 0..spend_count { - let (remaining_data, spend) = - Spend::parse_from_slice(&data[cursor.position() as usize..], None, Some(5))?; - shielded_spends.push(spend); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let output_count = CompactSize::read(&mut cursor)?; - if output_count >= (1 << 16) { - return Err(ParseError::InvalidData(format!( - "outputCount ({output_count}) must be less than 2^16" - ))); - } - let mut shielded_outputs = Vec::with_capacity(output_count as usize); - for _ in 0..output_count { - let (remaining_data, output) = - Output::parse_from_slice(&data[cursor.position() as usize..], None, Some(5))?; - shielded_outputs.push(output); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - let value_balance_sapling = if spend_count + output_count > 0 { - Some(read_i64( - &mut cursor, - "Error reading TransactionData::valueBalanceSapling", - )?) - } else { - None - }; - if spend_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::anchorSapling", - )?; - skip_bytes( - &mut cursor, - (192 * spend_count) as usize, - "Error skipping TransactionData::vSpendProofsSapling", - )?; - skip_bytes( - &mut cursor, - (64 * spend_count) as usize, - "Error skipping TransactionData::vSpendAuthSigsSapling", - )?; - } - if output_count > 0 { - skip_bytes( - &mut cursor, - (192 * output_count) as usize, - "Error skipping TransactionData::vOutputProofsSapling", - )?; - } - if spend_count + output_count > 0 { - skip_bytes( - &mut cursor, - 64, - "Error skipping TransactionData::bindingSigSapling", - )?; - } - - let actions_count = CompactSize::read(&mut cursor)?; - if actions_count >= (1 << 16) { - return Err(ParseError::InvalidData(format!( - "actionsCount ({actions_count}) must be less than 2^16" - ))); - } - let mut orchard_actions = Vec::with_capacity(actions_count as usize); - for _ in 0..actions_count { - let (remaining_data, action) = - Action::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - orchard_actions.push(action); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - let mut value_balance_orchard = None; - let mut anchor_orchard = None; - if actions_count > 0 { - skip_bytes( - &mut cursor, - 1, - "Error skipping TransactionData::flagsOrchard", - )?; - value_balance_orchard = Some(read_i64( - &mut cursor, - "Error reading TransactionData::valueBalanceOrchard", - )?); - anchor_orchard = Some(read_bytes( - &mut cursor, - 32, - "Error reading TransactionData::anchorOrchard", - )?); - let proofs_count = CompactSize::read(&mut cursor)?; - skip_bytes( - &mut cursor, - proofs_count as usize, - "Error skipping TransactionData::proofsOrchard", - )?; - skip_bytes( - &mut cursor, - (64 * actions_count) as usize, - "Error skipping TransactionData::vSpendAuthSigsOrchard", - )?; - skip_bytes( - &mut cursor, - 64, - "Error skipping TransactionData::bindingSigOrchard", - )?; - } - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - n_version_group_id: Some(n_version_group_id), - consensus_branch_id, - transparent_inputs, - transparent_outputs, - value_balance_sapling, - shielded_spends, - shielded_outputs, - join_splits: Vec::new(), - orchard_actions, - value_balance_orchard, - anchor_orchard, - }, - )) - } -} +use zebra_chain::{ + parameters::{OVERWINTER_VERSION_GROUP_ID, SAPLING_VERSION_GROUP_ID, TX_V5_VERSION_GROUP_ID}, + serialization::{ZcashDeserialize as _, ZcashSerialize as _}, + transparent, +}; /// Zingo-Indexer struct for a full zcash transaction. #[derive(Debug, Clone)] pub struct FullTransaction { - /// Full transaction data. - raw_transaction: TransactionData, + /// Parsed Zebra transaction. + transaction: Arc, /// Raw transaction bytes. raw_bytes: Vec, - /// Transaction Id, fetched using get_block JsonRPC with verbose = 1. + /// Transaction Id, fetched using get_block JsonRPC with verbose = 1 when available. tx_id: Vec, } @@ -948,154 +34,167 @@ impl ParseFromSlice for FullTransaction { txid: Option>>, tx_version: Option, ) -> Result<(&[u8], Self), ParseError> { - let txid = txid.ok_or_else(|| { - ParseError::InvalidData( - "txid must be used for FullTransaction::parse_from_slice".to_string(), - ) - })?; - // TODO: 🤯 if tx_version.is_some() { return Err(ParseError::InvalidData( "tx_version must be None for FullTransaction::parse_from_slice".to_string(), )); } - let mut cursor = Cursor::new(data); - - let header = read_u32(&mut cursor, "Error reading FullTransaction::header")?; - let f_overwintered = (header >> 31) == 1; - - let version = header & 0x7FFFFFFF; - - match version { - 1 | 2 => { - if f_overwintered { - return Err(ParseError::InvalidData( - "fOverwintered must be unset for tx versions 1 and 2".to_string(), - )); - } - } - 3..=5 => { - if !f_overwintered { - return Err(ParseError::InvalidData( - "fOverwintered must be set for tx versions 3 and above".to_string(), - )); - } - } - _ => { - return Err(ParseError::InvalidData(format!( - "Unsupported tx version {version}" - ))) - } - } - - let n_version_group_id: Option = match version { - 3..=5 => Some(read_u32( - &mut cursor, - "Error reading FullTransaction::n_version_group_id", - )?), - _ => None, - }; - - let (remaining_data, transaction_data) = match version { - 1 => TransactionData::parse_v1(&data[cursor.position() as usize..], version)?, - 2 => TransactionData::parse_v2(&data[cursor.position() as usize..], version)?, - 3 => TransactionData::parse_v3( - &data[cursor.position() as usize..], - version, - n_version_group_id.unwrap(), // This won't fail, because of the above match - )?, - 4 => TransactionData::parse_v4( - &data[cursor.position() as usize..], - version, - n_version_group_id.unwrap(), // This won't fail, because of the above match - )?, - 5 => TransactionData::parse_v5( - &data[cursor.position() as usize..], - version, - n_version_group_id.unwrap(), // This won't fail, because of the above match - )?, - - _ => { - return Err(ParseError::InvalidData(format!( - "Unsupported tx version {version}" - ))) - } - }; - let full_transaction = FullTransaction { - raw_transaction: transaction_data, - raw_bytes: data[..(data.len() - remaining_data.len())].to_vec(), - tx_id: txid[0].clone(), - }; + let mut cursor = Cursor::new(data); + let transaction = zebra_chain::transaction::Transaction::zcash_deserialize(&mut cursor)?; + let consumed = usize::try_from(cursor.position())?; + let tx_id = txid + .and_then(|txids| txids.into_iter().next()) + .unwrap_or_else(|| transaction.hash().0.to_vec()); - Ok((remaining_data, full_transaction)) + Ok(( + &data[consumed..], + Self { + transaction: Arc::new(transaction), + raw_bytes: data[..consumed].to_vec(), + tx_id, + }, + )) } } impl FullTransaction { - /// Returns overwintered bool + pub(super) fn from_zebra( + transaction: Arc, + tx_id: Option>, + ) -> Result { + let raw_bytes = transaction.zcash_serialize_to_vec()?; + let tx_id = tx_id.unwrap_or_else(|| transaction.hash().0.to_vec()); + + Ok(Self { + transaction, + raw_bytes, + tx_id, + }) + } + + /// Returns overwintered bool. pub fn f_overwintered(&self) -> bool { - self.raw_transaction.f_overwintered + self.transaction.is_overwintered() } /// Returns the transaction version. pub fn version(&self) -> u32 { - self.raw_transaction.version + self.transaction.version() } /// Returns the transaction version group id. pub fn n_version_group_id(&self) -> Option { - self.raw_transaction.n_version_group_id + match self.version() { + 3 => Some(OVERWINTER_VERSION_GROUP_ID), + 4 => Some(SAPLING_VERSION_GROUP_ID), + 5 => Some(TX_V5_VERSION_GROUP_ID), + 6 => Some(zebra_chain::parameters::TX_V6_VERSION_GROUP_ID), + _ => None, + } } - /// returns the consensus branch id of the transaction. + /// Returns the consensus branch id of the transaction. pub fn consensus_branch_id(&self) -> u32 { - self.raw_transaction.consensus_branch_id + self.transaction + .network_upgrade() + .and_then(|network_upgrade| network_upgrade.branch_id()) + .map(u32::from) + .unwrap_or(0) } /// Returns a vec of transparent inputs: (prev_txid, prev_index, script_sig). pub fn transparent_inputs(&self) -> Vec<(Vec, u32, Vec)> { - self.raw_transaction - .transparent_inputs + self.transaction + .inputs() .iter() - .map(|input| input.clone().into_inner()) + .map(|input| match input { + transparent::Input::PrevOut { + outpoint, + unlock_script, + .. + } => ( + outpoint.hash.0.to_vec(), + outpoint.index, + unlock_script.as_raw_bytes().to_vec(), + ), + transparent::Input::Coinbase { .. } => ( + vec![0; 32], + u32::MAX, + input.coinbase_script().unwrap_or_default(), + ), + }) .collect() } /// Returns a vec of transparent outputs: (value, script_hash). pub fn transparent_outputs(&self) -> Vec<(u64, Vec)> { - self.raw_transaction - .transparent_outputs + self.transaction + .outputs() .iter() - .map(|output| output.clone().into_inner()) + .filter_map(|output| { + u64::try_from(output.value().zatoshis()) + .ok() + .map(|value| (value, output.lock_script.as_raw_bytes().to_vec())) + }) .collect() } - /// Returns sapling and orchard value balances for the transaction. + /// Returns sapling, orchard, and ironwood value balances for the transaction. /// - /// Returned as (Option\, Option\). - pub fn value_balances(&self) -> (Option, Option) { - ( - self.raw_transaction.value_balance_sapling, - self.raw_transaction.value_balance_orchard, - ) + /// Returned as (Option\, Option\, + /// Option\). + pub fn value_balances(&self) -> (Option, Option, Option) { + let sapling = if self.version() == 4 || self.transaction.has_sapling_shielded_data() { + Some( + self.transaction + .sapling_value_balance() + .sapling_amount() + .zatoshis(), + ) + } else { + None + }; + + let orchard = self.transaction.has_orchard_shielded_data().then(|| { + self.transaction + .orchard_value_balance() + .orchard_amount() + .zatoshis() + }); + + let ironwood = self.transaction.has_ironwood_shielded_data().then(|| { + self.transaction + .ironwood_value_balance() + .ironwood_amount() + .zatoshis() + }); + + (sapling, orchard, ironwood) } /// Returns a vec of sapling nullifiers for the transaction. pub fn shielded_spends(&self) -> Vec> { - self.raw_transaction - .shielded_spends - .iter() - .map(|input| input.clone().into_inner()) + self.transaction + .sapling_nullifiers() + .map(|nullifier| <[u8; 32]>::from(*nullifier).to_vec()) .collect() } /// Returns a vec of sapling outputs (cmu, ephemeral_key, enc_ciphertext) for the transaction. pub fn shielded_outputs(&self) -> Vec<(Vec, Vec, Vec)> { - self.raw_transaction - .shielded_outputs - .iter() - .map(|input| input.clone().into_parts()) + self.transaction + .sapling_outputs() + .map(|output| { + let ephemeral_key: [u8; 32] = (&output.ephemeral_key).into(); + let enc_ciphertext: [u8; 580] = output.enc_ciphertext.into(); + + ( + output.cm_u.to_bytes().to_vec(), + ephemeral_key.to_vec(), + enc_ciphertext.to_vec(), + ) + }) .collect() } @@ -1107,10 +206,42 @@ impl FullTransaction { /// Returns a vec of orchard actions (nullifier, cmx, ephemeral_key, enc_ciphertext) for the transaction. #[allow(clippy::complexity)] pub fn orchard_actions(&self) -> Vec<(Vec, Vec, Vec, Vec)> { - self.raw_transaction - .orchard_actions - .iter() - .map(|input| input.clone().into_parts()) + self.transaction + .orchard_actions() + .map(|action| { + let nullifier: [u8; 32] = action.nullifier.into(); + let cmx: [u8; 32] = action.cm_x.into(); + let ephemeral_key: [u8; 32] = (&action.ephemeral_key).into(); + let enc_ciphertext: [u8; 580] = action.enc_ciphertext.into(); + + ( + nullifier.to_vec(), + cmx.to_vec(), + ephemeral_key.to_vec(), + enc_ciphertext.to_vec(), + ) + }) + .collect() + } + + /// Returns a vec of ironwood actions (nullifier, cmx, ephemeral_key, enc_ciphertext) for the transaction. + #[allow(clippy::complexity)] + pub fn ironwood_actions(&self) -> Vec<(Vec, Vec, Vec, Vec)> { + self.transaction + .ironwood_actions() + .map(|action| { + let nullifier: [u8; 32] = action.nullifier.into(); + let cmx: [u8; 32] = action.cm_x.into(); + let ephemeral_key: [u8; 32] = (&action.ephemeral_key).into(); + let enc_ciphertext: [u8; 580] = action.enc_ciphertext.into(); + + ( + nullifier.to_vec(), + cmx.to_vec(), + ephemeral_key.to_vec(), + enc_ciphertext.to_vec(), + ) + }) .collect() } @@ -1118,7 +249,9 @@ impl FullTransaction { /// /// If this is the Coinbase transaction then this returns the AuthDataRoot of the block. pub fn anchor_orchard(&self) -> Option> { - self.raw_transaction.anchor_orchard.clone() + self.transaction + .orchard_shielded_data() + .map(|shielded_data| <[u8; 32]>::from(&shielded_data.shared_anchor).to_vec()) } /// Returns the transaction as raw bytes. @@ -1126,7 +259,7 @@ impl FullTransaction { self.raw_bytes.clone() } - /// returns the TxId of the transaction. + /// Returns the TxId of the transaction. pub fn tx_id(&self) -> Vec { self.tx_id.clone() } @@ -1138,101 +271,106 @@ impl FullTransaction { } /// Converts a Zcash Transaction into a `CompactTx` of the Light wallet protocol. - /// if the transaction you want to convert is a mempool transaction you can specify `None`. - /// specify the `PoolType`s that the transaction should include in the `pool_types` argument + /// If the transaction you want to convert is a mempool transaction you can specify `None`. + /// Specify the `PoolType`s that the transaction should include in the `pool_types` argument /// with a `PoolTypeFilter` indicating which pools the compact block should include. pub fn to_compact_tx( self, index: Option, pool_types: &PoolTypeFilter, ) -> Result { - let hash = self.tx_id(); - - // NOTE: LightWalletD currently does not return a fee and is not currently priority here. - // Please open an Issue or PR at the Zingo-Indexer github (https://github.com/zingolabs/zingo-indexer) - // if you require this functionality. - let fee = 0; - let spends = if pool_types.includes_sapling() { - self.raw_transaction - .shielded_spends - .iter() - .map(|spend| CompactSaplingSpend { - nf: spend.nullifier.clone(), - }) + self.shielded_spends() + .into_iter() + .map(|nf| CompactSaplingSpend { nf }) .collect() } else { - vec![] + Vec::new() }; let outputs = if pool_types.includes_sapling() { - self.raw_transaction - .shielded_outputs - .iter() - .map(|output| CompactSaplingOutput { - cmu: output.cmu.clone(), - ephemeral_key: output.ephemeral_key.clone(), - ciphertext: output.enc_ciphertext[..52].to_vec(), - }) + self.shielded_outputs() + .into_iter() + .map( + |(cmu, ephemeral_key, enc_ciphertext)| CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext[..52].to_vec(), + }, + ) .collect() } else { - vec![] + Vec::new() }; let actions = if pool_types.includes_orchard() { - self.raw_transaction - .orchard_actions - .iter() - .map(|action| CompactOrchardAction { - nullifier: action.nullifier.clone(), - cmx: action.cmx.clone(), - ephemeral_key: action.ephemeral_key.clone(), - ciphertext: action.enc_ciphertext[..52].to_vec(), - }) + self.orchard_actions() + .into_iter() + .map( + |(nullifier, cmx, ephemeral_key, enc_ciphertext)| CompactOrchardAction { + nullifier, + cmx, + ephemeral_key, + ciphertext: enc_ciphertext[..52].to_vec(), + }, + ) .collect() } else { - vec![] + Vec::new() + }; + + let ironwood_actions = if pool_types.includes_ironwood() { + self.ironwood_actions() + .into_iter() + .map( + |(nullifier, cmx, ephemeral_key, enc_ciphertext)| CompactOrchardAction { + nullifier, + cmx, + ephemeral_key, + ciphertext: enc_ciphertext[..52].to_vec(), + }, + ) + .collect() + } else { + Vec::new() }; let vout = if pool_types.includes_transparent() { - self.raw_transaction - .transparent_outputs - .iter() - .map(|t_out| CompactTxOut { - value: t_out.value, - script_pub_key: t_out.script_hash.clone(), + self.transparent_outputs() + .into_iter() + .map(|(value, script_hash)| CompactTxOut { + value, + script_pub_key: script_hash, }) .collect() } else { - vec![] + Vec::new() }; let vin = if pool_types.includes_transparent() { - self.raw_transaction - .transparent_inputs + self.transaction + .inputs() .iter() - .filter_map(|t_in| { - if t_in.is_null() { - None - } else { - Some(CompactTxIn { - prevout_txid: t_in.prev_txid.clone(), - prevout_index: t_in.prev_index, - }) - } + .filter_map(|input| { + let outpoint = input.outpoint()?; + Some(CompactTxIn { + prevout_txid: outpoint.hash.0.to_vec(), + prevout_index: outpoint.index, + }) }) .collect() } else { - vec![] + Vec::new() }; Ok(CompactTx { index: index.unwrap_or(0), // this assumes that mempool txs have a zeroed index - txid: hash, - fee, + txid: self.tx_id(), + fee: 0, spends, outputs, actions, + ironwood_actions, vin, vout, }) @@ -1241,9 +379,9 @@ impl FullTransaction { /// Returns true if the transaction contains either sapling spends or outputs, or orchard actions. #[allow(dead_code)] pub(crate) fn has_shielded_elements(&self) -> bool { - !self.raw_transaction.shielded_spends.is_empty() - || !self.raw_transaction.shielded_outputs.is_empty() - || !self.raw_transaction.orchard_actions.is_empty() + self.transaction.has_sapling_shielded_data() + || self.transaction.has_orchard_shielded_data() + || self.transaction.has_ironwood_shielded_data() } } @@ -1252,210 +390,36 @@ mod tests { use super::*; use wire_serialized_transaction_test_data::transactions::get_test_vectors; - /// Test parsing v1 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v1 transaction format. #[test] - fn test_v1_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v1_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 1).collect(); - - assert!(!v1_vectors.is_empty(), "No v1 test vectors found"); - - for (i, vector) in v1_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( + fn parses_test_vectors_with_zebra_deserializer() -> Result<(), ParseError> { + for vector in get_test_vectors() { + let (remaining, transaction) = FullTransaction::parse_from_slice( &vector.tx, Some(vec![vector.txid.to_vec()]), None, - ); - - assert!( - result.is_ok(), - "Failed to parse v1 test vector #{}: {:?}. Description: {}", - i, - result.err(), - vector.description - ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v1 transaction #{i}" - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 1, - "Version mismatch for v1 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations - assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), - vector.transparent_inputs, - "Transparent inputs mismatch for v1 transaction #{i}" - ); + )?; + assert!(remaining.is_empty(), "{}", vector.description); assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), - vector.transparent_outputs, - "Transparent outputs mismatch for v1 transaction #{i}" - ); - } - } - - /// Test parsing v2 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v2 transaction format. - #[test] - fn test_v2_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v2_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 2).collect(); - - assert!(!v2_vectors.is_empty(), "No v2 test vectors found"); - - for (i, vector) in v2_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( - &vector.tx, - Some(vec![vector.txid.to_vec()]), - None, - ); - - assert!( - result.is_ok(), - "Failed to parse v2 test vector #{}: {:?}. Description: {}", - i, - result.err(), + transaction.version(), + vector.version, + "{}", vector.description ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v2 transaction #{}: {} bytes remaining, total length: {}", - i, remaining.len(), vector.tx.len() - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 2, - "Version mismatch for v2 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), + transaction.transparent_inputs().len(), vector.transparent_inputs, - "Transparent inputs mismatch for v2 transaction #{i}" - ); - - assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), - vector.transparent_outputs, - "Transparent outputs mismatch for v2 transaction #{i}" - ); - } - } - - /// Test parsing v3 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v3 transaction format. - #[test] - fn test_v3_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v3_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 3).collect(); - - assert!(!v3_vectors.is_empty(), "No v3 test vectors found"); - - for (i, vector) in v3_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( - &vector.tx, - Some(vec![vector.txid.to_vec()]), - None, - ); - - assert!( - result.is_ok(), - "Failed to parse v3 test vector #{}: {:?}. Description: {}", - i, - result.err(), + "{}", vector.description ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v3 transaction #{}: {} bytes remaining, total length: {}", - i, remaining.len(), vector.tx.len() - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 3, - "Version mismatch for v3 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations - assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), - vector.transparent_inputs, - "Transparent inputs mismatch for v3 transaction #{i}" - ); - assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), + transaction.transparent_outputs().len(), vector.transparent_outputs, - "Transparent outputs mismatch for v3 transaction #{i}" - ); - } - } - - /// Test parsing v4 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v4 transaction format. - /// This also serves as a regression test for current v4 functionality. - #[test] - fn test_v4_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v4_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 4).collect(); - - assert!(!v4_vectors.is_empty(), "No v4 test vectors found"); - - for (i, vector) in v4_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( - &vector.tx, - Some(vec![vector.txid.to_vec()]), - None, - ); - - assert!( - result.is_ok(), - "Failed to parse v4 test vector #{}: {:?}. Description: {}", - i, - result.err(), + "{}", vector.description ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v4 transaction #{i}" - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 4, - "Version mismatch for v4 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations - assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), - vector.transparent_inputs, - "Transparent inputs mismatch for v4 transaction #{i}" - ); - - assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), - vector.transparent_outputs, - "Transparent outputs mismatch for v4 transaction #{i}" - ); } + + Ok(()) } } diff --git a/packages/zaino-fetch/src/chain/utils.rs b/packages/zaino-fetch/src/chain/utils.rs index 4570a3a7c..de6b71beb 100644 --- a/packages/zaino-fetch/src/chain/utils.rs +++ b/packages/zaino-fetch/src/chain/utils.rs @@ -1,206 +1,3 @@ -//! Blockcache utility functionality. +//! Compatibility re-exports for chain parsing utilities. -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use std::io::{self, Cursor, Read, Write}; - -use crate::chain::error::ParseError; - -/// Used for decoding zcash blocks from a bytestring. -pub trait ParseFromSlice { - /// Reads data from a bytestring, consuming data read, and returns an instance of self along with the remaining data in the bytestring given. - /// - /// txid is giving as an input as this is taken from a get_block verbose=1 call. - /// - /// tx_version is used for deserializing sapling spends and outputs. - fn parse_from_slice( - data: &[u8], - // TODO: Why is txid a vec of vecs? - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> - where - Self: Sized; -} - -/// Skips the next n bytes in cursor, returns error message given if eof is reached. -pub(crate) fn skip_bytes( - cursor: &mut Cursor<&[u8]>, - n: usize, - error_msg: &str, -) -> Result<(), ParseError> { - if cursor.get_ref().len() < (cursor.position() + n as u64) as usize { - return Err(ParseError::InvalidData(error_msg.to_string())); - } - cursor.set_position(cursor.position() + n as u64); - Ok(()) -} - -/// Reads the next n bytes from cursor into a `vec`, returns error message given if eof is reached. -pub(crate) fn read_bytes( - cursor: &mut Cursor<&[u8]>, - n: usize, - error_msg: &str, -) -> Result, ParseError> { - let mut buf = vec![0; n]; - cursor - .read_exact(&mut buf) - .map_err(|_| ParseError::InvalidData(error_msg.to_string()))?; - Ok(buf) -} - -/// Reads the next 8 bytes from cursor into a u64, returns error message given if eof is reached. -pub(crate) fn read_u64(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_u64::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next 4 bytes from cursor into a u32, returns error message given if eof is reached. -pub(crate) fn read_u32(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_u32::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next 8 bytes from cursor into an i64, returns error message given if eof is reached. -pub(crate) fn read_i64(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_i64::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next 4 bytes from cursor into an i32, returns error message given if eof is reached. -pub(crate) fn read_i32(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_i32::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next byte from cursor into a bool, returns error message given if eof is reached. -#[allow(dead_code)] -pub(crate) fn read_bool(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - let byte = cursor - .read_u8() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string()))?; - match byte { - 0 => Ok(false), - 1 => Ok(true), - _ => Err(ParseError::InvalidData(error_msg.to_string())), - } -} - -/// read_zcash_script_int64 OP codes. -const OP_0: u8 = 0x00; -const OP_1_NEGATE: u8 = 0x4f; -const OP_1: u8 = 0x51; -const OP_16: u8 = 0x60; - -/// Reads and interprets a Zcash (Bitcoin) custom compact integer encoding used for int64 numbers in scripts. -pub(crate) fn read_zcash_script_i64(cursor: &mut Cursor<&[u8]>) -> Result { - let first_byte = read_bytes(cursor, 1, "Error reading first byte in i64 script hash")?[0]; - - match first_byte { - OP_1_NEGATE => Ok(-1), - OP_0 => Ok(0), - OP_1..=OP_16 => Ok((u64::from(first_byte) - u64::from(OP_1 - 1)) as i64), - _ => { - let num_bytes = - read_bytes(cursor, first_byte as usize, "Error reading i64 script hash")?; - let number = num_bytes - .iter() - .rev() - .fold(0, |acc, &byte| (acc << 8) | u64::from(byte)); - Ok(number as i64) - } - } -} - -/// Zcash CompactSize implementation taken from LibRustZcash::zcash_encoding to simplify dependency tree. -/// -/// Namespace for functions for compact encoding of integers. -/// -/// This codec requires integers to be in the range `0x0..=0x02000000`, for compatibility -/// with Zcash consensus rules. -pub(crate) struct CompactSize; - -/// The maximum allowed value representable as a `[CompactSize]` -pub(crate) const MAX_COMPACT_SIZE: u32 = 0x02000000; - -impl CompactSize { - /// Reads an integer encoded in compact form. - pub(crate) fn read(mut reader: R) -> io::Result { - let flag = reader.read_u8()?; - let result = if flag < 253 { - Ok(flag as u64) - } else if flag == 253 { - match reader.read_u16::()? { - n if n < 253 => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "non-canonical CompactSize", - )), - n => Ok(n as u64), - } - } else if flag == 254 { - match reader.read_u32::()? { - n if n < 0x10000 => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "non-canonical CompactSize", - )), - n => Ok(n as u64), - } - } else { - match reader.read_u64::()? { - n if n < 0x100000000 => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "non-canonical CompactSize", - )), - n => Ok(n), - } - }?; - - match result { - s if s > ::from(MAX_COMPACT_SIZE) => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "CompactSize too large", - )), - s => Ok(s), - } - } - - /// Reads an integer encoded in compact form and performs checked conversion - /// to the target type. - #[allow(dead_code)] - pub(crate) fn read_t>(mut reader: R) -> io::Result { - let n = Self::read(&mut reader)?; - ::try_from(n).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidInput, - "CompactSize value exceeds range of target type.", - ) - }) - } - - /// Writes the provided `usize` value to the provided Writer in compact form. - pub(crate) fn write(mut writer: W, size: usize) -> io::Result<()> { - match size { - s if s < 253 => writer.write_u8(s as u8), - s if s <= 0xFFFF => { - writer.write_u8(253)?; - writer.write_u16::(s as u16) - } - s if s <= 0xFFFFFFFF => { - writer.write_u8(254)?; - writer.write_u32::(s as u32) - } - s => { - writer.write_u8(255)?; - writer.write_u64::(s as u64) - } - } - } -} +pub use crate::utils::ParseFromSlice; diff --git a/packages/zaino-fetch/src/jsonrpsee/connector.rs b/packages/zaino-fetch/src/jsonrpsee/connector.rs index cc8090150..056821afa 100644 --- a/packages/zaino-fetch/src/jsonrpsee/connector.rs +++ b/packages/zaino-fetch/src/jsonrpsee/connector.rs @@ -210,6 +210,29 @@ fn record_outbound_rpc_metrics(method: &'static str, start: std::time::Instant, } } +/// Whether the outbound JSON-RPC client keeps a cookie store (required +/// for cookie-authenticated validators). +enum CookieSupport { + Enabled, + Disabled, +} + +/// Builds the outbound JSON-RPC HTTP client shared by every connector +/// entry point: 2s connect timeout, 5s request timeout, no redirects. +fn build_rpc_client(cookies: CookieSupport) -> Result { + // reqwest is built with `rustls-no-provider`, which never + // auto-selects a rustls CryptoProvider (zingolabs/zaino#1360). + zaino_common::crypto::ensure_default_crypto_provider(); + let mut builder = ClientBuilder::new() + .connect_timeout(Duration::from_secs(2)) + .timeout(Duration::from_secs(5)) + .redirect(reqwest::redirect::Policy::none()); + if matches!(cookies, CookieSupport::Enabled) { + builder = builder.cookie_store(true); + } + builder.build() +} + impl JsonRpSeeConnector { /// Creates a new JsonRpSeeConnector with Basic Authentication. pub fn new_with_basic_auth( @@ -217,12 +240,8 @@ impl JsonRpSeeConnector { username: String, password: String, ) -> Result { - let client = ClientBuilder::new() - .connect_timeout(Duration::from_secs(2)) - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(TransportError::ReqwestError)?; + let client = + build_rpc_client(CookieSupport::Disabled).map_err(TransportError::ReqwestError)?; Ok(Self { url, @@ -236,13 +255,8 @@ impl JsonRpSeeConnector { pub fn new_with_cookie_auth(url: Url, cookie_path: &Path) -> Result { let cookie_password = read_and_parse_cookie_token(cookie_path)?; - let client = ClientBuilder::new() - .connect_timeout(Duration::from_secs(2)) - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .cookie_store(true) - .build() - .map_err(TransportError::ReqwestError)?; + let client = + build_rpc_client(CookieSupport::Enabled).map_err(TransportError::ReqwestError)?; Ok(Self { url, @@ -979,12 +993,8 @@ async fn test_node_connection( url: Url, auth_method: AuthMethod, ) -> Result<(), TestNodeConnectionError> { - let client = Client::builder() - .connect_timeout(std::time::Duration::from_secs(2)) - .timeout(std::time::Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(TestNodeConnectionError::ClientBuild)?; + let client = + build_rpc_client(CookieSupport::Disabled).map_err(TestNodeConnectionError::ClientBuild)?; let request_body = r#"{"jsonrpc":"2.0","method":"getinfo","params":[],"id":1}"#; let mut request_builder = client diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index c90307213..f29e2c051 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -361,7 +361,7 @@ pub struct GetBlockchainInfoResponse { /// Value pool balances #[serde(rename = "valuePools")] - value_pools: [ChainBalance; 5], + value_pools: Vec, /// Branch IDs of the current and upcoming consensus rules pub consensus: zebra_rpc::methods::TipConsensusBranch, @@ -539,6 +539,7 @@ mod get_tx_out_set_info_tests { { "id": "sprout", "chainValue": 0.0, "chainValueZat": 0 }, { "id": "sapling", "chainValue": 0.0, "chainValueZat": 0 }, { "id": "orchard", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "ironwood", "chainValue": 0.0, "chainValueZat": 0 }, { "id": "deferred", "chainValue": 0.0, "chainValueZat": 0 } ], "consensus": { @@ -548,13 +549,8 @@ mod get_tx_out_set_info_tests { }"#; let parsed: GetBlockchainInfoResponse = serde_json::from_str(json).unwrap(); - let (name, activation_height, status) = parsed - .upgrades - .values() - .next() - .unwrap() - .clone() - .into_parts(); + let (name, activation_height, status) = + parsed.upgrades.values().next().unwrap().into_parts(); assert_eq!(name, zebra_chain::parameters::NetworkUpgrade::Nu6_2); assert_eq!(activation_height, zebra_chain::block::Height(2)); @@ -563,6 +559,56 @@ mod get_tx_out_set_info_tests { } } +#[cfg(test)] +mod get_blockchain_info_response { + use super::GetBlockchainInfoResponse; + + /// Regression test: a validator that predates Ironwood (zcashd, or an unpatched + /// zebrad) reports five value pools — no "ironwood" entry. The response must still + /// deserialize. `value_pools` was a fixed `[ChainBalance; 6]`, so every + /// getblockchaininfo against such a validator failed outright with + /// "invalid length 5, expected an array of length 6". + /// + #[test] + fn parses_five_value_pools() { + let json = r#"{ + "chain": "main", + "blocks": 2, + "bestblockhash": "0000000000000000000000000000000000000000000000000000000000000002", + "estimatedheight": 2, + "chainSupply": { + "chainValue": 0.0, + "chainValueZat": 0 + }, + "upgrades": {}, + "valuePools": [ + { "id": "transparent", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "sprout", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "sapling", "chainValue": 0.055, "chainValueZat": 5500000 }, + { "id": "orchard", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "lockbox", "chainValue": 0.0, "chainValueZat": 0 } + ], + "consensus": { + "chaintip": "c2d6d0b4", + "nextblock": "c2d6d0b4" + } + }"#; + + let parsed = serde_json::from_str::(json) + .expect("five-pool valuePools must deserialize (pre-Ironwood validator)"); + + let pools = super::value_pools_array(parsed.value_pools); + // Zebra's canonical order: transparent, sprout, sapling, orchard, deferred, ironwood. + assert_eq!(pools[2].id(), "sapling"); + assert_eq!(pools[2].chain_value_zat().zatoshis(), 5_500_000); + // zcashd's "lockbox" entry lands in the deferred slot (zebra also names it "lockbox"). + assert_eq!(pools[4].id(), "lockbox"); + // The pool the validator did not report is back-filled with a zero balance. + assert_eq!(pools[5].id(), "ironwood"); + assert_eq!(pools[5].chain_value_zat().zatoshis(), 0); + } +} + fn default_header() -> Height { Height(0) } @@ -660,6 +706,9 @@ impl<'de> Deserialize<'de> for ChainBalance { "orchard" => Ok(ChainBalance(GetBlockchainInfoBalance::orchard( amount, None, /*TODO: handle optional delta*/ ))), + "ironwood" => Ok(ChainBalance(GetBlockchainInfoBalance::ironwood( + amount, None, /*TODO: handle optional delta*/ + ))), // TODO: Investigate source of undocument 'lockbox' value // that likely is intended to be 'deferred' "lockbox" | "deferred" => Ok(ChainBalance(GetBlockchainInfoBalance::deferred( @@ -680,6 +729,21 @@ impl Default for ChainBalance { } } +/// Fills zebra's canonical six-pool array from however many pools the validator +/// reported, leaving zero balances for pools the validator does not know about +/// (a pre-Ironwood validator reports five: no "ironwood" entry). Entries are +/// matched by pool id, so the validator's ordering is irrelevant; ids outside +/// zebra's six canonical pools are ignored. +fn value_pools_array(pools: Vec) -> [GetBlockchainInfoBalance; 6] { + let mut canonical = GetBlockchainInfoBalance::zero_pools(); + for ChainBalance(pool) in pools { + if let Some(slot) = canonical.iter_mut().find(|slot| slot.id() == pool.id()) { + *slot = pool; + } + } + canonical +} + impl TryFrom for zebra_rpc::methods::GetBlockchainInfoResponse { fn try_from(response: GetBlockchainInfoResponse) -> Result { Ok(zebra_rpc::methods::GetBlockchainInfoResponse::new( @@ -688,7 +752,7 @@ impl TryFrom for zebra_rpc::methods::GetBlockchainInf response.best_block_hash, response.estimated_height, response.chain_supply.0, - response.value_pools.map(|pool| pool.0), + value_pools_array(response.value_pools), response.upgrades, response.consensus, response.headers, @@ -904,6 +968,14 @@ pub struct OrchardTrees { size: u64, } +/// Ironwood note commitment tree information. +/// +/// Wrapper struct for zebra's IronwoodTrees +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +pub struct IronwoodTrees { + size: u64, +} + /// Information about the sapling and orchard note commitment trees if any. /// /// Wrapper struct for zebra's GetBlockTrees @@ -913,6 +985,8 @@ pub struct GetBlockTrees { sapling: Option, #[serde(default)] orchard: Option, + #[serde(default)] + ironwood: Option, } impl GetBlockTrees { @@ -925,11 +999,16 @@ impl GetBlockTrees { pub fn orchard(&self) -> u64 { self.orchard.map_or(0, |o| o.size) } + + /// Returns ironwood data held by ['GetBlockTrees']. + pub fn ironwood(&self) -> u64 { + self.ironwood.map_or(0, |o| o.size) + } } impl From for zebra_rpc::methods::GetBlockTrees { fn from(val: GetBlockTrees) -> Self { - zebra_rpc::methods::GetBlockTrees::new(val.sapling(), val.orchard()) + zebra_rpc::methods::GetBlockTrees::new(val.sapling(), val.orchard(), val.ironwood()) } } @@ -1134,8 +1213,10 @@ pub struct BlockObject { #[serde(default, skip_serializing_if = "Option::is_none")] pub difficulty: Option, - /// List of transaction IDs in block order, hex-encoded. - pub tx: Vec, + /// The block's transactions in block order: hex-encoded transaction IDs + /// at verbosity 1, full transaction objects at verbosity 2. Both shapes + /// share zebra-rpc's untagged enum. + pub tx: Vec, /// Chain supply balance #[serde(default)] @@ -1144,7 +1225,7 @@ pub struct BlockObject { /// Value pool balances /// #[serde(rename = "valuePools")] - value_pools: Option<[ChainBalance; 5]>, + value_pools: Option>, /// Information about the note commitment trees. pub trees: GetBlockTrees, @@ -1166,6 +1247,21 @@ pub struct BlockObject { pub next_block_hash: Option, } +impl BlockObject { + /// Per-pool chain value balances as of this block, when the validator + /// reports them (`getblock` verbosity 2). + pub fn value_pools(&self) -> Option<&[ChainBalance]> { + self.value_pools.as_deref() + } +} + +impl ChainBalance { + /// The underlying per-pool balance entry. + pub fn balance(&self) -> &GetBlockchainInfoBalance { + &self.0 + } +} + impl TryFrom for zebra_rpc::methods::GetBlock { type Error = zebra_chain::serialization::SerializationError; @@ -1175,15 +1271,7 @@ impl TryFrom for zebra_rpc::methods::GetBlock { Ok(zebra_rpc::methods::GetBlock::Raw(serialized_block.0)) } GetBlockResponse::Object(block) => { - let tx: Result, _> = block - .tx - .into_iter() - .map(|txid| { - txid.parse::() - .map(zebra_rpc::methods::GetBlockTransaction::Hash) - }) - .collect(); - let tx = tx?; + let tx = block.tx; Ok(zebra_rpc::methods::GetBlock::Object(Box::new( zebra_rpc::client::BlockObject::new( @@ -1204,11 +1292,7 @@ impl TryFrom for zebra_rpc::methods::GetBlock { block.bits, block.difficulty, block.chain_supply.map(|supply| supply.0), - block.value_pools.map( - |[transparent, sprout, sapling, orchard, deferred]| { - [transparent.0, sprout.0, sapling.0, orchard.0, deferred.0] - }, - ), + block.value_pools.map(value_pools_array), block.trees.into(), block.previous_block_hash.map(|hash| hash.0), block.next_block_hash.map(|hash| hash.0), @@ -1326,6 +1410,11 @@ pub struct GetTreestateResponse { /// (i.e. in regtest mode). #[serde(skip_serializing_if = "Option::is_none")] pub orchard: Option, + + /// A treestate containing an Ironwood note commitment tree, hex-encoded. Only present from + /// NU6.3, so that pre-NU6.3 responses are unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + pub ironwood: Option, } /// Error type for the `get_treestate` RPC request. @@ -1367,6 +1456,8 @@ impl TryFrom for zebra_rpc::client::GetTreestateResponse { .clone() .unwrap_or_else(|| Treestate::new(Commitments::new(None, None))); + let ironwood = value.ironwood.clone(); + Ok(zebra_rpc::client::GetTreestateResponse::new( parsed_hash, zebra_chain::block::Height(height_u32), @@ -1375,6 +1466,7 @@ impl TryFrom for zebra_rpc::client::GetTreestateResponse { None, sapling, orchard, + ironwood, )) } } @@ -1429,21 +1521,32 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { }, }; + // `let field: Kind = json;` reads the JSON key named exactly like the + // binding — the key cannot be restated, so sibling fields cannot be + // wired to each other's keys by copy-paste. Use the bracketed form + // `let field: Kind = json["jsonName"];` only when the JSON key differs + // from the binding (camelCase etc.). macro_rules! get_tx_value_fields{ - ($(let $field:ident: $kind:ty = $transaction_json:ident[$field_name:literal]; )+) => { - $(let $field = $transaction_json + () => {}; + (let $field:ident: $kind:ty = $transaction_json:ident; $($rest:tt)*) => { + let $field = $transaction_json + .get(stringify!($field)) + .map(|v| ::serde_json::from_value::<$kind>(v.clone())) + .transpose() + .map_err(::serde::de::Error::custom)?; + get_tx_value_fields! { $($rest)* } + }; + (let $field:ident: $kind:ty = $transaction_json:ident[$field_name:literal]; $($rest:tt)*) => { + let $field = $transaction_json .get($field_name) .map(|v| ::serde_json::from_value::<$kind>(v.clone())) .transpose() .map_err(::serde::de::Error::custom)?; - )+ - } + get_tx_value_fields! { $($rest)* } + }; } - let confirmations = tx_value - .get("confirmations") - .and_then(|v| v.as_u64()) - .map(|v| v as u32); + let confirmations = tx_value.get("confirmations").and_then(|v| v.as_i64()); // if let Some(vin_value) = tx_value.get("vin") { // match serde_json::from_value::>(vin_value.clone()) { @@ -1467,15 +1570,16 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { let outputs: Vec = tx_value["vout"]; let shielded_spends: Vec = tx_value["vShieldedSpend"]; let shielded_outputs: Vec = tx_value["vShieldedOutput"]; - let orchard: Orchard = tx_value["orchard"]; + let orchard: Orchard = tx_value; + let ironwood: Orchard = tx_value; let value_balance: f64 = tx_value["valueBalance"]; let value_balance_zat: i64 = tx_value["valueBalanceZat"]; - let size: i64 = tx_value["size"]; - let time: i64 = tx_value["time"]; - let txid: String = tx_value["txid"]; + let size: i64 = tx_value; + let time: i64 = tx_value; + let txid: String = tx_value; let auth_digest: String = tx_value["authdigest"]; - let overwintered: bool = tx_value["overwintered"]; - let version: u32 = tx_value["version"]; + let overwintered: bool = tx_value; + let version: u32 = tx_value; let version_group_id: String = tx_value["versiongroupid"]; let lock_time: u32 = tx_value["locktime"]; let expiry_height: Height = tx_value["expiryheight"]; @@ -1527,6 +1631,8 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { // optional orchard, // optional + ironwood, + // optional value_balance, // optional value_balance_zat, @@ -1583,6 +1689,7 @@ impl From for zebra_rpc::methods::GetRawTransaction { None, None, obj.orchard().clone(), + obj.ironwood().clone(), obj.value_balance(), obj.value_balance_zat(), obj.size(), @@ -1879,3 +1986,154 @@ pub struct GetMempoolInfoResponse { impl ResponseToError for GetMempoolInfoResponse { type RpcError = Infallible; } + +#[cfg(test)] +mod get_transaction_response { + use super::GetTransactionResponse; + + /// Regression test: the `ironwood` field must be read from the `"ironwood"` JSON key. + /// It was copy-pasted reading `"orchard"`, so a real ironwood bundle in a verbose + /// `getrawtransaction` response was silently dropped and the orchard bundle was + /// duplicated into the ironwood slot. + #[test] + fn ironwood_field_reads_ironwood_key() { + let tx_json = serde_json::json!({ + "hex": "00", + "txid": "0000000000000000000000000000000000000000000000000000000000000001", + "version": 6, + "locktime": 0, + "orchard": { + "actions": [], + "valueBalance": -1.11, + "valueBalanceZat": -111, + }, + "ironwood": { + "actions": [], + "valueBalance": -2.22, + "valueBalanceZat": -222, + }, + }); + + let response: GetTransactionResponse = + serde_json::from_value(tx_json).expect("test transaction JSON deserializes"); + let GetTransactionResponse::Object(tx_object) = response else { + panic!("verbose transaction JSON must deserialize to the Object variant"); + }; + + let orchard = tx_object + .orchard() + .as_ref() + .expect("orchard bundle present in JSON"); + assert_eq!(orchard.value_balance_zat(), -111); + + let ironwood = tx_object + .ironwood() + .as_ref() + .expect("ironwood bundle present in JSON"); + assert_eq!( + ironwood.value_balance_zat(), + -222, + "ironwood field must carry the \"ironwood\" JSON payload, not a copy of \"orchard\"" + ); + } +} + +#[cfg(test)] +mod get_block_response { + use super::GetBlockResponse; + + /// Builds the `tx` entry of a verbosity-2 `getblock` response exactly as + /// zebrad serializes it, by round-tripping a real test-vector + /// transaction through zebra-rpc's own response types. Constructing the + /// fixture from zebra's serializer keeps it faithful to the wire across + /// zebra upgrades, with no hand-maintained JSON to drift. + fn verbosity_2_transaction_json() -> serde_json::Value { + use zebra_chain::serialization::ZcashDeserializeInto as _; + + let vector = wire_serialized_transaction_test_data::transactions::get_test_vectors() + .first() + .cloned() + .expect("the test-vector crate provides at least one transaction"); + let transaction: zebra_chain::transaction::Transaction = vector + .tx + .as_slice() + .zcash_deserialize_into() + .expect("test-vector transactions deserialize"); + let txid = transaction.hash(); + let tx_object = zebra_rpc::client::TransactionObject::from_transaction( + std::sync::Arc::new(transaction), + Some(zebra_chain::block::Height(2)), + Some(1), + &zebra_chain::parameters::Network::Mainnet, + None, + None, + None, + txid, + ); + serde_json::to_value(zebra_rpc::methods::GetBlockTransaction::Object(Box::new( + tx_object, + ))) + .expect("zebra's response types serialize") + } + + fn block_json_with_tx(tx: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "hash": "0000000000000000000000000000000000000000000000000000000000000002", + "confirmations": 1, + "tx": [tx], + "trees": {}, + }) + } + + /// A verbosity-1 `getblock` response carries txid strings in `tx`. This + /// pins the shape the type already handles, so the verbosity-2 fix below + /// cannot regress it. + #[test] + fn parses_verbosity_1_txid_strings() { + let block_json = block_json_with_tx(serde_json::json!( + "e85e5729b34c34e8e62aa639302cb3434bb961666e13191da6dc39fcf775b320" + )); + + let parsed: GetBlockResponse = + serde_json::from_value(block_json).expect("verbosity-1 block objects deserialize"); + let GetBlockResponse::Object(block) = parsed else { + panic!("a JSON block object must parse as the object variant"); + }; + assert_eq!(block.tx.len(), 1); + } + + /// A verbosity-2 `getblock` response carries full transaction objects in + /// `tx`; the response type must deserialize that shape as well, and the + /// conversion into zebra-rpc's `GetBlock` must preserve the objects + /// (issue #1380). + #[test] + fn parses_verbosity_2_transaction_objects() { + let block_json = block_json_with_tx(verbosity_2_transaction_json()); + + let parsed: GetBlockResponse = + serde_json::from_value(block_json).expect("verbosity-2 block objects deserialize"); + let GetBlockResponse::Object(ref block) = parsed else { + panic!("a JSON block object must parse as the object variant"); + }; + assert!( + matches!( + block.tx.as_slice(), + [zebra_rpc::methods::GetBlockTransaction::Object(_)] + ), + "the verbosity-2 entry must parse as a transaction object" + ); + + let converted: zebra_rpc::methods::GetBlock = + parsed.try_into().expect("the object form converts"); + let zebra_rpc::methods::GetBlock::Object(converted) = converted else { + panic!("the conversion must keep the object variant"); + }; + assert!( + matches!( + converted.tx().as_slice(), + [zebra_rpc::methods::GetBlockTransaction::Object(_)] + ), + "the conversion must pass the transaction object through" + ); + } +} diff --git a/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs b/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs index ef422bb39..f509ec857 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs @@ -278,7 +278,7 @@ mod tests { fn sample_delta_with_block_index(i: u32, bi: Option) -> AddressDelta { AddressDelta { - satoshis: if i % 2 == 0 { 1_000 } else { -500 }, + satoshis: if i.is_multiple_of(2) { 1_000 } else { -500 }, txid: format!("deadbeef{:02x}", i), index: i, height: 123_456 + i, diff --git a/packages/zaino-fetch/src/lib.rs b/packages/zaino-fetch/src/lib.rs index 94778ad59..0299020a2 100644 --- a/packages/zaino-fetch/src/lib.rs +++ b/packages/zaino-fetch/src/lib.rs @@ -7,6 +7,7 @@ pub mod chain; pub mod jsonrpsee; +pub mod utils; /// Prometheus metric names emitted by this crate; the single source of truth shared with `zainod`'s `describe_*` registrations (which carry the descriptions). #[cfg(feature = "prometheus")] diff --git a/packages/zaino-fetch/src/utils.rs b/packages/zaino-fetch/src/utils.rs new file mode 100644 index 000000000..e66a1ae5d --- /dev/null +++ b/packages/zaino-fetch/src/utils.rs @@ -0,0 +1,19 @@ +//! Shared utility functionality for zaino-fetch. + +use crate::chain::error::ParseError; + +/// Used for decoding Zcash structures from a bytestring. +pub trait ParseFromSlice { + /// Reads data from a bytestring, consuming data read, and returns an instance of self along with the remaining data in the bytestring given. + /// + /// `txid` is accepted for compatibility with callers that already fetched transaction ids from a verbose block RPC response. + /// + /// `tx_version` is retained for compatibility with the legacy parser API and should be `None` for Zebra-backed parsing. + fn parse_from_slice( + data: &[u8], + txid: Option>>, + tx_version: Option, + ) -> Result<(&[u8], Self), ParseError> + where + Self: Sized; +} diff --git a/packages/zaino-proto/CHANGELOG.md b/packages/zaino-proto/CHANGELOG.md index 951b0949b..b221a8fc9 100644 --- a/packages/zaino-proto/CHANGELOG.md +++ b/packages/zaino-proto/CHANGELOG.md @@ -8,7 +8,13 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- Pool-type filter serves Ironwood by default (`include_ironwood: true`), so + clients that predate the field still receive `ironwoodActions` (unknown + protobuf fields are carried harmlessly). ### Changed +- Lightwallet protocol vendored subtree updated to v0.5.0: + `CompactTx.ironwoodActions` (field 9, `CompactOrchardAction`-shaped) and + `CompactBlock.ironwoodCommitmentTreeSize`. ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-proto/Cargo.toml b/packages/zaino-proto/Cargo.toml index 8b173ad0b..27b6df8c6 100644 --- a/packages/zaino-proto/Cargo.toml +++ b/packages/zaino-proto/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.1.4" +version = "0.2.0" [features] default = ["heavy"] diff --git a/packages/zaino-proto/README.md b/packages/zaino-proto/README.md index 17578a172..a114740a2 100644 --- a/packages/zaino-proto/README.md +++ b/packages/zaino-proto/README.md @@ -45,7 +45,7 @@ for example: when doing ``` -git subtree --prefix=zaino-proto/lightwallet-protocol pull git@github.com:zcash/lightwallet-protocol.git v0.4.0 --squash +git subtree --prefix=zaino-proto/lightwallet-protocol pull git@github.com:zcash/lightwallet-protocol.git v0.5.0 --squash ``` your branch's commits must be sequenced like this. @@ -62,4 +62,3 @@ If you are developing the `lightclient-protocol` and adopting it on Zaino, it is you don't do subsequent `git subtree` to revisions and always rebase against the latest latest version that you will be using in your latest commit to avoid rebasing issues and also keeping a coherent git commit history for when your branch merges to `dev`. - diff --git a/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md b/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md index 860d0d8e2..ffebb5da4 100644 --- a/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md +++ b/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md @@ -36,6 +36,14 @@ and the protocol adheres to an added `poolTypes` field, which allows the caller of this method to specify which pools the resulting `CompactTx` values should contain data for. +## [v0.5.0] - 2026-07-03 + +### Added +- `compact_formats.ChainMetadata` has added field `ironwoodCommitmentTreeSize` +- `compact_formats.CompactTx` has added field `ironwoodActions` +- `service.PoolType` has added `IRONWOOD` +- `service.TreeState` has added `ironwoodTree` + ### Deprecated - `service.CompactTxStreamer`: - The `GetBlockNullifiers` and `GetBlockRangeNullifiers` methods are diff --git a/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto b/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto index c62c7acbb..f96373023 100644 --- a/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto +++ b/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto @@ -14,6 +14,7 @@ option swift_prefix = ""; message ChainMetadata { uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block + uint32 ironwoodCommitmentTreeSize = 3; // the size of the Ironwood note commitment tree as of the end of this block } // A compact representation of a Zcash block. @@ -61,6 +62,7 @@ message CompactTx { repeated CompactSaplingSpend spends = 4; repeated CompactSaplingOutput outputs = 5; repeated CompactOrchardAction actions = 6; + repeated CompactOrchardAction ironwoodActions = 9; // `CompactTxIn` values corresponding to the `vin` entries of the full transaction. // diff --git a/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto b/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto index d3dc8ba04..3b3cebf43 100644 --- a/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto +++ b/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto @@ -14,6 +14,7 @@ enum PoolType { TRANSPARENT = 1; SAPLING = 2; ORCHARD = 3; + IRONWOOD = 4; } // A BlockID message contains identifiers to select a block: a height or a @@ -179,6 +180,7 @@ message TreeState { uint32 time = 4; // Unix epoch time when the block was mined string saplingTree = 5; // sapling commitment tree state string orchardTree = 6; // orchard commitment tree state + string ironwoodTree = 7; // ironwood commitment tree state } enum ShieldedProtocol { diff --git a/packages/zaino-proto/src/proto/compact_formats.rs b/packages/zaino-proto/src/proto/compact_formats.rs index 2de2751ed..cc0b2153e 100644 --- a/packages/zaino-proto/src/proto/compact_formats.rs +++ b/packages/zaino-proto/src/proto/compact_formats.rs @@ -8,6 +8,9 @@ pub struct ChainMetadata { /// the size of the Orchard note commitment tree as of the end of this block #[prost(uint32, tag = "2")] pub orchard_commitment_tree_size: u32, + /// the size of the Ironwood note commitment tree as of the end of this block + #[prost(uint32, tag = "3")] + pub ironwood_commitment_tree_size: u32, } /// A compact representation of a Zcash block. /// @@ -75,6 +78,8 @@ pub struct CompactTx { pub outputs: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "6")] pub actions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "9")] + pub ironwood_actions: ::prost::alloc::vec::Vec, /// `CompactTxIn` values corresponding to the `vin` entries of the full transaction. /// /// Note: the single null-outpoint input for coinbase transactions is omitted. Light diff --git a/packages/zaino-proto/src/proto/service.rs b/packages/zaino-proto/src/proto/service.rs index 559d2efbe..7e5a82ed9 100644 --- a/packages/zaino-proto/src/proto/service.rs +++ b/packages/zaino-proto/src/proto/service.rs @@ -257,6 +257,9 @@ pub struct TreeState { /// orchard commitment tree state #[prost(string, tag = "6")] pub orchard_tree: ::prost::alloc::string::String, + /// ironwood commitment tree state + #[prost(string, tag = "7")] + pub ironwood_tree: ::prost::alloc::string::String, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSubtreeRootsArg { @@ -322,6 +325,7 @@ pub enum PoolType { Transparent = 1, Sapling = 2, Orchard = 3, + Ironwood = 4, } impl PoolType { /// String value of the enum field names used in the ProtoBuf definition. @@ -334,6 +338,7 @@ impl PoolType { Self::Transparent => "TRANSPARENT", Self::Sapling => "SAPLING", Self::Orchard => "ORCHARD", + Self::Ironwood => "IRONWOOD", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -343,6 +348,7 @@ impl PoolType { "TRANSPARENT" => Some(Self::Transparent), "SAPLING" => Some(Self::Sapling), "ORCHARD" => Some(Self::Orchard), + "IRONWOOD" => Some(Self::Ironwood), _ => None, } } diff --git a/packages/zaino-proto/src/proto/utils.rs b/packages/zaino-proto/src/proto/utils.rs index c52a5d8df..72220ae45 100644 --- a/packages/zaino-proto/src/proto/utils.rs +++ b/packages/zaino-proto/src/proto/utils.rs @@ -19,9 +19,21 @@ pub enum PoolTypeError { /// Converts a vector of pool_types (i32) into its rich-type representation /// Returns `PoolTypeError::InvalidPoolType` when invalid `pool_types` are found /// or `PoolTypeError::UnknownPoolType` if unknown ones are found. +/// +/// An empty vector means the client did not filter, so every shielded pool is +/// served — including Ironwood, which clients that predate the field simply +/// ignore as an unknown protobuf field. Backfilling only the pre-NU6.3 pools +/// here would serve blocks whose `chainMetadata.ironwoodCommitmentTreeSize` +/// counts commitments from actions the block omits; a scanning wallet sees +/// that as a tree-size discontinuity and treats it as a chain reorg. +/// +/// The unfiltered pool set has exactly one definition: +/// [`PoolTypeFilter::default`]. This wire-decode path delegates to it so the +/// two cannot drift again (they had: the filter default gained Ironwood while +/// this backfill still listed only Sapling and Orchard). pub fn pool_types_from_vector(pool_types: &[i32]) -> Result, PoolTypeError> { let pools = if pool_types.is_empty() { - vec![PoolType::Sapling, PoolType::Orchard] + PoolTypeFilter::default().to_pool_types_vector() } else { let mut pools: Vec = vec![]; @@ -156,15 +168,17 @@ pub struct PoolTypeFilter { include_transparent: bool, include_sapling: bool, include_orchard: bool, + include_ironwood: bool, } impl std::default::Default for PoolTypeFilter { - /// By default PoolType includes `Sapling` and `Orchard` pools. + /// The unfiltered pool set: every shielded pool, transparent excluded. fn default() -> Self { PoolTypeFilter { include_transparent: false, include_sapling: true, include_orchard: true, + include_ironwood: true, } } } @@ -176,6 +190,7 @@ impl PoolTypeFilter { include_transparent: true, include_sapling: true, include_orchard: true, + include_ironwood: true, } } @@ -196,7 +211,7 @@ impl PoolTypeFilter { pub fn new_from_pool_types( pool_types: &Vec, ) -> Result { - if pool_types.len() > PoolType::Orchard as usize { + if pool_types.len() > PoolType::Ironwood as usize { return Err(PoolTypeError::InvalidPoolType); } @@ -211,6 +226,7 @@ impl PoolTypeFilter { PoolType::Transparent => filter.include_transparent = true, PoolType::Sapling => filter.include_sapling = true, PoolType::Orchard => filter.include_orchard = true, + PoolType::Ironwood => filter.include_ironwood = true, } } @@ -229,12 +245,16 @@ impl PoolTypeFilter { include_transparent: false, include_sapling: false, include_orchard: false, + include_ironwood: false, } } /// only internal use fn is_empty(&self) -> bool { - !self.include_transparent && !self.include_sapling && !self.include_orchard + !self.include_transparent + && !self.include_sapling + && !self.include_orchard + && !self.include_ironwood } /// retuns whether the filter includes transparent data @@ -252,6 +272,11 @@ impl PoolTypeFilter { self.include_orchard } + /// returns whether the filter includes ironwood data + pub fn includes_ironwood(&self) -> bool { + self.include_ironwood + } + /// Convert this filter into the corresponding `Vec`. /// /// The resulting vector contains each included pool type at most once. @@ -270,6 +295,10 @@ impl PoolTypeFilter { pool_types.push(PoolType::Orchard); } + if self.include_ironwood { + pool_types.push(PoolType::Ironwood); + } + pool_types } @@ -279,11 +308,13 @@ impl PoolTypeFilter { include_transparent: bool, include_sapling: bool, include_orchard: bool, + include_ironwood: bool, ) -> Self { PoolTypeFilter { include_transparent, include_sapling, include_orchard, + include_ironwood, } } } @@ -322,6 +353,7 @@ pub fn compact_block_with_pool_types( !compact_tx.spends.is_empty() || !compact_tx.outputs.is_empty() || !compact_tx.actions.is_empty() + || !compact_tx.ironwood_actions.is_empty() }); } else { for compact_tx in &mut block.vtx { @@ -339,6 +371,10 @@ pub fn compact_block_with_pool_types( if !pool_types.contains(&PoolType::Orchard) { compact_tx.actions.clear(); } + // strip out ironwood if not requested + if !pool_types.contains(&PoolType::Ironwood) { + compact_tx.ironwood_actions.clear(); + } } // Omit transactions that have no elements in any requested pool type. @@ -348,6 +384,7 @@ pub fn compact_block_with_pool_types( || !compact_tx.spends.is_empty() || !compact_tx.outputs.is_empty() || !compact_tx.actions.is_empty() + || !compact_tx.ironwood_actions.is_empty() }); } @@ -368,11 +405,18 @@ pub fn compact_block_to_nullifiers(mut block: CompactBlock) -> CompactBlock { ..Default::default() } } + for caction in &mut ctransaction.ironwood_actions { + *caction = CompactOrchardAction { + nullifier: caction.nullifier.clone(), + ..Default::default() + } + } } block.chain_metadata = Some(ChainMetadata { sapling_commitment_tree_size: 0, orchard_commitment_tree_size: 0, + ironwood_commitment_tree_size: 0, }); block } @@ -406,6 +450,7 @@ mod test { PoolType::Transparent, PoolType::Sapling, PoolType::Orchard, + PoolType::Ironwood, PoolType::Orchard, ] .to_vec(); @@ -418,11 +463,17 @@ mod test { #[test] fn test_pool_type_filter_t_z_o() { - let pools = [PoolType::Transparent, PoolType::Sapling, PoolType::Orchard].to_vec(); + let pools = [ + PoolType::Transparent, + PoolType::Sapling, + PoolType::Orchard, + PoolType::Ironwood, + ] + .to_vec(); assert_eq!( PoolTypeFilter::new_from_pool_types(&pools), - Ok(PoolTypeFilter::from_checked_parts(true, true, true)) + Ok(PoolTypeFilter::from_checked_parts(true, true, true, true)) ); } @@ -432,7 +483,9 @@ mod test { assert_eq!( PoolTypeFilter::new_from_pool_types(&pools), - Ok(PoolTypeFilter::from_checked_parts(true, false, false)) + Ok(PoolTypeFilter::from_checked_parts( + true, false, false, false + )) ); } @@ -447,8 +500,27 @@ mod test { #[test] fn test_pool_type_filter_includes_all() { assert_eq!( - PoolTypeFilter::from_checked_parts(true, true, true), + PoolTypeFilter::from_checked_parts(true, true, true, true), PoolTypeFilter::includes_all() ); } + + /// Regression: an unfiltered request (empty `poolTypes`, what every + /// pre-Ironwood client sends) must be served Ironwood actions. When the + /// empty-vector backfill listed only the pre-NU6.3 shielded pools, the + /// served compact blocks stripped `ironwoodActions` while + /// `chainMetadata.ironwoodCommitmentTreeSize` still counted them, and + /// scanning wallets reported a tree-size discontinuity (a phantom chain + /// reorg) at the first block with an Ironwood coinbase. + #[test] + fn empty_pool_types_request_includes_ironwood() { + let pools = crate::proto::utils::pool_types_from_vector(&[]).unwrap(); + assert!(pools.contains(&PoolType::Ironwood), "{pools:?}"); + + let filter = PoolTypeFilter::new_from_slice(&[]).unwrap(); + assert!(filter.includes_ironwood()); + assert!(filter.includes_sapling()); + assert!(filter.includes_orchard()); + assert!(!filter.includes_transparent()); + } } diff --git a/packages/zaino-serve/CHANGELOG.md b/packages/zaino-serve/CHANGELOG.md index 0df7dc8c7..8a6296e20 100644 --- a/packages/zaino-serve/CHANGELOG.md +++ b/packages/zaino-serve/CHANGELOG.md @@ -9,6 +9,8 @@ and this library adheres to Rust's notion of ### Added ### Changed +- tonic's TLS provider feature switches from `tls-ring` to `tls-aws-lc`, + following the workspace's aws-lc-rs preferred CryptoProvider (ADR-0006). ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-serve/Cargo.toml b/packages/zaino-serve/Cargo.toml index 630e54024..d33624cd1 100644 --- a/packages/zaino-serve/Cargo.toml +++ b/packages/zaino-serve/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.4.0" +version = "0.5.0" [features] default = [] @@ -53,9 +53,10 @@ metrics = { workspace = true, optional = true } # Miscellaneous Workspace tokio = { workspace = true, features = ["full"] } -tonic = { workspace = true, features = ["tls-native-roots"] } +tonic = { workspace = true, features = ["tls-native-roots", "tls-aws-lc"] } thiserror = { workspace = true } futures = { workspace = true } jsonrpsee = { workspace = true, features = ["server", "macros"] } serde = { workspace = true, features = ["derive"] } tower = { workspace = true } +zaino-common.workspace = true diff --git a/packages/zaino-serve/src/server/grpc.rs b/packages/zaino-serve/src/server/grpc.rs index 7d77b9ad5..e0854aea1 100644 --- a/packages/zaino-serve/src/server/grpc.rs +++ b/packages/zaino-serve/src/server/grpc.rs @@ -74,6 +74,9 @@ impl TonicServer { let mut server_builder = Server::builder(); if let Some(tls_config) = server_config.get_valid_tls().await? { + // Building the TLS acceptor requires a process-level rustls + // CryptoProvider (zingolabs/zaino#1360). + zaino_common::crypto::ensure_default_crypto_provider(); server_builder = server_builder.tls_config(tls_config).map_err(|e| { ServerError::ServerConfigError(format!("TLS configuration error: {e}")) })?; diff --git a/packages/zaino-serve/src/server/grpc/tests.rs b/packages/zaino-serve/src/server/grpc/tests.rs index 05eb1db71..946cbaa9a 100644 --- a/packages/zaino-serve/src/server/grpc/tests.rs +++ b/packages/zaino-serve/src/server/grpc/tests.rs @@ -1,3 +1,4 @@ //! Zaino-Serve gRPC server unit tests. mod spawn; +mod tls; diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem new file mode 100644 index 000000000..8c6998f49 --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBzzCCAXWgAwIBAgIUKx27V49MTw1PXcKunTwIjHzXWuswCgYIKoZIzj0EAwIw +PDE6MDgGA1UEAwwxemFpbm8tc2VydmUgVExTIHJlZ3Jlc3Npb24tdGVzdCBDQSAo +Tk9UIEEgU0VDUkVUKTAgFw0yNjA3MDQwMTQ0MTNaGA8yMTI2MDYxMDAxNDQxM1ow +PDE6MDgGA1UEAwwxemFpbm8tc2VydmUgVExTIHJlZ3Jlc3Npb24tdGVzdCBDQSAo +Tk9UIEEgU0VDUkVUKTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGT74i0fh1Vj +gYb0rPW36IYlyvvO/SdT0Mf8dJiFnKmoyqcXtORcw8antcjqe0myghHayx6xEOte +6blNT72knn+jUzBRMB0GA1UdDgQWBBSJ4axoCJxNd1mPygmrBgTI8IosCDAfBgNV +HSMEGDAWgBSJ4axoCJxNd1mPygmrBgTI8IosCDAPBgNVHRMBAf8EBTADAQH/MAoG +CCqGSM49BAMCA0gAMEUCIGhNgBQ5G/pars7fOccRedyOegikzDlMbdM8g4NIDErg +AiEAjlmMXJTBDW5Y/E1rLZgcC/dEdLodnTet3vatf6Vq1lM= +-----END CERTIFICATE----- diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem new file mode 100644 index 000000000..f78071a86 --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIB1TCCAXqgAwIBAgIBATAKBggqhkjOPQQDAjA8MTowOAYDVQQDDDF6YWluby1z +ZXJ2ZSBUTFMgcmVncmVzc2lvbi10ZXN0IENBIChOT1QgQSBTRUNSRVQpMCAXDTI2 +MDcwNDAxNDQxM1oYDzIxMjYwNjEwMDE0NDEzWjAUMRIwEAYDVQQDDAlsb2NhbGhv +c3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASvy0yCNcEp5I28O0EOd7bJOZ2f +tTYE3E1AHSaMFmiiN/Foxw5KZzF7tRDFZxl0CANx+o+igLjQaerNk/bhoZepo4GS +MIGPMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAMBgNVHRMBAf8EAjAAMA4G +A1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQUtHTz +QTMs8vfbh3m4IVJIah8jAt8wHwYDVR0jBBgwFoAUieGsaAicTXdZj8oJqwYEyPCK +LAgwCgYIKoZIzj0EAwIDSQAwRgIhAIIkUaCFwsFMyAsZOasP/fVQSiU8C5jU8gmt +1xmx/2XvAiEApJiQJ1Kxdd/vjpJ5H1JYktPwoqUpZDmDKJMF1O7zMDM= +-----END CERTIFICATE----- diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem new file mode 100644 index 000000000..290fd34eb --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg6+O2+tTz29iMo5yr +SS8d+lb3nG8G7x3s4bHfvC52G3KhRANCAASvy0yCNcEp5I28O0EOd7bJOZ2ftTYE +3E1AHSaMFmiiN/Foxw5KZzF7tRDFZxl0CANx+o+igLjQaerNk/bhoZep +-----END PRIVATE KEY----- diff --git a/packages/zaino-serve/src/server/grpc/tests/tls.rs b/packages/zaino-serve/src/server/grpc/tests/tls.rs new file mode 100644 index 000000000..9b60a7567 --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/tls.rs @@ -0,0 +1,83 @@ +//! Regression test for zingolabs/zaino#1360. +//! +//! Spawning the gRPC server with TLS enabled must succeed and serve a +//! working TLS handshake. In 0.5.0, dependency feature-unification +//! enabled both of rustls's `ring` and `aws-lc-rs` provider features, +//! which defeats rustls's automatic provider selection: building the +//! TLS acceptor panicked at runtime ("Could not automatically determine +//! the process-level CryptoProvider") — but only on TLS-enabled +//! deployments, a path no test exercised because every existing test +//! ran plaintext. The fix pins the feature graph to a single provider — +//! `aws-lc-rs`, the preferred provider per ADR-0006 — and installs it +//! explicitly (`ensure_default_crypto_provider`); this test covers +//! the full TLS startup + handshake path so a future provider-selection +//! or certificate-wiring regression fails in CI instead of production. +//! +//! The certificate fixtures were minted once with openssl (EC P-256, +//! ~100-year expiry): a self-signed test CA plus a `localhost` leaf +//! (SAN `DNS:localhost` + `IP:127.0.0.1`) signed by it — rustls-webpki +//! rejects a single self-signed CA cert presented as the server's +//! end-entity cert (`CaUsedAsEndEntity`). The committed leaf key is not +//! a secret; the CA key was discarded, so regeneration remints all +//! three files. The server reads the leaf pair from disk (the +//! production `GrpcTls` path); the client trusts the CA as its root. + +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::PathBuf; + +use tonic::service::Routes; +use tonic::transport::{server::TcpIncoming, Certificate, ClientTlsConfig, Endpoint}; + +use crate::server::config::{GrpcServerConfig, GrpcTls}; +use crate::server::grpc::TonicServer; + +fn fixture_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/server/grpc/tests/fixtures") + .join(name) +} + +#[tokio::test] +async fn tls_spawn_serves_a_handshake() { + // Pre-bind so the OS-assigned port is known before spawning; hand the + // open socket to `spawn_inner` directly (this module is a child of + // `grpc`, so the private constructor is reachable without the + // `test_dependencies`-gated `spawn_from_listener` wrapper). + let listener = tokio::net::TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))) + .await + .expect("bind on ephemeral port must succeed"); + let local_addr = listener + .local_addr() + .expect("local_addr on a bound listener is infallible"); + + let mut server = TonicServer::spawn_inner( + Routes::default(), + GrpcServerConfig { + listen_address: local_addr, + tls: Some(GrpcTls { + cert_path: fixture_path("tls_test_cert.pem"), + key_path: fixture_path("tls_test_key.pem"), + }), + }, + TcpIncoming::from(listener), + ) + .await + .expect("TLS-enabled spawn must succeed (zingolabs/zaino#1360)"); + + let channel = Endpoint::try_from(format!("https://127.0.0.1:{}", local_addr.port())) + .expect("loopback endpoint URI is valid") + .tls_config( + ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(include_str!( + "fixtures/tls_test_ca_cert.pem" + ))) + .domain_name("localhost"), + ) + .expect("client TLS config from the fixture cert must be valid") + .connect() + .await + .expect("TLS handshake against the spawned server must succeed"); + + drop(channel); + server.close().await; +} diff --git a/packages/zaino-state/CHANGELOG.md b/packages/zaino-state/CHANGELOG.md index 2ed018bd9..66e1602c6 100644 --- a/packages/zaino-state/CHANGELOG.md +++ b/packages/zaino-state/CHANGELOG.md @@ -8,6 +8,9 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- The chain index tracks Ironwood (NU6.3) note-commitment treestate roots, + storing `None` while the pool has no treestate rather than fabricating a + root. - `ChainIndex` / `NodeBackedChainIndexSubscriber` gain `get_outpoint_spenders` — for each transparent `Outpoint`, returns the txid that spent it on the best chain (index-aligned with the input, `None` if unspent or unknown). diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index b81e8ed09..058519476 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -6,11 +6,19 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.4.1" +version = "0.5.0" [features] default = [] +# TEST-ONLY: shrink the finalised/non-finalised seam to the tractable +# `FAST_TEST_MAX_NONFINALISED_DEPTH` so cross-crate live tests can exercise +# finalisation/eviction with small chains. MUST appear only in `[dev-dependencies]` +# of the live-test crates — never a normal/shipped dependency: resolver = "2" keeps +# dev-dependency features out of production dependency graphs, and `default-members` +# excludes the live-test crates from production builds. +fast-test-seam = [] + # Support for connecting to a zcashd validator node. Opt-in (default off, docs/adr/0005); being # deprecated. Forwards to zaino-fetch. See # docs/adr/0001-zcashd-support-feature-gate.md. @@ -75,22 +83,17 @@ lmdb = { workspace = true } lmdb-sys = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["preserve_order"] } -prost = { workspace = true } -primitive-types = { workspace = true } blake2 = { workspace = true } sha2 = { workspace = true } corez = { workspace = true } -bs58 = { workspace = true } -nonempty = { workspace = true } arc-swap = { workspace = true } reqwest.workspace = true bitflags = { workspace = true } -derive_more = { workspace = true, features = ["from"] } [dev-dependencies] tempfile = { workspace = true } tracing-subscriber = { workspace = true } -once_cell = { workspace = true } +primitive-types = { workspace = true } zebra-chain = { workspace = true, features = ["proptest-impl"] } proptest.workspace = true incrementalmerkletree = "0.8.2" diff --git a/packages/zaino-state/src/backends.rs b/packages/zaino-state/src/backends.rs deleted file mode 100644 index 019cfc5f1..000000000 --- a/packages/zaino-state/src/backends.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Zaino's chain fetch and tx submission backend services. - -pub mod fetch; - -pub mod state; - -fn latest_network_upgrade( - upgrades: &indexmap::IndexMap< - zebra_rpc::methods::ConsensusBranchIdHex, - zebra_rpc::methods::NetworkUpgradeInfo, - >, -) -> Result<&zebra_rpc::methods::NetworkUpgradeInfo, tonic::Status> { - upgrades.last().map(|(_, upgrade)| upgrade).ok_or_else(|| { - tonic::Status::failed_precondition("validator returned no network upgrade metadata") - }) -} - -/// Maximum number of addresses a single `get_address_utxos` / `get_address_utxos_stream` -/// request may carry. -/// -/// Both backends resolve the full backend UTXO set before applying `max_entries` / -/// `start_height` (issue #974). A complete pushdown fix needs upstream interface changes -/// the caller-supplied entry cap cannot reach today, so until then this bounds the one -/// input the service controls locally: the address fan-out. It stops an unauthenticated -/// caller forcing an unbounded number of backend address lookups in a single request, and -/// is set well above realistic wallet usage. -/// -/// TODO: make this deployment-configurable rather than a fixed constant. -const UTXO_MAX_ADDRESSES: usize = 1000; - -/// Reject a `get_address_utxos` request whose address list exceeds [`UTXO_MAX_ADDRESSES`]. -/// -/// `max_entries` bounds the response size, not the backend work; this guard bounds the -/// address fan-out, the part the service can cap without upstream changes. -fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { - if count > UTXO_MAX_ADDRESSES { - return Err(tonic::Status::invalid_argument(format!( - "Error: too many addresses in request: {count} exceeds the maximum of {UTXO_MAX_ADDRESSES}." - ))); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - #[test] - fn latest_network_upgrade_rejects_empty_metadata() { - let upgrades = indexmap::IndexMap::new(); - let err = super::latest_network_upgrade(&upgrades).expect_err("empty upgrades must fail"); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert_eq!( - err.message(), - "validator returned no network upgrade metadata" - ); - } - - #[test] - fn utxo_address_count_within_limit_is_accepted() { - assert!(super::validate_utxo_address_count(0).is_ok()); - assert!(super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES).is_ok()); - } - - #[test] - fn utxo_address_count_over_limit_is_rejected() { - let err = super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES + 1) - .expect_err("over-limit address count must fail"); - - assert_eq!(err.code(), tonic::Code::InvalidArgument); - } -} diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs deleted file mode 100644 index 62945c1df..000000000 --- a/packages/zaino-state/src/backends/state.rs +++ /dev/null @@ -1,2984 +0,0 @@ -//! Zcash chain fetch and tx submission service backed by Zebras [`ReadStateService`]. - -#[allow(deprecated)] -use crate::{ - chain_index::{ - mempool::{Mempool, MempoolSubscriber}, - source::ValidatorConnector, - types as chain_types, ChainIndex, - }, - config::{DonationAddress, StateServiceConfig}, - error::{BlockCacheError, StateServiceError}, - indexer::{ - handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, - }, - status::{NamedAtomicStatus, Status, StatusType}, - stream::{ - AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, - UtxoReplyStream, - }, - utils::{get_build_info, ServiceMetadata}, - BackendType, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, State, -}; -use crate::{ - chain_index::{types::BestChainLocation, NonFinalizedSnapshot}, - TransactionHash, -}; -use tokio_stream::StreamExt as _; -use zaino_fetch::{ - chain::{transaction::FullTransaction, utils::ParseFromSlice}, - jsonrpsee::{ - connector::{JsonRpSeeConnector, RpcError}, - raw_transaction::validate_raw_transaction_hex, - response::{ - address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, - block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, - block_header::GetBlockHeader, - block_subsidy::GetBlockSubsidy, - chain_tips::GetChainTipsResponse, - mining_info::GetMiningInfoWire, - peer_info::GetPeerInfo, - z_validate_address::{ - InvalidZValidateAddress, KnownZValidateAddress, ZValidateAddressResponse, - DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, - }, - GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, - GetSpentInfoResponse, GetSubtreesResponse, GetTxOutResponse, GetTxOutSetInfoResponse, - }, - }, -}; -use zaino_proto::proto::utils::{ - blockid_to_hashorheight, compact_block_to_nullifiers, GetBlockRangeError, PoolTypeError, - PoolTypeFilter, ValidatedBlockRangeRequest, -}; -use zaino_proto::proto::{ - compact_formats::CompactBlock, - service::{ - AddressList, Balance, BlockId, BlockRange, GetAddressUtxosArg, GetAddressUtxosReply, - GetAddressUtxosReplyList, GetMempoolTxRequest, LightdInfo, PingResponse, RawTransaction, - SendResponse, TransparentAddressBlockFilter, TreeState, TxFilter, - }, -}; -use zcash_keys::{address::Address, encoding::AddressCodec}; - -use zcash_protocol::consensus::NetworkType; -use zcash_transparent::address::TransparentAddress; -use zebra_chain::{ - amount::{Amount, NonNegative}, - block::{Header, Height, SerializedBlock}, - chain_tip::NetworkChainTipHeightEstimator, - parameters::{ConsensusBranchId, Network, NetworkKind, NetworkUpgrade}, - serialization::{BytesInDisplayOrder as _, ZcashDeserialize as _, ZcashSerialize}, - subtree::NoteCommitmentSubtreeIndex, -}; -use zebra_rpc::{ - client::{ - GetAddressBalanceRequest, GetBlockchainInfoBalance, GetSubtreesByIndexResponse, - GetTreestateResponse, HexData, Input, SubtreeRpcData, TransactionObject, - ValidateAddressResponse, - }, - methods::{ - chain_tip_difficulty, AddressBalance, ConsensusBranchIdHex, GetAddressTxIdsRequest, - GetAddressUtxos, GetBlock, GetBlockHash, GetBlockHeader as GetBlockHeaderZebra, - GetBlockHeaderObject, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, - GetInfo, GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, - TipConsensusBranch, - }, - server::error::LegacyCode, - sync::init_read_state_with_syncer, -}; -use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; - -use chrono::{DateTime, Utc}; -use futures::TryFutureExt as _; -use hex::{FromHex as _, ToHex}; -use indexmap::IndexMap; -use std::{collections::HashMap, error::Error, fmt, str::FromStr, sync::Arc}; -use tokio::{ - sync::mpsc, - time::{self, timeout}, -}; -use tower::{Service, ServiceExt}; -use tracing::{info, instrument, warn}; - -macro_rules! expected_read_response { - ($response:ident, $expected_variant:ident) => { - match $response { - ReadResponse::$expected_variant(inner) => inner, - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - } - }; -} - -/// Chain fetch service backed by Zebra's `ReadStateService` and `TrustedChainSync`. -/// -/// NOTE: We currently dop not implement clone for chain fetch services -/// as this service is responsible for maintaining and closing its child processes. -/// ServiceSubscribers are used to create separate chain fetch processes -/// while allowing central state processes to be managed in a single place. -/// If we want the ability to clone Service all JoinHandle's should be -/// converted to Arc\. -#[derive(Debug)] -// #[deprecated = "Will be eventually replaced by `BlockchainSource"] -pub struct StateService { - /// `ReadeStateService` from Zebra-State. - read_state_service: ReadStateService, - - /// Internal mempool. - mempool: Mempool, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, - - /// Listener for when the chain tip changes - chain_tip_change: zebra_state::ChainTipChange, - - /// Sync task handle. - sync_task_handle: Option>>, - - /// JsonRPC Client. - rpc_client: JsonRpSeeConnector, - - /// Core indexer. - indexer: NodeBackedChainIndex, - - /// Service metadata. - data: ServiceMetadata, - - /// Thread-safe status indicator. - status: NamedAtomicStatus, -} - -impl StateService { - #[cfg(feature = "test_dependencies")] - /// Helper for tests - pub fn read_state_service(&self) -> &ReadStateService { - &self.read_state_service - } -} - -impl Status for StateService { - fn status(&self) -> StatusType { - let current_status = self.status.load(); - if current_status == StatusType::Closing { - current_status - } else { - self.indexer.status() - } - } -} - -// #[allow(deprecated)] -impl ZcashService for StateService { - const BACKEND_TYPE: BackendType = BackendType::State; - - type Subscriber = StateServiceSubscriber; - type Config = StateServiceConfig; - - /// Initializes a new StateService instance and starts sync process. - #[instrument(name = "StateService::spawn", skip(config), fields(network = %config.common.network))] - async fn spawn(config: StateServiceConfig) -> Result { - info!( - rpc_address = %config.common.validator_rpc_address, - network = %config.common.network, - "Spawning State Service" - ); - - let rpc_client = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), - ) - .await?; - - let zebra_build_data = rpc_client.get_info().await?; - - let data = ServiceMetadata::new( - get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), - zebra_build_data.build, - zebra_build_data.subversion, - ); - info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - - info!( - grpc_address = %config.validator_grpc_address, - "Launching Chain Syncer" - ); - let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = - init_read_state_with_syncer( - config.validator_state_config.clone(), - &config.common.network.to_zebra_network(), - config.validator_grpc_address, - ) - .await??; - - info!("Chain syncer launched"); - - // Wait for ReadStateService to catch up to the validator's best chain tip. - // Height alone is insufficient during reorgs: the same height can refer to - // different blocks until JSON-RPC and ReadStateService agree on tip hash. - loop { - let blockchain_info = rpc_client.get_blockchain_info().await?; - let server_height = blockchain_info.blocks; - let server_tip_hash = blockchain_info.best_block_hash; - - let syncer_response = read_state_service - .ready() - .and_then(|service| service.call(ReadRequest::Tip)) - .await?; - let (syncer_height, syncer_tip_hash) = - expected_read_response!(syncer_response, Tip).ok_or( - RpcError::new_from_legacycode(LegacyCode::Misc, "no blocks in chain"), - )?; - - if server_height == syncer_height && server_tip_hash == syncer_tip_hash { - info!( - height = syncer_height.0, - tip_hash = %syncer_tip_hash, - "ReadStateService synced with Zebra" - ); - break; - } else { - info!( - syncer_height = syncer_height.0, - validator_height = server_height.0, - syncer_tip_hash = %syncer_tip_hash, - validator_tip_hash = %server_tip_hash, - "ReadStateService syncing with Zebra" - ); - tokio::time::sleep(std::time::Duration::from_millis(1000)).await; - continue; - } - } - - let mempool_source = ValidatorConnector::State(crate::chain_index::source::State { - read_state_service: read_state_service.clone(), - mempool_fetcher: rpc_client.clone(), - network: config.common.network, - }); - - let mempool = Mempool::spawn(mempool_source, None).await?; - - let chain_index = NodeBackedChainIndex::new( - ValidatorConnector::State(State { - read_state_service: read_state_service.clone(), - mempool_fetcher: rpc_client.clone(), - network: config.common.network, - }), - config.clone().into(), - ) - .await - .unwrap(); - - let state_service = Self { - chain_tip_change, - read_state_service, - sync_task_handle: Some(Arc::new(sync_task_handle)), - rpc_client: rpc_client.clone(), - mempool, - indexer: chain_index, - data, - config, - status: NamedAtomicStatus::new("StateService", StatusType::Spawning), - }; - - // wait for sync to complete, return error on sync fail. - loop { - match state_service.status() { - StatusType::Ready => { - state_service.status.store(StatusType::Ready); - break; - } - StatusType::CriticalError => { - return Err(StateServiceError::Critical( - "Chain index sync failed".to_string(), - )); - } - StatusType::Closing => break, - _ => { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - } - } - - Ok(state_service) - } - - fn get_subscriber(&self) -> IndexerSubscriber { - IndexerSubscriber::new(StateServiceSubscriber { - read_state_service: self.read_state_service.clone(), - rpc_client: self.rpc_client.clone(), - mempool: self.mempool.subscriber(), - indexer: self.indexer.subscriber(), - data: self.data.clone(), - config: self.config.clone(), - chain_tip_change: self.chain_tip_change.clone(), - }) - } - - /// Shuts down the StateService. - fn close(&mut self) { - if self.sync_task_handle.is_some() { - if let Some(handle) = self.sync_task_handle.take() { - handle.abort(); - } - } - } -} - -#[allow(deprecated)] -impl Drop for StateService { - fn drop(&mut self) { - self.close() - } -} - -/// A fetch service subscriber. -/// -/// Subscribers should be -#[derive(Debug, Clone)] -// #[deprecated] -pub struct StateServiceSubscriber { - /// Remote wrappper functionality for zebra's [`ReadStateService`]. - pub read_state_service: ReadStateService, - - /// Internal mempool. - pub mempool: MempoolSubscriber, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, - - /// Listener for when the chain tip changes - chain_tip_change: zebra_state::ChainTipChange, - - /// JsonRPC Client. - pub rpc_client: JsonRpSeeConnector, - - /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, - - /// Service metadata. - pub data: ServiceMetadata, -} - -impl Status for StateServiceSubscriber { - fn status(&self) -> StatusType { - self.indexer.status() - } -} - -/// A subscriber to any chaintip updates -#[derive(Clone)] -pub struct ChainTipSubscriber { - monitor: zebra_state::ChainTipChange, -} - -impl ChainTipSubscriber { - /// Waits until the tip hash has changed (relative to the last time this method - /// was called), then returns the best tip's block hash. - pub async fn next_tip_hash( - &mut self, - ) -> Result { - self.monitor - .wait_for_tip_change() - .await - .map(|tip| tip.best_tip_hash()) - } -} - -/// Private RPC methods, which are used as helper methods by the public ones -/// -/// These would be simple to add to the public interface if -/// needed, there are currently no plans to do so. -// #[allow(deprecated)] -impl StateServiceSubscriber { - /// Gets a Subscriber to any updates to the latest chain tip - pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { - ChainTipSubscriber { - monitor: self.chain_tip_change.clone(), - } - } - /// Returns the requested block header by hash or height, as a [`GetBlockHeader`] JSON string. - /// If the block is not in Zebra's state, - /// returns [error code `-8`.](https://github.com/zcash/zcash/issues/5758) - /// if a height was passed or -5 if a hash was passed. - /// - /// zcashd reference: [`getblockheader`](https://zcash.github.io/rpc/getblockheader.html) - /// method: post - /// tags: blockchain - /// - /// # Parameters - /// - /// - `hash_or_height`: (string, required, example="1") The hash or height - /// for the block to be returned. - /// - `verbose`: (bool, optional, default=false, example=true) false for hex encoded data, - /// true for a json object - /// - /// # Notes - /// - /// The undocumented `chainwork` field is not returned. - /// - /// This rpc is used by get_block(verbose), there is currently no - /// plan to offer this RPC publicly. - async fn get_block_header_inner( - state: &ReadStateService, - network: &Network, - hash_or_height: HashOrHeight, - verbose: Option, - ) -> Result { - let mut state = state.clone(); - let verbose = verbose.unwrap_or(true); - let network = network.clone(); - - let zebra_state::ReadResponse::BlockHeader { - header, - hash, - height, - next_block_hash, - } = state - .ready() - .and_then(|service| service.call(zebra_state::ReadRequest::BlockHeader(hash_or_height))) - .await - .map_err(|_| { - StateServiceError::RpcError(RpcError { - // Compatibility with zcashd. Note that since this function - // is reused by getblock(), we return the errors expected - // by it (they differ whether a hash or a height was passed) - code: LegacyCode::InvalidParameter as i64, - message: "block height not in best chain".to_string(), - data: None, - }) - })? - else { - return Err(StateServiceError::Custom( - "Unexpected response to BlockHeader request".to_string(), - )); - }; - - let response = if !verbose { - GetBlockHeaderZebra::Raw(HexData(header.zcash_serialize_to_vec()?)) - } else { - let zebra_state::ReadResponse::SaplingTree(sapling_tree) = state - .ready() - .and_then(|service| { - service.call(zebra_state::ReadRequest::SaplingTree(hash_or_height)) - }) - .await? - else { - return Err(StateServiceError::Custom( - "Unexpected response to SaplingTree request".to_string(), - )); - }; - // This could be `None` if there's a chain reorg between state queries. - let sapling_tree = sapling_tree.ok_or_else(|| { - StateServiceError::RpcError(zaino_fetch::jsonrpsee::connector::RpcError { - code: LegacyCode::InvalidParameter as i64, - message: "missing sapling tree for block".to_string(), - data: None, - }) - })?; - - let zebra_state::ReadResponse::Depth(depth) = state - .ready() - .and_then(|service| service.call(zebra_state::ReadRequest::Depth(hash))) - .await? - else { - return Err(StateServiceError::Custom( - "Unexpected response to Depth request".to_string(), - )); - }; - - // From - // TODO: Deduplicate const definition, consider - // refactoring this to avoid duplicate logic - const NOT_IN_BEST_CHAIN_CONFIRMATIONS: i64 = -1; - - // Confirmations are one more than the depth. - // Depth is limited by height, so it will never overflow an i64. - let confirmations = depth - .map(|depth| i64::from(depth) + 1) - .unwrap_or(NOT_IN_BEST_CHAIN_CONFIRMATIONS); - - let mut nonce = *header.nonce; - nonce.reverse(); - - let sapling_activation = NetworkUpgrade::Sapling.activation_height(&network); - let sapling_tree_size = sapling_tree.count(); - let final_sapling_root: [u8; 32] = - if sapling_activation.is_some() && height >= sapling_activation.unwrap() { - let mut root: [u8; 32] = sapling_tree.root().into(); - root.reverse(); - root - } else { - [0; 32] - }; - - let difficulty = header.difficulty_threshold.relative_to_network(&network); - let block_commitments = - header_to_block_commitments(&header, &network, height, final_sapling_root)?; - - let block_header = GetBlockHeaderObject::new( - hash, - confirmations, - height, - header.version, - header.merkle_root, - block_commitments, - final_sapling_root, - sapling_tree_size, - header.time.timestamp(), - nonce, - header.solution, - header.difficulty_threshold, - difficulty, - header.previous_block_hash, - next_block_hash, - ); - - GetBlockHeaderZebra::Object(Box::new(block_header)) - }; - - Ok(response) - } - - /// Return a list of consecutive compact blocks. - #[allow(dead_code, deprecated)] - async fn get_block_range_inner( - &self, - request: BlockRange, - nullifiers_only: bool, - ) -> Result { - let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(StateServiceError::from)?; - - let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) - .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(StateServiceError::from)?; - - // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] - let start = validated_request.start() as u32; - let end = validated_request.end() as u32; - - let state_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = state_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; - - tokio::spawn(async move { - let timeout_result = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - // This method does not support passthrough. Just return. - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else {return}; - let chain_height = non_finalized_snapshot.best_tip.height.0; - - match state_service_clone - .indexer - .get_compact_block_stream( - &snapshot, - chain_types::Height(start), - chain_types::Height(end), - pool_type_filter.clone(), - ) - .await - { - Ok(Some(mut compact_block_stream)) => { - if nullifiers_only { - while let Some(stream_item) = compact_block_stream.next().await { - match stream_item { - Ok(block) => { - if channel_tx - .send(Ok(compact_block_to_nullifiers(block))) - .await - .is_err() - { - break; - } - } - Err(status) => { - if channel_tx.send(Err(status)).await.is_err() { - break; - } - } - } - } - } else { - while let Some(stream_item) = compact_block_stream.next().await { - if channel_tx.send(stream_item).await.is_err() { - break; - } - } - } - } - Ok(None) => { - // Per `get_compact_block_stream` semantics: `None` means at least one bound is above the tip. - let offending_height = if start > chain_height { start } else { end }; - - match channel_tx - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{offending_height}]. Height requested is greater than the best chain tip [{chain_height}].", - )))) - .await - { - Ok(_) => {} - Err(e) => { - warn!(%e, "GetBlockRange channel closed unexpectedly"); - } - } - } - Err(e) => { - // Preserve previous behaviour: if the request is above tip, surface OutOfRange; - // otherwise return the error (currently exposed for dev). - if start > chain_height || end > chain_height { - let offending_height = if start > chain_height { start } else { end }; - - match channel_tx - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{offending_height}]. Height requested is greater than the best chain tip [{chain_height}].", - )))) - .await - { - Ok(_) => {} - Err(e) => { - warn!(%e, "GetBlockRange channel closed unexpectedly"); - } - } - } else { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .is_err() - { - warn!(%e, "GetBlockRangeStream closed unexpectedly"); - } - } - } - } - }, - ) - .await; - - if timeout_result.is_err() { - channel_tx - .send(Err(tonic::Status::deadline_exceeded( - "Error: get_block_range gRPC request timed out.", - ))) - .await - .ok(); - } - }); - - Ok(CompactBlockStream::new(channel_rx)) - } - - async fn error_get_block( - &self, - e: BlockCacheError, - height: u32, - ) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let chain_height = snapshot.max_serviceable_height().0; - Err(if height >= chain_height { - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{height}]. Height requested \ - is greater than Zaino's best chain tip [{chain_height}].", - ))) - } else { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - StateServiceError::TonicStatusError(tonic::Status::unknown(format!( - "Error: Failed to retrieve block from node. Server Error: {e}", - ))) - }) - } - - pub(crate) async fn get_block_inner( - state: &ReadStateService, - network: &Network, - hash_or_height: HashOrHeight, - verbosity: Option, - ) -> Result { - let mut state_1 = state.clone(); - - let verbosity = verbosity.unwrap_or(1); - match verbosity { - 0 => { - let request = ReadRequest::Block(hash_or_height); - let response = state_1 - .ready() - .and_then(|service| service.call(request)) - .await?; - let block = expected_read_response!(response, Block); - block.map(SerializedBlock::from).map(GetBlock::Raw).ok_or( - StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - "block not found", - )), - ) - } - 1 | 2 => { - let state_2 = state.clone(); - let state_3 = state.clone(); - let state_4 = state.clone(); - - let blockandsize_future = { - let req = ReadRequest::BlockAndSize(hash_or_height); - async move { state_1.ready().and_then(|service| service.call(req)).await } - }; - let orchard_future = { - let req = ReadRequest::OrchardTree(hash_or_height); - async move { - state_2 - .clone() - .ready() - .and_then(|service| service.call(req)) - .await - } - }; - - let block_info_future = { - let req = ReadRequest::BlockInfo(hash_or_height); - async move { - state_4 - .clone() - .ready() - .and_then(|service| service.call(req)) - .await - } - }; - let (fullblock, orchard_tree_response, header, block_info) = futures::join!( - blockandsize_future, - orchard_future, - StateServiceSubscriber::get_block_header_inner( - &state_3, - network, - hash_or_height, - Some(true) - ), - block_info_future - ); - - let header_obj = match header? { - GetBlockHeaderZebra::Raw(_hex_data) => unreachable!( - "`true` was passed to get_block_header, an object should be returned" - ), - GetBlockHeaderZebra::Object(get_block_header_object) => get_block_header_object, - }; - - let (transactions_response, size, block_info): (Vec, _, _) = - match (fullblock, block_info) { - ( - Ok(ReadResponse::BlockAndSize(Some((block, size)))), - Ok(ReadResponse::BlockInfo(Some(block_info))), - ) => Ok(( - block - .transactions - .iter() - .map(|transaction| { - match verbosity { - 1 => GetBlockTransaction::Hash(transaction.hash()), - 2 => GetBlockTransaction::Object(Box::new( - TransactionObject::from_transaction( - transaction.clone(), - Some(header_obj.height()), - Some(header_obj.confirmations() as u32), - network, - DateTime::::from_timestamp( - header_obj.time(), - 0, - ), - Some(header_obj.hash()), - // block header has a non-optional height, which indicates - // a mainchain block. It is implied this method cannot return sidechain - // data, at least for now. This is subject to change: TODO - // return Some(true/false) after this assumption is resolved - None, - transaction.hash(), - ), - )), - _ => unreachable!("verbosity known to be 1 or 2"), - } - }) - .collect(), - size, - block_info, - )), - (Ok(ReadResponse::Block(None)), Ok(ReadResponse::BlockInfo(None))) => { - Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - "block not found", - ))) - } - (Ok(unexpected), Ok(unexpected2)) => { - unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") - } - (Err(e), _) | (_, Err(e)) => Err(e.into()), - }?; - - let orchard_tree_response = orchard_tree_response?; - let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - "missing orchard tree", - )))?; - - let final_orchard_root = match NetworkUpgrade::Nu5.activation_height(network) { - Some(activation_height) if header_obj.height() >= activation_height => { - Some(orchard_tree.root().into()) - } - _otherwise => None, - }; - - let trees = - GetBlockTrees::new(header_obj.sapling_tree_size(), orchard_tree.count()); - - let (chain_supply, value_pools) = ( - GetBlockchainInfoBalance::chain_supply(*block_info.value_pools()), - GetBlockchainInfoBalance::value_pools(*block_info.value_pools(), None), - ); - let transaction_count = transactions_response.len(); - - Ok(GetBlock::Object(Box::new( - zebra_rpc::client::BlockObject::new( - header_obj.hash(), - header_obj.confirmations(), - Some(size as i64), - Some(header_obj.height()), - Some(header_obj.version()), - Some(header_obj.merkle_root()), - Some(header_obj.block_commitments()), - Some(header_obj.final_sapling_root()), - final_orchard_root, - transaction_count, - transactions_response, - Some(header_obj.time()), - Some(header_obj.nonce()), - Some(header_obj.solution()), - Some(header_obj.bits()), - Some(header_obj.difficulty()), - Some(chain_supply), - Some(value_pools), - trees, - Some(header_obj.previous_block_hash()), - header_obj.next_block_hash(), - ), - ))) - } - more_than_two => Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - format!("invalid verbosity of {more_than_two}"), - ))), - } - } - - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network - } - - /// Returns the median time of the last 11 blocks. - async fn median_time_past( - &self, - start: &zebra_rpc::client::BlockObject, - ) -> Result { - const MEDIAN_TIME_PAST_WINDOW: usize = 11; - - let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); - - let start_hash = start.hash().to_string(); - let time_0 = start - .time() - .ok_or_else(|| MedianTimePast::StartMissingTime { - hash: start_hash.clone(), - })?; - times.push(time_0); - - let mut prev = start.previous_block_hash(); - - for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { - let hash = match prev { - Some(h) => h.to_string(), - None => break, // genesis - }; - - match self.z_get_block(hash.clone(), Some(1)).await { - Ok(GetBlock::Object(obj)) => { - if let Some(t) = obj.time() { - times.push(t); - } - prev = obj.previous_block_hash(); - } - Ok(GetBlock::Raw(_)) => { - return Err(MedianTimePast::UnexpectedRaw { hash }); - } - Err(_e) => { - // Use values up to this point - break; - } - } - } - - if times.is_empty() { - return Err(MedianTimePast::EmptyWindow); - } - - times.sort_unstable(); - Ok(times[times.len() / 2]) - } -} - -/// Extracts the diversifier and pk_d bytes from a validated Sapling -/// [`PaymentAddress`], returning pk_d in zcashd's big-endian byte order. -/// -/// # Deprecation -/// -/// See [`DEPRECATION_NOTICE`]. This function exists to support the -/// `z_validateaddress` RPC endpoint, which itself exists solely for zcashd -/// compatibility. The pk_d bytes are -/// reversed from `sapling-crypto`'s native little-endian representation to -/// match zcashd's big-endian hex output. -/// -/// # Precondition -/// -/// The caller must have obtained `s` through [`PaymentAddress::from_bytes`] or -/// equivalent (e.g. `ZcashAddress::convert_if_network`), which guarantees the -/// diversifier has a valid `g_d()` and pk_d is a non-identity Jubjub subgroup -/// point. No additional validation is performed here. -/// -/// # Layout -/// -/// `PaymentAddress::to_bytes()` returns 43 bytes: `diversifier (11) || pk_d (32)`. -/// `DiversifiedTransmissionKey::to_bytes()` is `pub(crate)` in `sapling-crypto`, -/// so we extract pk_d from the serialized form. -fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11], [u8; 32]) { - let bytes = s.to_bytes(); - let diversifier: [u8; 11] = bytes[..11].try_into().unwrap(); - let mut pk_d: [u8; 32] = bytes[11..].try_into().unwrap(); - pk_d.reverse(); - (diversifier, pk_d) -} - -// #[allow(deprecated)] -impl ZcashIndexer for StateServiceSubscriber { - type Error = StateServiceError; - - async fn get_info(&self) -> Result { - // A number of these fields are difficult to access from the state service - // TODO: Fix this - self.rpc_client - .get_info() - .await - .map(GetInfo::from) - .map_err(|e| StateServiceError::Custom(e.to_string())) - } - - /// Returns all changes for an address. - /// - /// Returns information about all changes to the given transparent addresses within the given (inclusive) - /// - /// block height range, default is the full blockchain. - /// If start or end are not specified, they default to zero. - /// If start is greater than the latest block height, it's interpreted as that height. - /// - /// If end is zero, it's interpreted as the latest block height. - /// - /// [Original zcashd implementation](https://github.com/zcash/zcash/blob/18238d90cd0b810f5b07d5aaa1338126aa128c06/src/rpc/misc.cpp#L881) - /// - /// zcashd reference: [`getaddressdeltas`](https://zcash.github.io/rpc/getaddressdeltas.html) - /// method: post - /// tags: address - async fn get_address_deltas( - &self, - params: GetAddressDeltasParams, - ) -> Result { - Ok(self.indexer.get_address_deltas(params).await?) - } - - async fn get_difficulty(&self) -> Result { - chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .map_err(|e| { - StateServiceError::RpcError(RpcError::new_from_errorobject( - e, - "failed to get difficulty", - )) - }) - } - - async fn get_block_subsidy(&self, height: u32) -> Result { - self.rpc_client - .get_block_subsidy(height) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) - } - - async fn get_blockchain_info(&self) -> Result { - let mut state = self.read_state_service.clone(); - - let response = state - .ready() - .and_then(|service| service.call(ReadRequest::TipPoolValues)) - .await?; - let (height, hash, balance) = match response { - ReadResponse::TipPoolValues { - tip_height, - tip_hash, - value_balance, - } => (tip_height, tip_hash, value_balance), - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - }; - - let usage_response = state - .ready() - .and_then(|service| service.call(ReadRequest::UsageInfo)) - .await?; - let size_on_disk = expected_read_response!(usage_response, UsageInfo); - - let request = zebra_state::ReadRequest::BlockHeader(hash.into()); - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let header = match response { - ReadResponse::BlockHeader { header, .. } => header, - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - }; - - let now = Utc::now(); - let zebra_estimated_height = NetworkChainTipHeightEstimator::new( - header.time, - height, - &self.config.common.network.into(), - ) - .estimate_height_at(now); - let estimated_height = if header.time > now || zebra_estimated_height < height { - height - } else { - zebra_estimated_height - }; - - let upgrades = IndexMap::from_iter( - self.config - .common - .network - .to_zebra_network() - .full_activation_list() - .into_iter() - .filter_map(|(activation_height, network_upgrade)| { - // Zebra defines network upgrades based on incompatible consensus rule changes, - // but zcashd defines them based on ZIPs. - // - // All the network upgrades with a consensus branch ID - // are the same in Zebra and zcashd. - network_upgrade.branch_id().map(|branch_id| { - // zcashd's RPC seems to ignore Disabled network upgrades, - // so Zebra does too. - let status = if height >= activation_height { - NetworkUpgradeStatus::Active - } else { - NetworkUpgradeStatus::Pending - }; - - ( - ConsensusBranchIdHex::new(branch_id.into()), - NetworkUpgradeInfo::from_parts( - network_upgrade, - activation_height, - status, - ), - ) - }) - }), - ); - - let next_block_height = - (height + 1).expect("valid chain tips are a lot less than Height::MAX"); - let consensus = TipConsensusBranch::from_parts( - ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), height) - .branch_id() - .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) - .into(), - ) - .inner(), - ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), next_block_height) - .branch_id() - .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) - .into(), - ) - .inner(), - ); - - // TODO: Remove unwrap() - let difficulty = chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .unwrap(); - - let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); - - Ok(GetBlockchainInfoResponse::new( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), - height, - hash, - estimated_height, - zebra_rpc::client::GetBlockchainInfoBalance::chain_supply(balance), - // TODO: account for new delta_pools arg? - zebra_rpc::client::GetBlockchainInfoBalance::value_pools(balance, None), - upgrades, - consensus, - height, - difficulty, - verification_progress, - // TODO: store work in the finalized state for each height - // see https://github.com/ZcashFoundation/zebra/issues/7109 - 0, - false, - size_on_disk, - // TODO (copied from zebra): Investigate whether this needs to - // be implemented (it's sprout-only in zcashd) - 0, - )) - } - - /// Returns details on the active state of the TX memory pool. - /// In Zaino, this RPC call information is gathered from the local Zaino state instead of directly reflecting the full node's mempool. This state is populated from a gRPC stream, sourced from the full node. - /// There are no request parameters. - /// The Zcash source code is considered canonical: - /// [from the rpc definition](), [this function is called to produce the return value](>). - /// There are no required or optional parameters. - /// the `size` field is called by [this line of code](), and returns an int64. - /// `size` represents the number of transactions currently in the mempool. - /// the `bytes` field is called by [this line of code](), and returns an int64 from [this variable](). - /// `bytes` is the sum memory size in bytes of all transactions in the mempool: the sum of all transaction byte sizes. - /// the `usage` field is called by [this line of code](), and returns an int64 derived from the return of this function(), which includes a number of elements. - /// `usage` is the total memory usage for the mempool, in bytes. - /// the [optional `fullyNotified` field](), is only utilized for zcashd regtests, is deprecated, and is not included. - async fn get_mempool_info(&self) -> Result { - Ok(self.indexer.get_mempool_info().await.into()) - } - - async fn get_peer_info(&self) -> Result { - Ok(self.rpc_client.get_peer_info().await?) - } - - async fn z_get_address_balance( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result { - Ok(self.indexer.get_address_balance(address_strings).await?) - } - - async fn send_raw_transaction( - &self, - raw_transaction_hex: String, - ) -> Result { - validate_raw_transaction_hex(&raw_transaction_hex).map_err(StateServiceError::RpcError)?; - // Offload to the json rpc connector, as ReadStateService - // doesn't yet interface with the mempool - self.rpc_client - .send_raw_transaction(raw_transaction_hex) - .await - .map(SentTransactionHash::from) - .map_err(Into::into) - } - - async fn get_block_header( - &self, - hash: String, - verbose: bool, - ) -> Result { - self.rpc_client - .get_block_header(hash, verbose) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) - } - - async fn z_get_block( - &self, - hash_or_height_string: String, - verbosity: Option, - ) -> Result { - let hash_or_height = HashOrHeight::from_str(&hash_or_height_string); - - StateServiceSubscriber::get_block_inner( - &self.read_state_service.clone(), - &self.data.network(), - hash_or_height?, - verbosity, - ) - .await - } - - async fn get_block_deltas(&self, hash: String) -> Result { - // Get the block WITH the transaction data - let zblock = self.z_get_block(hash, Some(2)).await?; - - match zblock { - GetBlock::Object(boxed_block) => { - // Resolve each transparent spend's prevout ourselves: the - // verbosity-2 object from Zebra's stateless - // `TransactionObject::from_transaction` leaves input - // value/address `None`, so we look the spent output up via the - // best-chain `ReadRequest::Transaction` (finalized + - // non-finalized, so same-block prevouts resolve too). - let network = self.data.network(); - let mut state = self.read_state_service.clone(); - // Per-call cache: many inputs may reference the same prevtxid, - // so each previous transaction is fetched at most once. - let mut prevtx_cache: HashMap< - zebra_chain::transaction::Hash, - Arc, - > = HashMap::new(); - - let mut deltas = Vec::with_capacity(boxed_block.tx().len()); - for (tx_index, tx) in boxed_block.tx().iter().enumerate() { - let txo = match tx { - GetBlockTransaction::Object(txo) => txo, - GetBlockTransaction::Hash(_) => { - return Err(StateServiceError::Custom( - "Unexpected hash when expecting object".to_string(), - )) - } - }; - - let txid = txo.txid().to_string(); - - let mut inputs: Vec = Vec::new(); - for (i, vin) in txo.inputs().iter().enumerate() { - let (prevtxid, prevout) = match vin { - Input::Coinbase { .. } => continue, - Input::NonCoinbase { - txid: prevtxid, - vout: prevout, - .. - } => (prevtxid, *prevout), - }; - - let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|e| { - StateServiceError::Custom(format!( - "getblockdeltas: invalid prevout txid {prevtxid}: {e}" - )) - })?; - - let prev_tx = match prevtx_cache.get(&prev_hash) { - Some(prev_tx) => prev_tx.clone(), - None => { - let response = state - .ready() - .and_then(|service| { - service.call(ReadRequest::Transaction(prev_hash)) - }) - .await?; - let mined_tx = expected_read_response!(response, Transaction) - .ok_or_else(|| { - StateServiceError::Custom(format!( - "getblockdeltas: prevout tx {prevtxid} not in best chain" - )) - })?; - prevtx_cache.insert(prev_hash, mined_tx.tx.clone()); - mined_tx.tx - } - }; - - let output = prev_tx.outputs().get(prevout as usize).ok_or_else(|| { - StateServiceError::Custom(format!( - "getblockdeltas: prevout index {prevout} out of range for {prevtxid}" - )) - })?; - - // Nonstandard script ⇒ no derivable address ⇒ skip - // (matches the outputs branch). This is the only - // legitimate skip; it is not loss of a resolvable spend. - let address = match output.address(&network) { - Some(a) => a.to_string(), - None => continue, - }; - - // Inputs are debits, so the amount leaves the address. - let satoshis: Amount = - (-output.value().zatoshis()).try_into().map_err(|e| { - StateServiceError::Custom(format!( - "getblockdeltas: input amount out of range for {prevtxid}:{prevout}: {e}" - )) - })?; - - inputs.push(InputDelta { - address, - satoshis, - index: i as u32, - prevtxid: prevtxid.clone(), - prevout, - }); - } - - let outputs: Vec = txo - .outputs() - .iter() - .filter_map(|vout| { - let addr_opt = vout - .script_pub_key() - .addresses() - .as_ref() - .and_then(|v| if v.len() == 1 { v.first() } else { None }); - - let addr = addr_opt?.clone(); - - let output_amt: Amount = match vout.value_zat().try_into() - { - Ok(a) => a, - Err(_) => return None, - }; - - Some(OutputDelta { - address: addr, - satoshis: output_amt, - index: vout.n(), - }) - }) - .collect::>(); - - deltas.push(BlockDelta { - txid, - index: tx_index as u32, - inputs, - outputs, - }); - } - - Ok(BlockDeltas { - hash: boxed_block.hash().to_string(), - confirmations: boxed_block.confirmations(), - size: boxed_block.size().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block size missing".into()) - })?, - height: boxed_block - .height() - .ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block height missing".into()) - })? - .0, - version: boxed_block.version().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block version missing".into()) - })?, - merkle_root: boxed_block - .merkle_root() - .ok_or_else(|| { - StateServiceError::Custom( - "getblockdeltas: block merkle root missing".into(), - ) - })? - .encode_hex::(), - deltas, - time: boxed_block.time().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block time missing".into()) - })?, - median_time: self.median_time_past(&boxed_block).await.map_err(|e| { - StateServiceError::Custom(format!("getblockdeltas: median_time_past: {e}")) - })?, - nonce: hex::encode(boxed_block.nonce().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block nonce missing".into()) - })?), - bits: boxed_block - .bits() - .ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block bits missing".into()) - })? - .to_string(), - difficulty: boxed_block.difficulty().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block difficulty missing".into()) - })?, - previous_block_hash: boxed_block - .previous_block_hash() - .map(|hash| hash.to_string()), - next_block_hash: boxed_block.next_block_hash().map(|h| h.to_string()), - }) - } - GetBlock::Raw(_serialized_block) => Err(StateServiceError::Custom( - "Unexpected raw block".to_string(), - )), - } - } - - async fn get_raw_mempool(&self) -> Result, Self::Error> { - Ok(self - .indexer - .get_mempool_txids() - .await? - .into_iter() - .map(|txid| txid.to_rpc_hex()) - .collect()) - } - - /// NOTE: This method currently has to fetch data from 2 places (get_treestate and get_indexed_block_by_*), - /// If `ValidatorConnector::GetTreeState` was updated to return the additional information - /// required, this second call could be removed, improving the performance of this method. - // Pre-existing lint: `StateServiceError` is a large error type; returning it by value here is - // flagged by `result_large_err`. Suppressed to satisfy `-D warnings` without an invasive - // boxing refactor of the shared error enum. - #[allow(clippy::result_large_err)] - async fn z_get_treestate( - &self, - hash_or_height: String, - ) -> Result { - let fallback_hash_or_height = hash_or_height.clone(); - let local_result: Result = async { - let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let block_data = match hash_or_height_struct { - HashOrHeight::Hash(hash) => self - .indexer - .get_indexed_block_by_hash(&snapshot, &hash.into()) - .await? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - HashOrHeight::Height(height) => self - .indexer - .get_indexed_block_by_height(&snapshot, &height.into()) - .await? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - }; - - let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; - let time: u32 = block_data.data().time().try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Block time is out of range for u32.", - )) - })?; - - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( - (*block_data.hash()).into(), - block_data.height().into(), - time, - sapling, - orchard, - )) - } - .await; - - if let Ok(response) = local_result { - return Ok(response); - } - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - if !self - .indexer - .hash_or_height_known_for_treestate(&snapshot, &fallback_hash_or_height) - .await? - { - return local_result; - } - - self.rpc_client - .get_treestate(fallback_hash_or_height) - .await - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch treestate.", - )) - }) - .and_then(|treestate| { - treestate.try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to parse treestate.", - )) - }) - }) - } - - async fn get_mining_info(&self) -> Result { - Ok(self.rpc_client.get_mining_info().await?) - } - - /// Returns statistics about the unspent transaction output set. - /// - /// zcashd reference: [`gettxoutsetinfo`](https://zcash.github.io/rpc/gettxoutsetinfo.html) - /// method: post - /// tags: blockchain - async fn get_tx_out_set_info(&self) -> Result { - Ok(self.indexer.get_tx_out_set_info().await?) - } - - // No request parameters. - /// Return the hex encoded hash of the best (tip) block, in the longest block chain. - /// The Zcash source code is considered canonical: - /// [In the rpc definition](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/common.h#L48) there are no required params, or optional params. - /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) - /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` - async fn get_best_blockhash(&self) -> Result { - // return should be valid hex encoded. - // Hash from zebra says: - // Return the hash bytes in big-endian byte-order suitable for printing out byte by byte. - // - // Zebra displays transaction and block hashes in big-endian byte-order, - // following the u256 convention set by Bitcoin and zcashd. - match self.read_state_service.best_tip() { - Some(x) => return Ok(GetBlockHash::new(x.1)), - None => { - // try RPC if state read fails: - Ok(self.rpc_client.get_best_blockhash().await?.into()) - } - } - } - - /// Returns the current block count in the best valid block chain. - /// - /// zcashd reference: [`getblockcount`](https://zcash.github.io/rpc/getblockcount.html) - /// method: post - /// tags: blockchain - async fn get_block_count(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let h = non_finalized_snapshot.best_tip.height; - Ok(h.into()) - } - - async fn get_chain_tips(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Ok(self.rpc_client.get_chain_tips().await?); - }; - - Ok(crate::chain_index::chain_tips_from_nonfinalized_snapshot( - non_finalized_snapshot, - )) - } - - async fn validate_address( - &self, - raw_address: String, - ) -> Result { - use zcash_transparent::address::TransparentAddress; - - let Ok(address) = raw_address.parse::() else { - return Ok(ValidateAddressResponse::invalid()); - }; - - let address = match address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { - NetworkKind::Mainnet => NetworkType::Main, - NetworkKind::Testnet => NetworkType::Test, - NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ValidateAddressResponse::invalid()); - } - }; - - Ok(match address { - Address::Transparent(taddr) => ValidateAddressResponse::new( - true, - Some(raw_address), - Some(matches!(taddr, TransparentAddress::ScriptHash(_))), - ), - _ => ValidateAddressResponse::invalid(), - }) - } - - #[allow(deprecated)] - async fn z_validate_address( - &self, - address: String, - ) -> Result { - tracing::warn!("{}", Z_VALIDATE_DEPRECATION); - - let Ok(parsed_address) = address.parse::() else { - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - }; - - let converted_address = match parsed_address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { - NetworkKind::Mainnet => NetworkType::Main, - NetworkKind::Testnet => NetworkType::Test, - NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - } - }; - - // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. - match converted_address { - Address::Transparent(t) => match t { - TransparentAddress::PublicKeyHash(_) => { - Ok(ZValidateAddressResponse::p2pkh(address)) - } - TransparentAddress::ScriptHash(_) => Ok(ZValidateAddressResponse::p2sh(address)), - }, - Address::Unified(u) => Ok(ZValidateAddressResponse::unified( - u.encode(&self.network().to_zebra_network()), - )), - Address::Sapling(s) => { - let (diversifier, pk_d) = sapling_key_bytes(&s); - Ok(ZValidateAddressResponse::sapling( - s.encode(&self.network().to_zebra_network()), - Some(hex::encode(diversifier)), - Some(hex::encode(pk_d)), - )) - } - _ => Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )), - } - } - - async fn z_get_subtrees_by_index( - &self, - pool: String, - start_index: NoteCommitmentSubtreeIndex, - limit: Option, - ) -> Result { - let mut state = self.read_state_service.clone(); - - match pool.as_str() { - "sapling" => { - let request = zebra_state::ReadRequest::SaplingSubtrees { start_index, limit }; - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let sapling_subtrees = expected_read_response!(response, SaplingSubtrees); - let subtrees = sapling_subtrees - .values() - .map(|subtree| { - SubtreeRpcData { - root: subtree.root.to_bytes().encode_hex(), - end_height: subtree.end_height, - } - .into() - }) - .collect(); - - Ok(GetSubtreesResponse { - pool, - start_index, - subtrees, - } - .into()) - } - "orchard" => { - let request = zebra_state::ReadRequest::OrchardSubtrees { start_index, limit }; - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let orchard_subtrees = expected_read_response!(response, OrchardSubtrees); - let subtrees = orchard_subtrees - .values() - .map(|subtree| { - SubtreeRpcData { - root: subtree.root.encode_hex(), - end_height: subtree.end_height, - } - .into() - }) - .collect(); - - Ok(GetSubtreesResponse { - pool, - start_index, - subtrees, - } - .into()) - } - otherwise => Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - format!("invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\""), - ))), - } - } - - async fn get_raw_transaction( - &self, - txid_hex: String, - verbose: Option, - ) -> Result { - #[allow(deprecated)] - let txid = TransactionHash::from_hex(&txid_hex).map_err(|error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, - error.to_string(), - )) - })?; - - #[allow(deprecated)] - let not_found_error = || { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, - "No such mempool or main chain transaction", - )) - }; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let Some((serialized_transaction, _consensus_branch_id)) = - self.indexer.get_raw_transaction(&snapshot, &txid).await? - else { - return Err(not_found_error()); - }; - - if verbose.is_none() { - return Ok(GetRawTransaction::Raw( - zebra_chain::transaction::SerializedTransaction::from(serialized_transaction), - )); - } - - let transaction = zebra_chain::transaction::Transaction::zcash_deserialize( - serialized_transaction.as_slice(), - ) - .map_err(|_| not_found_error())?; - - let (best_chain_location, _non_best_chain_locations) = self - .indexer - .get_transaction_status(&snapshot, &txid) - .await?; - - let (height, confirmations, block_hash, in_best_chain) = match best_chain_location { - Some(BestChainLocation::Block(block_hash, height)) => { - let confirmations = snapshot - .max_serviceable_height() - .0 - .saturating_sub(height.0) - .saturating_add(1); - - ( - Some(zebra_chain::block::Height::from(height)), - Some(confirmations), - Some(zebra_chain::block::Hash::from(block_hash)), - Some(true), - ) - } - Some(BestChainLocation::Mempool(_height)) => (None, Some(0), None, Some(false)), - None => (None, None, None, Some(false)), - }; - - Ok(GetRawTransaction::Object(Box::new( - TransactionObject::from_transaction( - transaction.into(), - height, - confirmations, - #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), - None, - block_hash, - in_best_chain, - zebra_chain::transaction::Hash::from(txid), - ), - ))) - } - - /// Returns details about an unspent transaction output. - /// - /// zcashd reference: [`gettxout`](https://zcash.github.io/rpc/gettxout.html) - /// method: post - /// tags: transaction - async fn get_tx_out( - &self, - txid: String, - n: u32, - include_mempool: Option, - ) -> Result { - Ok(self.rpc_client.get_tx_out(txid, n, include_mempool).await?) - } - - async fn get_spent_info( - &self, - request: GetSpentInfoRequest, - ) -> Result { - Ok(self.rpc_client.get_spent_info(request).await?) - } - - async fn get_address_tx_ids( - &self, - request: GetAddressTxIdsRequest, - ) -> Result, Self::Error> { - Ok(self - .indexer - .get_address_txids(request) - .await? - .into_iter() - .map(|transaction_hash| transaction_hash.to_rpc_hex()) - .collect()) - } - - async fn z_get_address_utxos( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result, Self::Error> { - Ok(self.indexer.get_address_utxos(address_strings).await?) - } - - /// Returns the estimated network solutions per second based on the last n blocks. - /// - /// zcashd reference: [`getnetworksolps`](https://zcash.github.io/rpc/getnetworksolps.html) - /// method: post - /// tags: blockchain - /// - /// This RPC is implemented in the [mining.cpp](https://github.com/zcash/zcash/blob/d00fc6f4365048339c83f463874e4d6c240b63af/src/rpc/mining.cpp#L104) - /// file of the Zcash repository. The Zebra implementation can be found [here](https://github.com/ZcashFoundation/zebra/blob/19bca3f1159f9cb9344c9944f7e1cb8d6a82a07f/zebra-rpc/src/methods.rs#L2687). - /// - /// # Parameters - /// - /// - `blocks`: (number, optional, default=120) Number of blocks, or -1 for blocks over difficulty averaging window. - /// - `height`: (number, optional, default=-1) To estimate network speed at the time of a specific block height. - async fn get_network_sol_ps( - &self, - blocks: Option, - height: Option, - ) -> Result { - self.rpc_client - .get_network_sol_ps(blocks, height) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) - } - - // Helper function, to get the chain height in rpc implementations - async fn chain_height(&self) -> Result { - let mut state = self.read_state_service.clone(); - let response = state - .ready() - .and_then(|service| service.call(ReadRequest::Tip)) - .await?; - let (chain_height, _chain_hash) = expected_read_response!(response, Tip).ok_or( - RpcError::new_from_legacycode(LegacyCode::Misc, "no blocks in chain"), - )?; - Ok(chain_height) - } -} - -// #[allow(deprecated)] -impl LightWalletIndexer for StateServiceSubscriber { - /// Return the height of the tip of the best chain - async fn get_latest_block(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - Ok(non_finalized_snapshot.best_tip.to_wire()) - } - - /// Return the compact block corresponding to the given block identifier - async fn get_block(&self, request: BlockId) -> Result { - let hash_or_height = blockid_to_hashorheight(request).ok_or( - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Invalid hash and/or height out of range. Failed to convert to u32.", - )), - )?; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - // Convert HashOrHeight to chain_types::Height - let block_height = match hash_or_height { - HashOrHeight::Height(h) => chain_types::Height(h.0), - HashOrHeight::Hash(h) => self - .indexer - .get_block_height(&snapshot, chain_types::BlockHash(h.0)) - .await - .map_err(StateServiceError::ChainIndexError)? - .ok_or_else(|| { - StateServiceError::TonicStatusError(tonic::Status::not_found( - "Error: Block not found for given hash.", - )) - })?, - }; - - match self - .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) - .await - { - Ok(Some(block)) => Ok(block), - Ok(None) => { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let chain_height = non_finalized_snapshot.best_tip.height.0; - match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ - is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), - } - } - Err(e) => { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let chain_height = non_finalized_snapshot.best_tip.height.0; - match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ - is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - { - Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) - } - } - } - } - } - - /// Same as GetBlock except actions contain only nullifiers, - /// and saling outputs are not returned (Sapling spends still are) - async fn get_block_nullifiers(&self, request: BlockId) -> Result { - let height: u32 = request.height.try_into().map_err(|_| { - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Height out of range. Failed to convert to u32.", - )) - })?; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let block_height = chain_types::Height(height); - - match self - .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) - .await - { - Ok(Some(block)) => Ok(compact_block_to_nullifiers(block)), - Ok(None) => { - self.error_get_block( - BlockCacheError::Custom("Block not found".to_string()), - height, - ) - .await - } - Err(e) => Err(StateServiceError::ChainIndexError(e)), - } - } - - /// Return a list of consecutive compact blocks - async fn get_block_range( - &self, - blockrange: BlockRange, - ) -> Result { - self.get_block_range_inner(blockrange, false).await - } - /// Same as GetBlockRange except actions contain only nullifiers - async fn get_block_range_nullifiers( - &self, - request: BlockRange, - ) -> Result { - self.get_block_range_inner(request, true).await - } - - /// Return the requested full (not compact) transaction (as from zcashd) - async fn get_transaction(&self, request: TxFilter) -> Result { - let hash = zebra_chain::transaction::Hash::from( - <[u8; 32]>::try_from(request.hash).map_err(|_| { - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Transaction hash incorrect", - )) - })?, - ); - let hex = hash.encode_hex(); - - // explicit over method call syntax to make it clear where this method is coming from - #[allow(clippy::result_large_err)] - ::get_raw_transaction(self, hex, Some(1)) - .await - .and_then(|grt| match grt { - GetRawTransaction::Raw(_serialized_transaction) => Err(StateServiceError::Custom( - "unreachable, verbose transaction expected".to_string(), - )), - GetRawTransaction::Object(transaction_object) => Ok(RawTransaction { - data: transaction_object.hex().as_ref().to_vec(), - height: transaction_object.height().unwrap_or(0) as u64, - }), - }) - } - - /// Submit the given transaction to the Zcash network - async fn send_transaction(&self, request: RawTransaction) -> Result { - let hex_tx = hex::encode(request.data); - let tx_output = self.send_raw_transaction(hex_tx).await?; - - Ok(SendResponse { - error_code: 0, - error_message: tx_output.hash().to_string(), - }) - } - - /// Return the transactions corresponding to the given t-address within the given block range - async fn get_taddress_transactions( - &self, - request: TransparentAddressBlockFilter, - ) -> Result { - let chain_height = self.chain_height().await?; - let txids = self.get_taddress_txids_helper(request).await?; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (transmitter, receiver) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - for txid in txids { - let transaction = - fetch_service_clone.get_raw_transaction(txid, Some(1)).await; - if handle_raw_transaction::( - chain_height.0 as u64, - transaction, - transmitter.clone(), - ) - .await - .is_err() - { - break; - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - transmitter - .send(Err(tonic::Status::internal( - "Error: get_taddress_txids gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(RawTransactionStream::new(receiver)) - } - - /// Return the txids corresponding to the given t-address within the given block range - /// This function is deprecated. Use `get_taddress_transactions`. - async fn get_taddress_txids( - &self, - request: TransparentAddressBlockFilter, - ) -> Result { - self.get_taddress_transactions(request).await - } - - /// Returns the total balance for a list of taddrs - async fn get_taddress_balance( - &self, - request: AddressList, - ) -> Result { - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let balance = self.z_get_address_balance(taddrs).await?; - let checked_balance: i64 = match i64::try_from(balance.balance()) { - Ok(balance) => balance, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); - } - }; - Ok(Balance { - value_zat: checked_balance, - }) - } - - /// Returns the total balance for a list of taddrs - /// - /// TODO: This is taken from fetch.rs, we could / probably should reconfigure into a trait implementation. - async fn get_taddress_balance_stream( - &self, - mut request: AddressStream, - ) -> Result { - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, mut channel_rx) = - mpsc::channel::(self.config.common.service.channel_size as usize); - let fetcher_task_handle = tokio::spawn(async move { - let fetcher_timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let mut total_balance: u64 = 0; - loop { - match channel_rx.recv().await { - Some(taddr) => { - let taddrs = GetAddressBalanceRequest::new(vec![taddr]); - let balance = - fetch_service_clone.z_get_address_balance(taddrs).await?; - total_balance += balance.balance(); - } - None => { - return Ok(total_balance); - } - } - } - }, - ) - .await; - match fetcher_timeout { - Ok(result) => result, - Err(_) => Err(tonic::Status::deadline_exceeded( - "Error: get_taddress_balance_stream request timed out.", - )), - } - }); - // NOTE: This timeout is so slow due to the blockcache not - // being implemented. This should be reduced to 30s once functionality is in place. - // TODO: Make [rpc_timout] a configurable system variable - // with [default = 30s] and [mempool_rpc_timout = 4*rpc_timeout] - let addr_recv_timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - while let Some(address_result) = request.next().await { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - let address = address_result.map_err(|e| { - tonic::Status::unknown(format!("Failed to read from stream: {e}")) - })?; - if channel_tx.send(address.address).await.is_err() { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - return Err(tonic::Status::unknown( - "Error: Failed to send address to balance task.", - )); - } - } - drop(channel_tx); - Ok::<(), tonic::Status>(()) - }, - ) - .await; - match addr_recv_timeout { - Ok(Ok(())) => {} - Ok(Err(e)) => { - fetcher_task_handle.abort(); - return Err(StateServiceError::TonicStatusError(e)); - } - Err(_) => { - fetcher_task_handle.abort(); - return Err(StateServiceError::TonicStatusError( - tonic::Status::deadline_exceeded( - "Error: get_taddress_balance_stream request timed out in address loop.", - ), - )); - } - } - match fetcher_task_handle.await { - Ok(Ok(total_balance)) => { - let checked_balance: i64 = match i64::try_from(total_balance) { - Ok(balance) => balance, - Err(_) => { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); - } - }; - Ok(Balance { - value_zat: checked_balance, - }) - } - Ok(Err(e)) => Err(StateServiceError::TonicStatusError(e)), - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - Err(e) => Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - format!("Fetcher Task failed: {e}"), - ))), - } - } - - /// Return the compact transactions currently in the mempool; the results - /// can be a few seconds out of date. If the Exclude list is empty, return - /// all transactions; otherwise return all *except* those in the Exclude list - /// (if any); this allows the client to avoid receiving transactions that it - /// already has (from an earlier call to this rpc). The transaction IDs in the - /// Exclude list can be shortened to any number of bytes to make the request - /// more bandwidth-efficient; if two or more transactions in the mempool - /// match a shortened txid, they are all sent (none is excluded). Transactions - /// in the exclude list that don't exist in the mempool are ignored. - async fn get_mempool_tx( - &self, - request: GetMempoolTxRequest, - ) -> Result { - let mut exclude_txids: Vec = vec![]; - - for (i, excluded_id) in request.exclude_txid_suffixes.iter().enumerate() { - if excluded_id.len() > 32 { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument(format!( - "Error: excluded txid {} is larger than 32 bytes", - i - )), - )); - } - - // NOTE: the TransactionHash methods cannot be used for this hex encoding as exclusions could be truncated to less than 32 bytes - let reversed_txid_bytes: Vec = excluded_id.iter().cloned().rev().collect(); - let hex_string_txid: String = hex::encode(&reversed_txid_bytes); - exclude_txids.push(hex_string_txid); - } - - let pool_types = match PoolTypeFilter::new_from_slice(&request.pool_types) { - Ok(pool_type_filter) => pool_type_filter, - Err(PoolTypeError::InvalidPoolType) => { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument( - "Error: An invalid `PoolType' was found".to_string(), - ), - )) - } - Err(PoolTypeError::UnknownPoolType(unknown_pool_type)) => { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument(format!( - "Error: Unknown `PoolType' {} was found", - unknown_pool_type - )), - )) - } - }; - - let mempool = self.mempool.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - for (mempool_key, mempool_value) in - mempool.get_filtered_mempool(exclude_txids).await - { - let txid_bytes = match hex::decode(mempool_key.txid) { - Ok(bytes) => bytes, - Err(error) => { - if channel_tx - .send(Err(tonic::Status::unknown(error.to_string()))) - .await - .is_err() - { - break; - } else { - continue; - } - } - }; - match ::parse_from_slice( - mempool_value.serialized_tx.as_ref().as_ref(), - Some(vec![txid_bytes]), - None, - ) { - Ok(transaction) => { - // ParseFromSlice returns any data left after the conversion to a - // FullTransaction, If the conversion has succeeded this should be empty. - if transaction.0.is_empty() { - if channel_tx - .send( - transaction - .1 - .to_compact_tx(None, &pool_types) - .map_err(|e| tonic::Status::unknown(e.to_string())), - ) - .await - .is_err() - { - break; - } - } else { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown("Error: "))) - .await - .is_err() - { - break; - } - } - } - Err(e) => { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .is_err() - { - break; - } - } - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::internal( - "Error: get_mempool_tx gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - - Ok(CompactTransactionStream::new(channel_rx)) - } - - /// Return a stream of current Mempool transactions. This will keep the output stream open while - /// there are mempool transactions. It will close the returned stream when a new block is mined. - async fn get_mempool_stream(&self) -> Result { - let mut mempool = self.mempool.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let mempool_height = non_finalized_snapshot.best_tip.height.0; - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 6) as u64), - async { - let (mut mempool_stream, _mempool_handle) = match mempool - .get_mempool_stream(None) - .await - { - Ok(stream) => stream, - Err(e) => { - warn!(?e, "error fetching mempool stream"); - channel_tx - .send(Err(tonic::Status::internal("Error getting mempool stream"))) - .await - .ok(); - return; - } - }; - while let Some(result) = mempool_stream.recv().await { - match result { - Ok((_mempool_key, mempool_value)) => { - if channel_tx - .send(Ok(RawTransaction { - data: mempool_value - .serialized_tx - .as_ref() - .as_ref() - .to_vec(), - height: mempool_height as u64, - })) - .await - .is_err() - { - break; - } - } - Err(e) => { - channel_tx - .send(Err(tonic::Status::internal(format!( - "Error in mempool stream: {e:?}" - )))) - .await - .ok(); - break; - } - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::internal( - "Error: get_mempool_stream gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - - Ok(RawTransactionStream::new(channel_rx)) - } - - /// GetTreeState returns the note commitment tree state corresponding to the given block. - /// See section 3.7 of the Zcash protocol specification. It returns several other useful - /// values also (even though they can be obtained using GetBlock). - /// The block can be specified by either height or hash. - async fn get_tree_state(&self, request: BlockId) -> Result { - let hash_or_height = blockid_to_hashorheight(request).ok_or( - crate::error::StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Invalid hash or height", - )), - )?; - #[allow(deprecated)] - let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); - Ok(TreeState { - network: self - .config - .common - .network - .to_zebra_network() - .bip70_network_name(), - height: height.0 as u64, - hash: hash.to_string(), - time, - sapling_tree: sapling.map(hex::encode).unwrap_or_default(), - orchard_tree: orchard.map(hex::encode).unwrap_or_default(), - }) - } - - /// GetLatestTreeState returns the note commitment tree state corresponding to the chain tip. - async fn get_latest_tree_state(&self) -> Result { - let latest_block = self.chain_height().await?; - self.get_tree_state(BlockId { - height: latest_block.0 as u64, - hash: vec![], - }) - .await - } - - fn timeout_channel_size(&self) -> (u32, u32) { - ( - self.config.common.service.timeout, - self.config.common.service.channel_size, - ) - } - - /// Returns all unspent outputs for a list of addresses. - /// - /// Ignores all utxos below block height [GetAddressUtxosArg.start_height]. - /// Returns max [GetAddressUtxosArg.max_entries] utxos, or unrestricted if - /// [GetAddressUtxosArg.max_entries] = 0. - /// Utxos are collected and returned as a single Vec. - async fn get_address_utxos( - &self, - request: GetAddressUtxosArg, - ) -> Result { - super::validate_utxo_address_count(request.addresses.len())?; - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let utxos = self.z_get_address_utxos(taddrs).await?; - let mut address_utxos: Vec = Vec::new(); - let mut entries: u32 = 0; - for utxo in utxos { - let (address, tx_hash, output_index, script, satoshis, height) = utxo.into_parts(); - if (height.0 as u64) < request.start_height { - continue; - } - entries += 1; - if request.max_entries > 0 && entries > request.max_entries { - break; - } - let checked_index = match i32::try_from(output_index.index()) { - Ok(index) => index, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))); - } - }; - let checked_satoshis = match i64::try_from(satoshis) { - Ok(satoshis) => satoshis, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))); - } - }; - let utxo_reply = GetAddressUtxosReply { - address: address.to_string(), - txid: tx_hash.0.to_vec(), - index: checked_index, - script: script.as_raw_bytes().to_vec(), - value_zat: checked_satoshis, - height: height.0 as u64, - }; - address_utxos.push(utxo_reply) - } - Ok(GetAddressUtxosReplyList { address_utxos }) - } - - /// Returns all unspent outputs for a list of addresses. - /// - /// Ignores all utxos below block height [GetAddressUtxosArg.start_height]. - /// Returns max [GetAddressUtxosArg.max_entries] utxos, or unrestricted if - /// [GetAddressUtxosArg.max_entries] = 0. - /// Utxos are returned in a stream. - async fn get_address_utxos_stream( - &self, - request: GetAddressUtxosArg, - ) -> Result { - super::validate_utxo_address_count(request.addresses.len())?; - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let utxos = self.z_get_address_utxos(taddrs).await?; - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let mut entries: u32 = 0; - for utxo in utxos { - let (address, tx_hash, output_index, script, satoshis, height) = - utxo.into_parts(); - if (height.0 as u64) < request.start_height { - continue; - } - entries += 1; - if request.max_entries > 0 && entries > request.max_entries { - break; - } - let checked_index = match i32::try_from(output_index.index()) { - Ok(index) => index, - Err(_) => { - let _ = channel_tx - .send(Err(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))) - .await; - return; - } - }; - let checked_satoshis = match i64::try_from(satoshis) { - Ok(satoshis) => satoshis, - Err(_) => { - let _ = channel_tx - .send(Err(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))) - .await; - return; - } - }; - let utxo_reply = GetAddressUtxosReply { - address: address.to_string(), - txid: tx_hash.0.to_vec(), - index: checked_index, - script: script.as_raw_bytes().to_vec(), - value_zat: checked_satoshis, - height: height.0 as u64, - }; - if channel_tx.send(Ok(utxo_reply)).await.is_err() { - return; - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::deadline_exceeded( - "Error: get_mempool_stream gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(UtxoReplyStream::new(channel_rx)) - } - - /// Return information about this lightwalletd instance and the blockchain - /// - /// TODO: This could be made more efficient by fetching data directly (not using self.get_blockchain_info()) - async fn get_lightd_info(&self) -> Result { - let blockchain_info = self.get_blockchain_info().await?; - let sapling_id = zebra_rpc::methods::ConsensusBranchIdHex::new( - zebra_chain::parameters::ConsensusBranchId::from_hex("76b809bb") - .map_err(|_e| { - tonic::Status::internal( - "Internal Error - Consesnsus Branch ID hex conversion failed", - ) - })? - .into(), - ); - let sapling_activation_height = blockchain_info - .upgrades() - .get(&sapling_id) - .map_or(Height(1), |sapling_json| sapling_json.into_parts().1); - - let consensus_branch_id = zebra_chain::parameters::ConsensusBranchId::from( - blockchain_info.consensus().into_parts().0, - ) - .to_string(); - - let latest_upgrade = super::latest_network_upgrade(blockchain_info.upgrades()) - .map_err(StateServiceError::TonicStatusError)? - .into_parts(); - - let nu_name = latest_upgrade.0; - let nu_height = latest_upgrade.1; - - Ok(LightdInfo { - version: self.data.build_info().version(), - vendor: "ZingoLabs ZainoD".to_string(), - taddr_support: true, - chain_name: blockchain_info.chain().clone(), - sapling_activation_height: sapling_activation_height.0 as u64, - consensus_branch_id, - block_height: blockchain_info.blocks().0 as u64, - git_commit: self.data.build_info().commit_hash(), - branch: self.data.build_info().branch(), - build_date: self.data.build_info().build_date(), - build_user: self.data.build_info().build_user(), - estimated_height: blockchain_info.estimated_height().0 as u64, - zcashd_build: self.data.zebra_build(), - zcashd_subversion: self.data.zebra_subversion(), - donation_address: self - .config - .common - .donation_address - .as_ref() - .map(DonationAddress::encode) - .unwrap_or_default(), - upgrade_name: nu_name.to_string(), - upgrade_height: nu_height.0 as u64, - lightwallet_protocol_version: "v0.4.0".to_string(), - }) - } - - /// Testing-only, requires lightwalletd --ping-very-insecure (do not enable in production) - /// - /// NOTE: Currently unimplemented in Zaino. - async fn ping( - &self, - _request: zaino_proto::proto::service::Duration, - ) -> Result { - Err(crate::error::StateServiceError::TonicStatusError( - tonic::Status::unimplemented( - "Ping not yet implemented. If you require this RPC please open an \ - issue or PR at the Zaino github (https://github.com/zingolabs/zaino.git).", - ), - )) - } -} - -#[allow(clippy::result_large_err, deprecated)] -fn header_to_block_commitments( - header: &Header, - network: &Network, - height: Height, - final_sapling_root: [u8; 32], -) -> Result<[u8; 32], StateServiceError> { - let hash = match header.commitment(network, height).map_err(|e| { - StateServiceError::SerializationError( - zebra_chain::serialization::SerializationError::Parse( - // For some reason this error type takes a - // &'static str, and the the only way to create one - // dynamically is to leak a String. This shouldn't - // be a concern, as this error case should - // never occur when communing with a zebrad, which - // validates this field before serializing it - e.to_string().leak(), - ), - ) - })? { - zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, - zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, - zebra_chain::block::Commitment::ChainHistoryActivationReserved => [0; 32], - zebra_chain::block::Commitment::ChainHistoryRoot(root) => root.bytes_in_display_order(), - zebra_chain::block::Commitment::ChainHistoryBlockTxAuthCommitment(hash) => { - hash.bytes_in_display_order() - } - }; - Ok(hash) -} - -/// An error type for median time past calculation errors -#[derive(Debug, Clone)] -pub enum MedianTimePast { - /// The start block has no `time`. - StartMissingTime { hash: String }, - - /// Ignored verbosity. - UnexpectedRaw { hash: String }, - - /// No timestamps collected at all. - EmptyWindow, -} - -impl fmt::Display for MedianTimePast { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - MedianTimePast::StartMissingTime { hash } => { - write!(f, "start block {hash} is missing `time`") - } - MedianTimePast::UnexpectedRaw { hash } => { - write!(f, "unexpected raw payload for block {hash}") - } - MedianTimePast::EmptyWindow => { - write!(f, "no timestamps collected (empty MTP window)") - } - } - } -} - -impl Error for MedianTimePast {} - -#[cfg(test)] -mod tests { - /// Classifies the byte-level relationship between two slices. - #[derive(Debug, PartialEq)] - enum ByteRelation { - /// The slices are identical. - Equal, - /// `actual` fully byte-reversed equals `expected` (endian swap). - FullByteReversal, - /// Each byte's bits reversed maps `actual` to `expected`. - PerByteBitReversal, - /// Reversing bytes within 16-bit chunks maps `actual` to `expected`. - ChunkSwap16, - /// Reversing bytes within 32-bit chunks maps `actual` to `expected`. - ChunkSwap32, - /// Reversing bytes within 64-bit chunks maps `actual` to `expected`. - ChunkSwap64, - /// No recognized transformation. - Unrecognized, - } - - impl std::fmt::Display for ByteRelation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Equal => write!(f, "equal"), - Self::FullByteReversal => write!(f, "full byte-reversal (endian swap)"), - Self::PerByteBitReversal => write!(f, "per-byte bit-reversal"), - Self::ChunkSwap16 => write!(f, "16-bit pairwise byte-swap"), - Self::ChunkSwap32 => write!(f, "32-bit chunk byte-reversal"), - Self::ChunkSwap64 => write!(f, "64-bit chunk byte-reversal"), - Self::Unrecognized => write!(f, "unrecognized mismatch"), - } - } - } - - /// Applies each candidate byte transformation to `actual` and returns - /// the first that produces `expected`, or [`ByteRelation::Unrecognized`]. - // `u32::is_multiple_of` is only stable from Rust 1.87; keep `% n == 0` for our older MSRV. - #[allow(clippy::manual_is_multiple_of)] - fn classify_byte_relation(actual: &[u8], expected: &[u8]) -> ByteRelation { - if actual == expected { - return ByteRelation::Equal; - } - - let chunk_swap = |size: usize| -> Vec { - actual - .chunks(size) - .flat_map(|c| c.iter().rev()) - .copied() - .collect() - }; - - let mut reversed = actual.to_vec(); - reversed.reverse(); - if reversed == expected { - return ByteRelation::FullByteReversal; - } - - let bit_reversed: Vec = actual.iter().map(|b| b.reverse_bits()).collect(); - if bit_reversed == expected { - return ByteRelation::PerByteBitReversal; - } - - if actual.len() % 2 == 0 && chunk_swap(2) == expected { - return ByteRelation::ChunkSwap16; - } - if actual.len() % 4 == 0 && chunk_swap(4) == expected { - return ByteRelation::ChunkSwap32; - } - if actual.len() % 8 == 0 && chunk_swap(8) == expected { - return ByteRelation::ChunkSwap64; - } - - ByteRelation::Unrecognized - } - - /// Verifies that our Sapling address parsing logic produces the same - /// diversifier and diversified transmission key (pk_d) hex strings as - /// zcashd's `z_validateaddress` RPC. - /// - /// # Guarantees - /// - /// - Exercises the production `sapling_key_bytes` function directly. - /// - The 11-byte diversifier matches the zcashd-derived test vector. - /// - The 32-byte pk_d (after the endian reversal inside `sapling_key_bytes`) - /// matches the zcashd-derived test vector. - /// - If the upstream serialization changes, the failure message - /// classifies the mismatch (endian swap, bit-reversal, chunk swap, - /// or unrecognized) to aid diagnosis. - /// - /// # Non-guarantees - /// - /// - Does not prove the test vector constants themselves are correct; - /// they were captured from zcashd and are trusted as ground truth. - /// - Does not exercise the full `z_validate_address` RPC path through - /// `StateService` — only the `sapling_key_bytes` extraction function. - /// - Does not verify behavior for malformed Sapling addresses or - /// addresses on other networks (mainnet, testnet). - #[test] - fn sapling_pk_d_byte_order_matches_test_vector() { - use super::sapling_key_bytes; - use zcash_keys::address::Address; - use zcash_protocol::consensus::NetworkType; - - // Canonical source: live-tests/clientless/src/lib.rs::rpc::json_rpc - // Tracked for DRY consolidation: https://github.com/zingolabs/zaino/issues/988 - const SAPLING_ADDRESS: &str = "zregtestsapling1jalqhycwumq3unfxlzyzcktq3n478n82k2wacvl8gwfxk6ahshkxmtp2034qj28n7gl92ka5wca"; - const EXPECTED_DIVERSIFIER: &str = "977e0b930ee6c11e4d26f8"; - const EXPECTED_PK_D: &str = - "553ef2f328096a7c2aac6dec85b76b6b9243e733dc9db2eacce3eb8c60592c88"; - - let parsed: zcash_address::ZcashAddress = SAPLING_ADDRESS.parse().unwrap(); - let converted = parsed - .convert_if_network::
(NetworkType::Regtest) - .unwrap(); - - let Address::Sapling(s) = converted else { - panic!("expected Sapling address"); - }; - - let (diversifier, pk_d) = sapling_key_bytes(&s); - - let expected_diversifier = hex::decode(EXPECTED_DIVERSIFIER).unwrap(); - let expected_pk_d = hex::decode(EXPECTED_PK_D).unwrap(); - - // Diversifier - match classify_byte_relation(&diversifier, &expected_diversifier) { - ByteRelation::Equal => {} - relation => panic!( - "diversifier mismatch.\n relation: {relation}\n actual: {}\n expected: {}", - hex::encode(diversifier), - hex::encode(expected_diversifier), - ), - } - - // pk_d (sapling_key_bytes already applies the endian reversal) - match classify_byte_relation(&pk_d, &expected_pk_d) { - ByteRelation::Equal => {} - relation => panic!( - "pk_d mismatch — upstream serialization may have changed.\ - \n relation: {relation}\n actual: {}\n expected: {}", - hex::encode(pk_d), - hex::encode(expected_pk_d), - ), - } - } - - #[test] - fn classify_byte_relation_detects_known_transforms() { - let original = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - - assert_eq!( - classify_byte_relation(&original, &original), - ByteRelation::Equal, - ); - - let mut reversed = original.to_vec(); - reversed.reverse(); - assert_eq!( - classify_byte_relation(&original, &reversed), - ByteRelation::FullByteReversal, - ); - - let bit_rev: Vec = original.iter().map(|b| b.reverse_bits()).collect(); - assert_eq!( - classify_byte_relation(&original, &bit_rev), - ByteRelation::PerByteBitReversal, - ); - - let swapped_16: Vec = original - .chunks(2) - .flat_map(|c| c.iter().rev()) - .copied() - .collect(); - assert_eq!( - classify_byte_relation(&original, &swapped_16), - ByteRelation::ChunkSwap16, - ); - - let garbage = [0xFF; 8]; - assert_eq!( - classify_byte_relation(&original, &garbage), - ByteRelation::Unrecognized, - ); - } -} diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 182fd6107..1571d3b8a 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2,7 +2,7 @@ //! //! Components: //! - Mempool: Holds mempool transactions -//! - NonFinalisedState: Holds block data for the top 100 blocks of all chains. +//! - NonFinalisedState: Holds block data for the top `OPERATIONAL_NFS_DEPTH` blocks of all chains. //! - FinalisedState: Holds block data for the remainder of the best chain. //! //! - Chain: Holds chain / block structs used internally by the ChainIndex. @@ -37,10 +37,17 @@ use source::{BlockchainSource, ValidatorConnector}; use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; use tracing::{info, instrument}; +use zaino_fetch::jsonrpsee::raw_transaction::validate_raw_transaction_hex; use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, + block_deltas::BlockDeltas, + block_header::GetBlockHeader, + block_subsidy::GetBlockSubsidy, chain_tips::{ChainTip, ChainTipStatus, GetChainTipsResponse}, - EmptyTxOutSetInfo, GetTxOutSetInfo, GetTxOutSetInfoResponse, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + EmptyTxOutSetInfo, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, + GetTxOutResponse, GetTxOutSetInfo, GetTxOutSetInfoResponse, }; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use zebra_chain::parameters::ConsensusBranchId; @@ -48,16 +55,19 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos}, + methods::{ + AddressBalance, GetAddressUtxos, GetBlock, GetBlockchainInfoResponse, GetInfo, + SentTransactionHash, + }, }; use zebra_state::HashOrHeight; pub mod encoding; -/// All state below [`NON_FINALIZED_DEPTH`] blocks of the best-known chain tip. +/// All state below [`OPERATIONAL_NFS_DEPTH`] blocks of the best-known chain tip. pub mod finalised_state; /// State in the mempool, not yet on-chain pub mod mempool; -/// State within [`NON_FINALIZED_DEPTH`] blocks of the best-known chain tip; +/// State within [`OPERATIONAL_NFS_DEPTH`] blocks of the best-known chain tip; /// stored separately as it may be reorged. pub mod non_finalised_state; /// BlockchainSource @@ -68,28 +78,26 @@ pub mod types; #[cfg(test)] mod tests; -/// Distance (in blocks) between the best-known chain tip and the -/// highest block that zaino treats as part of the finalized DB. +/// Distance (in blocks) between the best-known chain tip and the highest block that +/// zaino treats as part of the finalised DB — the finalised / non-finalised seam. /// -/// Sourced from Zebra's protocol-derived reorg bound. The `+ 1` -/// preserves the original literal-`100` behavior; deriving the -/// depth from an explicit wider-consensus reference is tracked in -/// zingolabs/zaino#1130. -#[cfg(not(test))] -pub(crate) const NON_FINALIZED_DEPTH: u32 = zebra_state::MAX_BLOCK_REORG_HEIGHT + 1; - -/// In-crate unit tests pin the depth at the pre-zebra-10 value (`100`). +/// Sourced from the workspace's single source of truth, +/// [`zaino_common::consensus`]. Production uses the real +/// [`MAX_NONFINALISED_DEPTH`]. The tractable [`FAST_TEST_MAX_NONFINALISED_DEPTH`] +/// (= depth / 10) is selected for in-crate unit tests (`cfg(test)`) *and* for +/// cross-crate live tests that enable the `fast-test-seam` feature — so short mock +/// fixtures and small live chains still exercise a *moving* finalised seam. At the +/// real depth `finalized_height_floor` saturates to genesis for those fixtures and +/// the eviction/seam invariants become untestable (see zingolabs/zaino#1288). Both +/// arms derive from the same upstream reorg bound, so neither is a hard-coded literal. /// -/// Zebra 10 raised `MAX_BLOCK_REORG_HEIGHT` from 99 to 1000, so the -/// production depth is now 1001. The 201-block mock-chain test vector is -/// far shorter than that, so at the production depth `finalized_height_floor` -/// saturates to genesis for the whole fixture: the finalized seam never moves -/// off block 0 and the eviction/seam invariants become untestable (see -/// zingolabs/zaino#1288). The eviction and seam invariants are scale-free, so -/// exercising them at a tractable depth is sound; the production depth is -/// covered by the clientless suite, which reaches real chain heights. -#[cfg(test)] -pub(crate) const NON_FINALIZED_DEPTH: u32 = 100; +/// [`MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::MAX_NONFINALISED_DEPTH +/// [`FAST_TEST_MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH +#[cfg(not(any(test, feature = "fast-test-seam")))] +pub(crate) const OPERATIONAL_NFS_DEPTH: u32 = zaino_common::consensus::MAX_NONFINALISED_DEPTH; +#[cfg(any(test, feature = "fast-test-seam"))] +pub(crate) const OPERATIONAL_NFS_DEPTH: u32 = + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; /// Lower bound on zaino's finalized-DB tip, derived from the current /// best-known chain tip. @@ -100,7 +108,7 @@ pub(crate) const NON_FINALIZED_DEPTH: u32 = 100; /// `finalized_height` should account for the asymmetry /// (see zingolabs/zaino#1128). pub(crate) fn finalized_height_floor(chain_tip: u32) -> crate::Height { - crate::Height(chain_tip.saturating_sub(NON_FINALIZED_DEPTH)) + crate::Height(chain_tip.saturating_sub(OPERATIONAL_NFS_DEPTH)) } /// Current wall-clock time as a Unix timestamp in fractional seconds, for @@ -196,7 +204,7 @@ fn branch_len_to_active_chain( /// The interface to the chain index. /// /// `ChainIndex` provides a unified interface for querying blockchain data from different -/// backend sources. It combines access to both finalized state (older than 100 blocks) and +/// backend sources. It combines access to both finalized state (older than `OPERATIONAL_NFS_DEPTH` blocks) and /// non-finalized state (recent blocks that may still be reorganized). /// /// # Implementation @@ -315,6 +323,14 @@ fn branch_len_to_active_chain( /// /// When a call asks for info (e.g. a block), Zaino selects sources in this order: #[doc = simple_mermaid::mermaid!("chain_index_passthrough.mmd")] +/// +/// This trait holds the core methods required by the embedded wallet consumer +/// (zallet). RPC-server-only methods live on the [`ChainIndexRpcExt`] extension +/// trait. +/// +/// TODO: The core/extension split is a provisional first pass. It should be refined +/// into finer capability-based traits (zallet / lwd / block-explorer) in a follow-up +/// PR. pub trait ChainIndex { /// A snapshot of the nonfinalized state, needed for atomic access type Snapshot: NonFinalizedSnapshot; @@ -387,50 +403,6 @@ pub trait ChainIndex { end: Option, ) -> Option, Self::Error>>>; - /// Returns the *compact* block for the given height. - /// - /// Returns `None` if the specified `height` is greater than the snapshot's tip. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). - #[allow(clippy::type_complexity)] - fn get_compact_block( - &self, - nonfinalized_snapshot: &Self::Snapshot, - height: types::Height, - pool_types: PoolTypeFilter, - ) -> impl std::future::Future< - Output = Result, Self::Error>, - >; - - /// Streams *compact* blocks for an inclusive height range. - /// - /// Returns `None` if the requested range is entirely above the snapshot's tip. - /// - /// - The stream covers `[start_height, end_height]` (inclusive). - /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). - #[allow(clippy::type_complexity)] - fn get_compact_block_stream( - &self, - nonfinalized_snapshot: &Self::Snapshot, - start_height: types::Height, - end_height: types::Height, - pool_types: PoolTypeFilter, - ) -> impl std::future::Future, Self::Error>>; - // ********** Transaction methods ********** /// given a transaction id, returns the transaction, along with @@ -503,7 +475,16 @@ pub trait ChainIndex { fn get_treestate( &self, hash: &types::BlockHash, - ) -> impl std::future::Future>, Option>), Self::Error>>; + ) -> impl std::future::Future< + Output = Result< + ( + Option, + Option, + Option, + ), + Self::Error, + >, + >; /// Returns the subtree roots fn get_subtree_roots( @@ -515,12 +496,6 @@ pub trait ChainIndex { // ********** Transparent address history methods ********** - /// Returns all changes for the given transparent addresses. - fn get_address_deltas( - &self, - params: GetAddressDeltasParams, - ) -> impl std::future::Future>; - /// Returns the total transparent balance for the given addresses. fn get_address_balance( &self, @@ -556,6 +531,169 @@ pub trait ChainIndex { outpoints: Vec, scope: types::ChainScope, ) -> impl std::future::Future>, Self::Error>>; +} + +/// RPC-server extension methods layered on top of [`ChainIndex`]. +/// +/// The core [`ChainIndex`] trait holds the subset required by the embedded wallet +/// consumer (zallet). This extension holds the additional functionality required by +/// the gRPC (lightwalletd) and JSON-RPC servers: compact-block serving, mempool +/// metadata, address deltas, and the block-explorer / node-passthrough RPCs. +/// +/// TODO: This two-way core/extension split is a provisional first pass. It should be +/// refined into finer capability-based traits (zallet / lwd / block-explorer) in a +/// follow-up PR, at which point methods will be redistributed to their narrowest +/// capability. +pub trait ChainIndexRpcExt: ChainIndex { + // ********** Block methods ********** + + /// Returns the *compact* block for the given height. + /// + /// Returns `None` if the specified `height` is greater than the snapshot's tip. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + #[allow(clippy::type_complexity)] + fn get_compact_block( + &self, + nonfinalized_snapshot: &Self::Snapshot, + height: types::Height, + pool_types: PoolTypeFilter, + ) -> impl std::future::Future< + Output = Result, Self::Error>, + >; + + /// Streams *compact* blocks for an inclusive height range. + /// + /// Returns `None` if the requested range is entirely above the snapshot's tip. + /// + /// - The stream covers `[start_height, end_height]` (inclusive). + /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + #[allow(clippy::type_complexity)] + fn get_compact_block_stream( + &self, + nonfinalized_snapshot: &Self::Snapshot, + start_height: types::Height, + end_height: types::Height, + pool_types: PoolTypeFilter, + ) -> impl std::future::Future, Self::Error>>; + + /// Returns the `getblock`-shaped block for the given hash-or-height string. + /// + /// `verbosity` follows the zcashd `getblock` convention (0 = raw, 1 = object with + /// txids, 2 = object with full transaction data). + /// + /// zcashd reference: [`getblock`](https://zcash.github.io/rpc/getblock.html) + fn z_get_block( + &self, + hash_or_height: String, + verbosity: Option, + ) -> impl std::future::Future>; + + /// Returns the `getblockheader`-shaped header for the given block hash. + /// + /// zcashd reference: [`getblockheader`](https://zcash.github.io/rpc/getblockheader.html) + fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> impl std::future::Future>; + + /// Returns the `getblockdeltas`-shaped transparent input/output deltas for the block + /// with the given hash. + /// + /// zcashd reference: [`getblockdeltas`](https://zcash.github.io/rpc/getblockdeltas.html) + fn get_block_deltas( + &self, + hash: String, + ) -> impl std::future::Future>; + + /// Returns the proof-of-work difficulty of the best chain as a multiple of the + /// minimum difficulty. + /// + /// zcashd reference: [`getdifficulty`](https://zcash.github.io/rpc/getdifficulty.html) + fn get_difficulty(&self) -> impl std::future::Future>; + + // ********** Node-passthrough methods ********** + // + // No local-index equivalent; always delegate to the backing validator. + + /// Returns the `getinfo` response. + fn get_info(&self) -> impl std::future::Future>; + + /// Returns the `getblockchaininfo` response. + fn get_blockchain_info( + &self, + ) -> impl std::future::Future>; + + /// Returns the `getpeerinfo` response. + fn get_peer_info(&self) -> impl std::future::Future>; + + /// Returns the `getblocksubsidy` response at the given height. + fn get_block_subsidy( + &self, + height: u32, + ) -> impl std::future::Future>; + + /// Returns the `getmininginfo` response. + fn get_mining_info( + &self, + ) -> impl std::future::Future>; + + /// Returns the `gettxout` response for the given outpoint. + fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> impl std::future::Future>; + + /// Returns the `getspentinfo` response for the given request. + fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> impl std::future::Future>; + + /// Returns the `getnetworksolps` response. + fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> impl std::future::Future>; + + /// Submits a raw transaction to the network (`sendrawtransaction`). + fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> impl std::future::Future>; + + /// Returns the full `z_gettreestate` response for the given hash-or-height, via the + /// backing validator (node-passthrough fallback for treestates not locally serviceable). + fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> impl std::future::Future>; + + // ********** Transparent address history methods ********** + + /// Returns all changes for the given transparent addresses. + fn get_address_deltas( + &self, + params: GetAddressDeltasParams, + ) -> impl std::future::Future>; // ********** Metadata methods ********** @@ -802,7 +940,7 @@ impl NodeBackedChainIndex { finalized_db, sync_loop_handle: None, status: NamedAtomicStatus::new("ChainIndex", StatusType::Spawning), - network: config.network.to_zebra_network(), + network: config.network.clone(), source, sync_timings, cancel_token: CancellationToken::new(), @@ -838,11 +976,24 @@ impl NodeBackedChainIndex { /// the design we have. Cancelling first just removes the wasted /// failure-path round trip. pub async fn shutdown(&self) -> Result<(), FinalisedStateError> { + // The synchronous teardown (cancellation, mempool close, source + // release) runs before the fallible DB shutdown so a DB error cannot + // skip it — the source's Zebra syncer task must not outlive the index. + self.shutdown_sync_best_effort(); + self.finalized_db.shutdown().await + } + + /// Synchronous best-effort teardown for contexts that cannot run async + /// work (a `Drop` on a current-thread runtime, or on a thread with no + /// runtime at all): cancels the sync worker, closes the mempool, and + /// releases source-owned background work. The finalised DB's async + /// [`shutdown`](Self::shutdown) step is skipped; the DB's own `Drop` + /// remains responsible for releasing its resources. + pub(crate) fn shutdown_sync_best_effort(&self) { self.cancel_token.cancel(); self.status.store(StatusType::Closing); - self.finalized_db.shutdown().await?; self.mempool.close(); - Ok(()) + self.source.shutdown(); } /// Displays the status of the chain_index @@ -939,7 +1090,7 @@ impl NodeBackedChainIndex { Some(ref nfs) => nfs, None => { // Anchor the non-finalised state at `finalised_height` - // (= chain tip − NON_FINALIZED_DEPTH), never at genesis: a missing + // (= chain tip − OPERATIONAL_NFS_DEPTH), never at genesis: a missing // anchor used to fall through to genesis and then re-anchor up to the // lagging finalised tip, grinding millions of blocks one at a time // (#1261). `resolve_anchor_block` serves the anchor from the finalised @@ -1068,7 +1219,11 @@ impl Drop for NodeBackedChainIndex { /// on `cancel_token.cancelled()` wrapping the iter body), the worker /// exits at its next await checkpoint instead. fn drop(&mut self) { - self.cancel_token.cancel(); + // The full synchronous teardown, not just the cancellation: the + // source release in particular must run here, or the `State` + // connector's Zebra syncer task outlives an index that is dropped + // without an explicit `shutdown()` call. + self.shutdown_sync_best_effort(); } } @@ -1124,8 +1279,8 @@ async fn compact_block_from_source( .get_commitment_tree_roots(types::BlockHash::from(block.hash())) .await .map_err(ChainIndexError::backing_validator)?; - let (sapling_root, sapling_size, orchard_root, orchard_size) = - TreeRootData::new(tree_roots.0, tree_roots.1).extract_with_defaults(); + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood) = + TreeRootData::new(tree_roots.0, tree_roots.1, tree_roots.2).extract_with_defaults(); let metadata = BlockMetadata::new( sapling_root, @@ -1142,6 +1297,19 @@ async fn compact_block_from_source( "orchard commitment tree size overflow", )) })?, + ironwood + .map(|(root, size)| { + Ok::<_, ChainIndexError>(( + root, + size.try_into().map_err(|_| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "ironwood commitment tree size overflow", + )) + })?, + )) + }) + .transpose()?, None, // parent chainwork unknown — single-block construction network, ); @@ -1160,10 +1328,20 @@ async fn compact_block_from_source( } impl NodeBackedChainIndexSubscriber { - fn source(&self) -> &Source { + pub(crate) fn source(&self) -> &Source { &self.source } + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` + /// values directly off the mempool's entries. Production code goes through + /// the `ChainIndex` mempool API. + #[cfg(feature = "test_dependencies")] + pub(crate) fn mempool_subscriber(&self) -> &mempool::MempoolSubscriber { + &self.mempool + } + /// Returns the combined status of all chain index components. pub fn combined_status(&self) -> StatusType { let finalized_status = self.finalized_state.status(); @@ -1621,7 +1799,7 @@ impl ChainIndex for NodeBackedChainIndexSubscriber ChainIndex for NodeBackedChainIndexSubscriber Result, Option)>, Self::Error> { + // ChainIndex step 1 + if let Some(mempool_tx) = self + .mempool + .get_transaction(&mempool::MempoolKey { + txid: txid.to_rpc_hex(), + }) + .await + { + let bytes = mempool_tx.serialized_tx.as_ref().as_ref().to_vec(); + let mempool_branch_id = self.mempool_branch_id(snapshot); + + return Ok(Some((bytes, mempool_branch_id))); + } + + let Some((transaction, location)) = self + .source() + .get_transaction(*txid) + .await + .map_err(ChainIndexError::backing_validator)? + else { + return Ok(None); + }; + // as the reorg process cannot modify a transaction + // it's safe to serve nonfinalized state directly here + let height = match location { + GetTransactionLocation::BestChain(height) => height, + GetTransactionLocation::NonbestChain => { + // if the tranasction isn't on the best chain + // check our indexes. We need to find out the height from our index + // to determine the consensus branch ID + let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { + // If we don't have a block containing the transaction + // locally and the transaction's not on the validator's + // best chain, we can't determine its consensus branch ID + return Ok(None); + }; + + match self + .blocks_containing_transaction(non_finalized_snapshot, txid.0) + .await? + .next() + { + Some(block) => block.context.index.height.into(), + // As above Ok(None) + None => return Ok(None), + } + } + // We've already checked the mempool. Should be unreachable? + // todo: error here? + GetTransactionLocation::Mempool => return Ok(None), + }; + + Ok(Some(( + zebra_chain::transaction::SerializedTransaction::from(transaction) + .as_ref() + .to_vec(), + ConsensusBranchId::current(&self.network, height).map(u32::from), + ))) + } + + /// Given a transaction ID, returns all known blocks containing this transaction /// - /// **NOTE: This Method is currently not "passthrough aware", this should be added by - /// fetching block data from the backing validator when not locally available.** - async fn get_compact_block( + /// If the transaction is in the mempool, it will be in the `BestChainLocation` + /// if the mempool and snapshot are up-to-date, and the `NonBestChainLocation` set + /// if the snapshot is out-of-date compared to the mempool + async fn get_transaction_status( &self, snapshot: &Self::Snapshot, - height: types::Height, - pool_types: PoolTypeFilter, - ) -> Result, Self::Error> { + txid: &types::TransactionHash, + ) -> Result<(Option, HashSet), ChainIndexError> { match snapshot { ChainIndexSnapshot::NonFinalizedStateExists { non_finalized_snapshot, } => { - if height <= non_finalized_snapshot.best_tip.height { - Ok(Some(match snapshot.get_chainblock_by_height(&height) { - Some(block) => compact_block_with_pool_types( - block.to_compact_block(), - &pool_types.to_pool_types_vector(), - ), - None => { - match self - .finalized_state - .get_compact_block(height, pool_types.clone()) - .await - { - Ok(block) => block, - Err(_) => self - .get_compact_block_from_node(height, &pool_types) - .await? - .ok_or(ChainIndexError::database_hole(height, None))?, - } + let blocks_containing_transaction = self + .blocks_containing_transaction(non_finalized_snapshot, txid.0) + .await? + .collect::>(); + let Some(start_of_nonfinalized) = + non_finalized_snapshot.heights_to_hashes.keys().min() + else { + return Err(ChainIndexError::database_hole("no blocks", None)); + }; + let mut best_chain_block = blocks_containing_transaction + .iter() + .find(|block| { + non_finalized_snapshot + .heights_to_hashes + .get(&block.height()) + == Some(block.hash()) + || block.height() < *start_of_nonfinalized + // this block is either in the best chain ``heights_to_hashes`` or finalized. + }) + .map(|block| BestChainLocation::Block(*block.hash(), block.height())); + let mut non_best_chain_blocks: HashSet = + blocks_containing_transaction + .iter() + .filter(|block| { + non_finalized_snapshot + .heights_to_hashes + .get(&block.height()) + != Some(block.hash()) + && block.height() >= *start_of_nonfinalized + }) + .map(|block| NonBestChainLocation::Block(*block.hash(), block.height())) + .collect(); + let in_mempool = self + .mempool + .contains_txid(&mempool::MempoolKey { + txid: txid.to_rpc_hex(), + }) + .await; + if in_mempool { + let mempool_tip_hash = self.mempool.mempool_chain_tip(); + if mempool_tip_hash == non_finalized_snapshot.best_tip.hash { + if best_chain_block.is_some() { + return Err(ChainIndexError { + kind: ChainIndexErrorKind::InvalidSnapshot, + message: + "Best chain and up-to-date mempool both contain the same transaction" + .to_string(), + source: None, + }); + } else { + best_chain_block = Some(BestChainLocation::Mempool( + non_finalized_snapshot.best_tip.height + 1, + )); } - })) - } else { - Ok(None) + } else { + // the best chain and the mempool have divergent tip hashes + // get a new snapshot and use it to find the height of the mempool + if let ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot: new_snapshot, + } = self.snapshot_nonfinalized_state().await? + { + let target_height = + new_snapshot.blocks.iter().find_map(|(hash, block)| { + if *hash == mempool_tip_hash { + Some(block.height() + 1) + // found the block that is the tip that the mempool is hanging on to + } else { + None + } + }); + non_best_chain_blocks + .insert(NonBestChainLocation::Mempool(target_height)); + } + } } + Ok((best_chain_block, non_best_chain_blocks)) } - ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height: _, - //TODO: Once we make chainwork an option field we should be able to - // support passthrougth for this - } => Ok(None), + validator_finalized_height, + } => { + if let Some((_transaction, GetTransactionLocation::BestChain(height))) = self + .source() + .get_transaction(*txid) + .await + .map_err(ChainIndexError::backing_validator)? + { + if height <= *validator_finalized_height { + if let Some(block) = self + .source() + .get_block(HashOrHeight::Height(height)) + .await + .map_err(ChainIndexError::backing_validator)? + { + return Ok(( + Some(BestChainLocation::Block(block.hash().into(), height.into())), + HashSet::new(), + )); + } + } + } + Ok((None, HashSet::new())) + } } } - /// Streams *compact* blocks for an inclusive height range. - /// - /// Returns `Ok(None)` if the request is descending and `start_height` exceeds the chain tip. - /// For ascending requests that exceed the tip, returns a stream that ends with an - /// `out_of_range` error after all available blocks have been sent. - /// - /// - The stream covers `[start_height, end_height]` (inclusive). - /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. - /// - /// ## Pool filtering + /// Returns all txids currently in the mempool. + async fn get_mempool_txids(&self) -> Result, Self::Error> { + self.mempool + .get_mempool() + .await + .into_iter() + .map(|(txid_key, _)| { + TransactionHash::from_hex(&txid_key.txid) + .map_err(ChainIndexError::backing_validator) + }) + .collect::>() + } + + /// Returns all transactions currently in the mempool, filtered by `exclude_list`. /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). + /// The `exclude_list` may contain shortened transaction ID hex prefixes (client-endian). + /// The transaction IDs in the Exclude list can be shortened to any number of bytes to make the request + /// more bandwidth-efficient; if two or more transactions in the mempool + /// match a shortened txid, they are all sent (none is excluded). Transactions + /// in the exclude list that don't exist in the mempool are ignored. + async fn get_mempool_transactions( + &self, + exclude_list: Vec, + ) -> Result>, Self::Error> { + // Use the mempool's own filtering (it already handles client-endian shortened prefixes). + let pairs: Vec<(mempool::MempoolKey, mempool::MempoolValue)> = + self.mempool.get_filtered_mempool(exclude_list).await; + + // Transform to the Vec> that the trait requires. + let bytes: Vec> = pairs + .into_iter() + .map(|(_, v)| v.serialized_tx.as_ref().as_ref().to_vec()) + .collect(); + + Ok(bytes) + } + + /// Returns a stream of mempool transactions, ending the stream when the chain tip block hash + /// changes (a new block is mined or a reorg occurs). /// - /// **NOTE: This Method is currently not "passthrough aware", this should be added by - /// fetching block data from the backing validator when not locally available.** - #[allow(clippy::type_complexity)] - async fn get_compact_block_stream( + /// If a snapshot is given and the chain tip has changed from the given spanshot, returns None. + fn get_mempool_stream( &self, - nonfinalized_snapshot: &Self::Snapshot, - start_height: types::Height, - end_height: types::Height, - pool_types: PoolTypeFilter, - ) -> Result, Self::Error> { - let chain_tip_height = self.best_chaintip(nonfinalized_snapshot).await?.height; + snapshot: Option<&Self::Snapshot>, + ) -> Option, Self::Error>>> { + let non_finalized_snapshot = match snapshot { + Some(s) => match s { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => Some(non_finalized_snapshot), + // If we're still syncing the finalized state, the chain tip + // is newer than the snapshot's tip. Return None. + ChainIndexSnapshot::StillSyncingFinalizedState { .. } => return None, + }, + None => None, + }; + let expected_chain_tip = non_finalized_snapshot.map(|snapshot| snapshot.best_tip.hash); + let mut subscriber = self.mempool.clone(); - // The nonfinalized cache holds the tip block plus the previous 99 blocks (100 total), - // so the lowest possible cached height is `tip - 99` (saturating at 0). - let lowest_nonfinalized_height = types::Height(chain_tip_height.0.saturating_sub(99)); + match subscriber + .get_mempool_stream(expected_chain_tip) + .now_or_never() + { + Some(Ok((in_rx, _handle))) => { + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(32); - let is_ascending = start_height <= end_height; + tokio::spawn(async move { + let mut in_stream = tokio_stream::wrappers::ReceiverStream::new(in_rx); + while let Some(item) = in_stream.next().await { + match item { + Ok((_key, value)) => { + let _ = out_tx + .send(Ok(value.serialized_tx.as_ref().as_ref().to_vec())) + .await; + } + Err(e) => { + let _ = out_tx + .send(Err(ChainIndexError::child_process_status_error( + "mempool", e, + ))) + .await; + break; + } + } + } + }); - // Descending: the first block we'd try to return is already above the tip → error immediately. - if !is_ascending && start_height > chain_tip_height { - return Ok(None); + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + Some(Err(crate::error::MempoolError::IncorrectChainTip { .. })) => None, + Some(Err(e)) => { + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(1); + let _ = out_tx.try_send(Err(e.into())); + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + None => { + // Should not happen because the inner tip check is synchronous, but fail safe. + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(1); + let _ = out_tx.try_send(Err(ChainIndexError::child_process_status_error( + "mempool", + crate::error::StatusError { + server_status: crate::StatusType::RecoverableError, + }, + ))); + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } } + } - let pool_types_vector = pool_types.to_pool_types_vector(); + // ********** Chain methods ********** - // For ascending requests that extend past the tip: cap the streaming range at the tip, - // then append a trailing out_of_range error after all valid blocks have been sent. - let needs_out_of_range = is_ascending && end_height > chain_tip_height; - let capped_end_height = if needs_out_of_range { - chain_tip_height - } else { - end_height - }; - - // Pre-create any finalized-state stream(s) we will need so that errors are returned - // from this method (not deferred into the spawned task). - let finalized_stream: Option = if is_ascending { - if start_height < lowest_nonfinalized_height { - let finalized_end_height = types::Height(std::cmp::min( - capped_end_height.0, - lowest_nonfinalized_height.0.saturating_sub(1), - )); - - if start_height <= finalized_end_height { - Some( - self.finalized_state - .get_compact_block_stream( - start_height, - finalized_end_height, - pool_types.clone(), - ) - .await - .map_err(ChainIndexError::from)?, - ) - } else { - None - } - } else { - None - } - // Serve in reverse order. - } else if end_height < lowest_nonfinalized_height { - let finalized_start_height = if start_height < lowest_nonfinalized_height { - start_height - } else { - types::Height(lowest_nonfinalized_height.0.saturating_sub(1)) - }; - - Some( - self.finalized_state - .get_compact_block_stream( - finalized_start_height, - end_height, - pool_types.clone(), - ) - .await - .map_err(ChainIndexError::from)?, - ) - } else { - None - }; - - let nonfinalized_snapshot = nonfinalized_snapshot.clone(); - let source = self.source.clone(); - let network = self.network.clone(); - let pool_types_for_node = pool_types.clone(); - // TODO: Investigate whether channel size should be changed, added to config, or set dynamically based on resources. - let (channel_sender, channel_receiver) = tokio::sync::mpsc::channel(128); + /// For a given block, + /// find its newest main-chain ancestor, + /// or the block itself if it is on the main-chain. + /// Returns Ok(None) if no fork point found. This is not an error, + /// as zaino does not guarentee knowledge of all sidechain data. + async fn find_fork_point( + &self, + snapshot: &Self::Snapshot, + hash: &types::BlockHash, + ) -> Result, Self::Error> { + // ChainIndex step 1: Skip + // mempool blocks have no canon height, guaranteed to return None + // todo: possible efficiency boost by checking mempool for a negative? - tokio::spawn(async move { - if is_ascending { - // 1) Finalized segment (if any), ascending. - if let Some(mut finalized_stream) = finalized_stream { - while let Some(stream_item) = finalized_stream.next().await { - if channel_sender.send(stream_item).await.is_err() { - return; + match snapshot { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => { + match non_finalized_snapshot.get_chainblock_by_hash(hash) { + Some(block) => { + // At this point, we know that + // The block is non-FINALIZED in the INDEXER + // ChainIndex step 3: + if non_finalized_snapshot + .heights_to_hashes + .get(&block.height()) + == Some(block.hash()) + { + // The block is in the best chain. + Ok(Some((*block.hash(), block.height()))) + } else { + // Otherwise, it's non-best chain! Grab its parent, and recurse + Box::pin(self.find_fork_point(snapshot, &block.context.parent_hash)) + .await + // gotta pin recursive async functions to prevent infinite-sized + // Future-implementing types } } - } - - // 2) Nonfinalized segment, ascending. - let nonfinalized_start_height = - types::Height(std::cmp::max(start_height.0, lowest_nonfinalized_height.0)); - - for height_value in nonfinalized_start_height.0..=capped_end_height.0 { - let Some(indexed_block) = nonfinalized_snapshot - .get_chainblock_by_height(&types::Height(height_value)) - else { - match compact_block_from_source( - &source, - network.clone(), - types::Height(height_value), - &pool_types_for_node, - ) - .await - { - Ok(Some(compact_block)) => { - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - continue; - } - Ok(None) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(format!( - "Internal error, missing nonfinalized block at height [{height_value}].", - )))) - .await; - return; - } - Err(error) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(error.to_string()))) - .await; - return; + None => { + // At this point, we know that + // the block is NOT non-FINALIZED in the INDEXER. + // as the non finalzed state is known to be populated, + // we now check the finalized state + match self.finalized_state.get_block_height(*hash).await { + Ok(Some(height)) => { + // the block is FINALIZED in the INDEXER + Ok(Some((*hash, height))) } + Err(e) => Err(ChainIndexError::database_hole(hash, Some(Box::new(e)))), + Ok(None) => Ok(None), } - }; - let compact_block = compact_block_with_pool_types( - indexed_block.to_compact_block(), - &pool_types_vector, - ); - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; } } - // If the original end_height was above the tip, signal out_of_range after all valid blocks. - if needs_out_of_range { - let _ = channel_sender - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{}]. Height requested is greater than the best chain tip [{}].", - end_height.0, chain_tip_height.0, - )))) - .await; - } - } else { - // 1) Nonfinalized segment, descending. - if start_height >= lowest_nonfinalized_height { - let nonfinalized_end_height = - types::Height(std::cmp::max(end_height.0, lowest_nonfinalized_height.0)); - - for height_value in (nonfinalized_end_height.0..=start_height.0).rev() { - let Some(indexed_block) = nonfinalized_snapshot - .get_chainblock_by_height(&types::Height(height_value)) - else { - match compact_block_from_source( - &source, - network.clone(), - types::Height(height_value), - &pool_types_for_node, - ) - .await - { - Ok(Some(compact_block)) => { - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - continue; - } - Ok(None) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(format!( - "Internal error, missing nonfinalized block at height [{height_value}].", - )))) - .await; - return; - } - Err(error) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(error.to_string()))) - .await; - return; + } + ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height, + } => { + // We're not fully synced, so we pass through. + // Now, we ask the VALIDATOR. + // ChainIndex step 5 + match self + .source() + .get_block(HashOrHeight::Hash(zebra_chain::block::Hash::from(*hash))) + .await + { + Ok(Some(block)) => { + // At this point, we know that + // the block is in the VALIDATOR. + match block.coinbase_height() { + None => { + // the block is in the VALIDATOR. but doesnt have a height. That would imply a bug. + Err(ChainIndexError::validator_data_error_block_coinbase_height_missing()) + } + Some(height) => { + // The VALIDATOR returned a block with a height. + // However, there is as of yet no guaranteed the Block is FINALIZED + if height <= *validator_finalized_height { + Ok(Some(( + types::BlockHash::from(block.hash()), + types::Height::from(height), + ))) + } else { + // non-finalized block + // no passthrough + Ok(None) } } - }; - let compact_block = compact_block_with_pool_types( - indexed_block.to_compact_block(), - &pool_types_vector, - ); - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; } } - } - // 2) Finalized segment (if any), descending. - if let Some(mut finalized_stream) = finalized_stream { - while let Some(stream_item) = finalized_stream.next().await { - if channel_sender.send(stream_item).await.is_err() { - return; - } + Ok(None) => { + // At this point, we know that + // the block is NOT FINALIZED in the VALIDATOR. + // Return Ok(None) = no block found. + Ok(None) } + Err(e) => Err(ChainIndexError::backing_validator(e)), } } - }); - - Ok(Some(CompactBlockStream::new(channel_receiver))) + } } - // ********** Transaction methods ********** + /// Returns the block commitment tree data by hash. + async fn get_treestate( + &self, + hash: &types::BlockHash, + ) -> Result< + ( + Option, + Option, + Option, + ), + Self::Error, + > { + let snapshot = self.snapshot_nonfinalized_state().await?; + if !self.block_hash_known_for_treestate(&snapshot, hash).await? { + return Err(ChainIndexError::internal(format!( + "block hash {hash} not found in local chain index" + ))); + } - /// given a transaction id, returns the transaction - /// and the consensus branch ID for the block the transaction - /// is in - async fn get_raw_transaction( + match self.source().get_treestate(*hash).await { + Ok(resp) => Ok(resp), + Err(e) => Err(ChainIndexError { + kind: ChainIndexErrorKind::InternalServerError, + message: "failed to fetch treestate from validator".to_string(), + source: Some(Box::new(e)), + }), + } + } + + /// Gets the subtree roots of a given pool and the end heights of each root, + /// starting at the provided index, up to an optional maximum number of roots. + async fn get_subtree_roots( &self, - snapshot: &Self::Snapshot, - txid: &types::TransactionHash, - ) -> Result, Option)>, Self::Error> { - // ChainIndex step 1 - if let Some(mempool_tx) = self - .mempool - .get_transaction(&mempool::MempoolKey { - txid: txid.to_rpc_hex(), - }) + pool: ShieldedPool, + start_index: u16, + max_entries: Option, + ) -> Result, Self::Error> { + self.source() + .get_subtree_roots(pool, start_index, max_entries) .await - { - let bytes = mempool_tx.serialized_tx.as_ref().as_ref().to_vec(); - let mempool_branch_id = self.mempool_branch_id(snapshot); + .map_err(ChainIndexError::backing_validator) + } - return Ok(Some((bytes, mempool_branch_id))); - } + // ********** Transparent address history methods ********** - let Some((transaction, location)) = self - .source() - .get_transaction(*txid) + /// Returns the total transparent balance for the given addresses. + async fn get_address_balance( + &self, + address_strings: GetAddressBalanceRequest, + ) -> Result { + self.source() + .get_address_balance(address_strings) .await - .map_err(ChainIndexError::backing_validator)? - else { - return Ok(None); - }; - // as the reorg process cannot modify a transaction - // it's safe to serve nonfinalized state directly here - let height = match location { - GetTransactionLocation::BestChain(height) => height, - GetTransactionLocation::NonbestChain => { - // if the tranasction isn't on the best chain - // check our indexes. We need to find out the height from our index - // to determine the consensus branch ID - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // If we don't have a block containing the transaction - // locally and the transaction's not on the validator's - // best chain, we can't determine its consensus branch ID - return Ok(None); - }; + .map_err(ChainIndexError::backing_validator) + } - match self - .blocks_containing_transaction(non_finalized_snapshot, txid.0) - .await? - .next() - { - Some(block) => block.context.index.height.into(), - // As above Ok(None) - None => return Ok(None), - } - } - // We've already checked the mempool. Should be unreachable? - // todo: error here? - GetTransactionLocation::Mempool => return Ok(None), - }; + /// Returns the transaction ids made by the given transparent addresses. + async fn get_address_txids( + &self, + request: GetAddressTxIdsRequest, + ) -> Result, Self::Error> { + self.source() + .get_address_txids(request) + .await + .map_err(ChainIndexError::backing_validator) + } - Ok(Some(( - zebra_chain::transaction::SerializedTransaction::from(transaction) - .as_ref() - .to_vec(), - ConsensusBranchId::current(&self.network, height).map(u32::from), - ))) + /// Returns all unspent transparent outputs for the given addresses. + async fn get_address_utxos( + &self, + address_strings: GetAddressBalanceRequest, + ) -> Result, Self::Error> { + self.source() + .get_address_utxos(address_strings) + .await + .map_err(ChainIndexError::backing_validator) } - /// Given a transaction ID, returns all known blocks containing this transaction - /// - /// If the transaction is in the mempool, it will be in the `BestChainLocation` - /// if the mempool and snapshot are up-to-date, and the `NonBestChainLocation` set - /// if the snapshot is out-of-date compared to the mempool - async fn get_transaction_status( + async fn get_outpoint_spenders( &self, snapshot: &Self::Snapshot, - txid: &types::TransactionHash, - ) -> Result<(Option, HashSet), ChainIndexError> { - match snapshot { + outpoints: Vec, + scope: types::ChainScope, + ) -> Result>, Self::Error> { + use std::collections::HashMap; + + let mut result: Vec> = vec![None; outpoints.len()]; + + // 1) Non-finalised best chain (FullChain scope only). Scan only the blocks reachable + // via `heights_to_hashes` (the `blocks` map also holds reorged-away blocks, which + // must not count). One pass builds an outpoint -> spending-txid map regardless of + // how many outpoints we look up. Under `Finalised` scope this is skipped so results + // are reorg-stable. + if let ( + types::ChainScope::FullChain, ChainIndexSnapshot::NonFinalizedStateExists { non_finalized_snapshot, - } => { - let blocks_containing_transaction = self - .blocks_containing_transaction(non_finalized_snapshot, txid.0) - .await? - .collect::>(); - let Some(start_of_nonfinalized) = - non_finalized_snapshot.heights_to_hashes.keys().min() - else { - return Err(ChainIndexError::database_hole("no blocks", None)); + }, + ) = (scope, snapshot) + { + let mut nfs_spenders: HashMap = HashMap::new(); + for hash in non_finalized_snapshot.heights_to_hashes.values() { + let Some(block) = non_finalized_snapshot.blocks.get(hash) else { + continue; }; - let mut best_chain_block = blocks_containing_transaction - .iter() - .find(|block| { - non_finalized_snapshot - .heights_to_hashes - .get(&block.height()) - == Some(block.hash()) - || block.height() < *start_of_nonfinalized - // this block is either in the best chain ``heights_to_hashes`` or finalized. - }) - .map(|block| BestChainLocation::Block(*block.hash(), block.height())); - let mut non_best_chain_blocks: HashSet = - blocks_containing_transaction - .iter() - .filter(|block| { - non_finalized_snapshot - .heights_to_hashes - .get(&block.height()) - != Some(block.hash()) - && block.height() >= *start_of_nonfinalized - }) - .map(|block| NonBestChainLocation::Block(*block.hash(), block.height())) - .collect(); - let in_mempool = self - .mempool - .contains_txid(&mempool::MempoolKey { - txid: txid.to_rpc_hex(), - }) - .await; - if in_mempool { - let mempool_tip_hash = self.mempool.mempool_chain_tip(); - if mempool_tip_hash == non_finalized_snapshot.best_tip.hash { - if best_chain_block.is_some() { - return Err(ChainIndexError { - kind: ChainIndexErrorKind::InvalidSnapshot, - message: - "Best chain and up-to-date mempool both contain the same transaction" - .to_string(), - source: None, - }); - } else { - best_chain_block = Some(BestChainLocation::Mempool( - non_finalized_snapshot.best_tip.height + 1, - )); - } - } else { - // the best chain and the mempool have divergent tip hashes - // get a new snapshot and use it to find the height of the mempool - if let ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot: new_snapshot, - } = self.snapshot_nonfinalized_state().await? - { - let target_height = - new_snapshot.blocks.iter().find_map(|(hash, block)| { - if *hash == mempool_tip_hash { - Some(block.height() + 1) - // found the block that is the tip that the mempool is hanging on to - } else { - None - } - }); - non_best_chain_blocks - .insert(NonBestChainLocation::Mempool(target_height)); - } + for tx in block.transactions() { + let txid = *tx.txid(); + // `spent_outpoints` already skips coinbase null prevouts and builds each + // `Outpoint`, keeping the construction in one place (see #1332). + for outpoint in tx.transparent().spent_outpoints() { + nfs_spenders.insert(outpoint, txid); } } - Ok((best_chain_block, non_best_chain_blocks)) } - ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height, - } => { - if let Some((_transaction, GetTransactionLocation::BestChain(height))) = self - .source() - .get_transaction(*txid) - .await - .map_err(ChainIndexError::backing_validator)? - { - if height <= *validator_finalized_height { - if let Some(block) = self - .source() - .get_block(HashOrHeight::Height(height)) - .await - .map_err(ChainIndexError::backing_validator)? - { - return Ok(( - Some(BestChainLocation::Block(block.hash().into(), height.into())), - HashSet::new(), - )); - } - } - } - Ok((None, HashSet::new())) + for (i, outpoint) in outpoints.iter().enumerate() { + if let Some(txid) = nfs_spenders.get(outpoint) { + result[i] = Some(*txid); + } } } - } - - /// Returns all txids currently in the mempool. - async fn get_mempool_txids(&self) -> Result, Self::Error> { - self.mempool - .get_mempool() - .await - .into_iter() - .map(|(txid_key, _)| { - TransactionHash::from_hex(&txid_key.txid) - .map_err(ChainIndexError::backing_validator) - }) - .collect::>() - } - - /// Returns all transactions currently in the mempool, filtered by `exclude_list`. - /// - /// The `exclude_list` may contain shortened transaction ID hex prefixes (client-endian). - /// The transaction IDs in the Exclude list can be shortened to any number of bytes to make the request - /// more bandwidth-efficient; if two or more transactions in the mempool - /// match a shortened txid, they are all sent (none is excluded). Transactions - /// in the exclude list that don't exist in the mempool are ignored. - async fn get_mempool_transactions( - &self, - exclude_list: Vec, - ) -> Result>, Self::Error> { - // Use the mempool's own filtering (it already handles client-endian shortened prefixes). - let pairs: Vec<(mempool::MempoolKey, mempool::MempoolValue)> = - self.mempool.get_filtered_mempool(exclude_list).await; - // Transform to the Vec> that the trait requires. - let bytes: Vec> = pairs - .into_iter() - .map(|(_, v)| v.serialized_tx.as_ref().as_ref().to_vec()) + // 2) Finalised lookup for the still-unresolved outpoints, batched into one DB call. + let unresolved_indices: Vec = result + .iter() + .enumerate() + .filter_map(|(i, r)| r.is_none().then_some(i)) .collect(); + if unresolved_indices.is_empty() { + return Ok(result); + } + let unresolved_outpoints: Vec = + unresolved_indices.iter().map(|&i| outpoints[i]).collect(); + let locations = self + .finalized_state + .get_outpoint_spenders(unresolved_outpoints) + .await?; - Ok(bytes) - } + // 3) Resolve each finalised `TxLocation` to a txid. Dedup identical locations so a + // block spending several queried outpoints is only fetched once. `get_txid` is a + // single keyed lookup, far cheaper than reconstructing the whole block. + let mut slots_by_location: HashMap> = HashMap::new(); + for (slot, location) in unresolved_indices.into_iter().zip(locations) { + if let Some(location) = location { + slots_by_location.entry(location).or_default().push(slot); + } + } + for (location, slots) in slots_by_location { + let txid = self.finalized_state.get_txid(location).await?; + for slot in slots { + result[slot] = Some(txid); + } + } - /// Returns a stream of mempool transactions, ending the stream when the chain tip block hash - /// changes (a new block is mined or a reorg occurs). - /// - /// If a snapshot is given and the chain tip has changed from the given spanshot, returns None. - fn get_mempool_stream( - &self, - snapshot: Option<&Self::Snapshot>, - ) -> Option, Self::Error>>> { - let non_finalized_snapshot = match snapshot { - Some(s) => match s { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => Some(non_finalized_snapshot), - // If we're still syncing the finalized state, the chain tip - // is newer than the snapshot's tip. Return None. - ChainIndexSnapshot::StillSyncingFinalizedState { .. } => return None, - }, - None => None, - }; - let expected_chain_tip = non_finalized_snapshot.map(|snapshot| snapshot.best_tip.hash); - let mut subscriber = self.mempool.clone(); + Ok(result) + } - match subscriber - .get_mempool_stream(expected_chain_tip) - .now_or_never() - { - Some(Ok((in_rx, _handle))) => { - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(32); + // ********** Metadata methods ********** - tokio::spawn(async move { - let mut in_stream = tokio_stream::wrappers::ReceiverStream::new(in_rx); - while let Some(item) = in_stream.next().await { - match item { - Ok((_key, value)) => { - let _ = out_tx - .send(Ok(value.serialized_tx.as_ref().as_ref().to_vec())) - .await; - } - Err(e) => { - let _ = out_tx - .send(Err(ChainIndexError::child_process_status_error( - "mempool", e, - ))) - .await; - break; - } - } - } - }); + async fn best_chaintip(&self, snapshot: &Self::Snapshot) -> Result { + Ok(match snapshot { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => non_finalized_snapshot.best_tip, - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) - } - Some(Err(crate::error::MempoolError::IncorrectChainTip { .. })) => None, - Some(Err(e)) => { - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(1); - let _ = out_tx.try_send(Err(e.into())); - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) - } - None => { - // Should not happen because the inner tip check is synchronous, but fail safe. - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(1); - let _ = out_tx.try_send(Err(ChainIndexError::child_process_status_error( - "mempool", - crate::error::StatusError { - server_status: crate::StatusType::RecoverableError, - }, - ))); - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height, + } => { + BlockIndex { + height: *validator_finalized_height, + hash: self + .source() + // TODO: do something more efficient than getting the whole block + .get_block(HashOrHeight::Height((*validator_finalized_height).into())) + .await + .map_err(|e| { + ChainIndexError::database_hole( + validator_finalized_height, + Some(Box::new(e)), + ) + })? + .ok_or(ChainIndexError::database_hole( + validator_finalized_height, + None, + ))? + .hash() + .into(), + } } - } + }) } +} - // ********** Chain methods ********** - - /// For a given block, - /// find its newest main-chain ancestor, - /// or the block itself if it is on the main-chain. - /// Returns Ok(None) if no fork point found. This is not an error, - /// as zaino does not guarentee knowledge of all sidechain data. - async fn find_fork_point( +impl ChainIndexRpcExt for NodeBackedChainIndexSubscriber { + /// Returns the *compact* block for the given height. + /// + /// Returns `None` if the specified `height` is greater than the snapshot's tip. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + /// + /// Returns None if the specified height + /// is greater than the snapshot's tip + /// + /// **NOTE: This Method is currently not "passthrough aware", this should be added by + /// fetching block data from the backing validator when not locally available.** + async fn get_compact_block( &self, snapshot: &Self::Snapshot, - hash: &types::BlockHash, - ) -> Result, Self::Error> { - // ChainIndex step 1: Skip - // mempool blocks have no canon height, guaranteed to return None - // todo: possible efficiency boost by checking mempool for a negative? - + height: types::Height, + pool_types: PoolTypeFilter, + ) -> Result, Self::Error> { match snapshot { ChainIndexSnapshot::NonFinalizedStateExists { non_finalized_snapshot, } => { - match non_finalized_snapshot.get_chainblock_by_hash(hash) { - Some(block) => { - // At this point, we know that - // The block is non-FINALIZED in the INDEXER - // ChainIndex step 3: - if non_finalized_snapshot - .heights_to_hashes - .get(&block.height()) - == Some(block.hash()) - { - // The block is in the best chain. - Ok(Some((*block.hash(), block.height()))) - } else { - // Otherwise, it's non-best chain! Grab its parent, and recurse - Box::pin(self.find_fork_point(snapshot, &block.context.parent_hash)) + if height <= non_finalized_snapshot.best_tip.height { + Ok(Some(match snapshot.get_chainblock_by_height(&height) { + Some(block) => compact_block_with_pool_types( + block.to_compact_block(), + &pool_types.to_pool_types_vector(), + ), + None => { + match self + .finalized_state + .get_compact_block(height, pool_types.clone()) .await - // gotta pin recursive async functions to prevent infinite-sized - // Future-implementing types - } - } - None => { - // At this point, we know that - // the block is NOT non-FINALIZED in the INDEXER. - // as the non finalzed state is known to be populated, - // we now check the finalized state - match self.finalized_state.get_block_height(*hash).await { - Ok(Some(height)) => { - // the block is FINALIZED in the INDEXER - Ok(Some((*hash, height))) + { + Ok(block) => block, + Err(_) => self + .get_compact_block_from_node(height, &pool_types) + .await? + .ok_or(ChainIndexError::database_hole(height, None))?, } - Err(e) => Err(ChainIndexError::database_hole(hash, Some(Box::new(e)))), - Ok(None) => Ok(None), } - } + })) + } else { + Ok(None) } } + ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height, - } => { - // We're not fully synced, so we pass through. - // Now, we ask the VALIDATOR. - // ChainIndex step 5 - match self - .source() - .get_block(HashOrHeight::Hash(zebra_chain::block::Hash::from(*hash))) + validator_finalized_height: _, + //TODO: Once we make chainwork an option field we should be able to + // support passthrougth for this + } => Ok(None), + } + } + + /// Streams *compact* blocks for an inclusive height range. + /// + /// Returns `Ok(None)` if the request is descending and `start_height` exceeds the chain tip. + /// For ascending requests that exceed the tip, returns a stream that ends with an + /// `out_of_range` error after all available blocks have been sent. + /// + /// - The stream covers `[start_height, end_height]` (inclusive). + /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + /// + /// **NOTE: This Method is currently not "passthrough aware", this should be added by + /// fetching block data from the backing validator when not locally available.** + #[allow(clippy::type_complexity)] + async fn get_compact_block_stream( + &self, + nonfinalized_snapshot: &Self::Snapshot, + start_height: types::Height, + end_height: types::Height, + pool_types: PoolTypeFilter, + ) -> Result, Self::Error> { + let chain_tip_height = self.best_chaintip(nonfinalized_snapshot).await?.height; + + // The finalised state serves heights up to `finalized_height_floor(tip)` + // (= `tip - OPERATIONAL_NFS_DEPTH`, saturating at genesis); the non-finalised cache serves + // everything above it, so the lowest non-finalised height is one past the finalised floor. + let lowest_nonfinalized_height = + types::Height(finalized_height_floor(chain_tip_height.0).0 + 1); + + let is_ascending = start_height <= end_height; + + // Descending: the first block we'd try to return is already above the tip → error immediately. + if !is_ascending && start_height > chain_tip_height { + return Ok(None); + } + + let pool_types_vector = pool_types.to_pool_types_vector(); + + // For ascending requests that extend past the tip: cap the streaming range at the tip, + // then append a trailing out_of_range error after all valid blocks have been sent. + let needs_out_of_range = is_ascending && end_height > chain_tip_height; + let capped_end_height = if needs_out_of_range { + chain_tip_height + } else { + end_height + }; + + // Pre-create any finalized-state stream(s) we will need so that errors are returned + // from this method (not deferred into the spawned task). + let finalized_stream: Option = if is_ascending { + if start_height < lowest_nonfinalized_height { + let finalized_end_height = types::Height(std::cmp::min( + capped_end_height.0, + lowest_nonfinalized_height.0.saturating_sub(1), + )); + + if start_height <= finalized_end_height { + Some( + self.finalized_state + .get_compact_block_stream( + start_height, + finalized_end_height, + pool_types.clone(), + ) + .await + .map_err(ChainIndexError::from)?, + ) + } else { + None + } + } else { + None + } + // Serve in reverse order. + } else if end_height < lowest_nonfinalized_height { + let finalized_start_height = if start_height < lowest_nonfinalized_height { + start_height + } else { + types::Height(lowest_nonfinalized_height.0.saturating_sub(1)) + }; + + Some( + self.finalized_state + .get_compact_block_stream( + finalized_start_height, + end_height, + pool_types.clone(), + ) .await - { - Ok(Some(block)) => { - // At this point, we know that - // the block is in the VALIDATOR. - match block.coinbase_height() { - None => { - // the block is in the VALIDATOR. but doesnt have a height. That would imply a bug. - Err(ChainIndexError::validator_data_error_block_coinbase_height_missing()) + .map_err(ChainIndexError::from)?, + ) + } else { + None + }; + + let nonfinalized_snapshot = nonfinalized_snapshot.clone(); + let source = self.source.clone(); + let network = self.network.clone(); + let pool_types_for_node = pool_types.clone(); + // TODO: Investigate whether channel size should be changed, added to config, or set dynamically based on resources. + let (channel_sender, channel_receiver) = tokio::sync::mpsc::channel(128); + + tokio::spawn(async move { + if is_ascending { + // 1) Finalized segment (if any), ascending. + if let Some(mut finalized_stream) = finalized_stream { + while let Some(stream_item) = finalized_stream.next().await { + if channel_sender.send(stream_item).await.is_err() { + return; + } + } + } + + // 2) Nonfinalized segment, ascending. + let nonfinalized_start_height = + types::Height(std::cmp::max(start_height.0, lowest_nonfinalized_height.0)); + + for height_value in nonfinalized_start_height.0..=capped_end_height.0 { + let Some(indexed_block) = nonfinalized_snapshot + .get_chainblock_by_height(&types::Height(height_value)) + else { + match compact_block_from_source( + &source, + network.clone(), + types::Height(height_value), + &pool_types_for_node, + ) + .await + { + Ok(Some(compact_block)) => { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; } - Some(height) => { - // The VALIDATOR returned a block with a height. - // However, there is as of yet no guaranteed the Block is FINALIZED - if height <= *validator_finalized_height { - Ok(Some(( - types::BlockHash::from(block.hash()), - types::Height::from(height), - ))) - } else { - // non-finalized block - // no passthrough - Ok(None) + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } + }; + let compact_block = compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types_vector, + ); + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + } + // If the original end_height was above the tip, signal out_of_range after all valid blocks. + if needs_out_of_range { + let _ = channel_sender + .send(Err(tonic::Status::out_of_range(format!( + "Error: Height out of range [{}]. Height requested is greater than the best chain tip [{}].", + end_height.0, chain_tip_height.0, + )))) + .await; + } + } else { + // 1) Nonfinalized segment, descending. + if start_height >= lowest_nonfinalized_height { + let nonfinalized_end_height = + types::Height(std::cmp::max(end_height.0, lowest_nonfinalized_height.0)); + + for height_value in (nonfinalized_end_height.0..=start_height.0).rev() { + let Some(indexed_block) = nonfinalized_snapshot + .get_chainblock_by_height(&types::Height(height_value)) + else { + match compact_block_from_source( + &source, + network.clone(), + types::Height(height_value), + &pool_types_for_node, + ) + .await + { + Ok(Some(compact_block)) => { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; + } + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; } } + }; + let compact_block = compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types_vector, + ); + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; } } + } - Ok(None) => { - // At this point, we know that - // the block is NOT FINALIZED in the VALIDATOR. - // Return Ok(None) = no block found. - Ok(None) + // 2) Finalized segment (if any), descending. + if let Some(mut finalized_stream) = finalized_stream { + while let Some(stream_item) = finalized_stream.next().await { + if channel_sender.send(stream_item).await.is_err() { + return; + } } - Err(e) => Err(ChainIndexError::backing_validator(e)), } } - } + }); + + Ok(Some(CompactBlockStream::new(channel_receiver))) } - /// Returns the block commitment tree data by hash. - async fn get_treestate( + async fn z_get_block( &self, - hash: &types::BlockHash, - ) -> Result<(Option>, Option>), Self::Error> { + hash_or_height: String, + verbosity: Option, + ) -> Result { + // Resolve tip-relative negative heights against the best chaintip, + // matching zebra's own `getblock` semantics (`-1` is the tip). A + // rejected identifier carries zcashd's legacy InvalidParameter code + // as a typed `RpcError` source, which the serve layer recovers by + // downcast-walking the error chain. let snapshot = self.snapshot_nonfinalized_state().await?; - if !self.block_hash_known_for_treestate(&snapshot, hash).await? { - return Err(ChainIndexError::internal(format!( - "block hash {hash} not found in local chain index" - ))); - } - - match self.source().get_treestate(*hash).await { - Ok(resp) => Ok(resp), - Err(e) => Err(ChainIndexError { - kind: ChainIndexErrorKind::InternalServerError, - message: "failed to fetch treestate from validator".to_string(), - source: Some(Box::new(e)), - }), - } + let tip = self.best_chaintip(&snapshot).await?; + let id = HashOrHeight::new(&hash_or_height, Some(tip.height.into())).map_err(|error| { + ChainIndexError::internal_from( + zaino_fetch::jsonrpsee::connector::RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + error, + ), + ) + })?; + self.source() + .get_block_verbose(id, verbosity) + .await + .map_err(ChainIndexError::backing_validator) } - /// Gets the subtree roots of a given pool and the end heights of each root, - /// starting at the provided index, up to an optional maximum number of roots. - async fn get_subtree_roots( + async fn get_block_header( &self, - pool: ShieldedPool, - start_index: u16, - max_entries: Option, - ) -> Result, Self::Error> { + hash: String, + verbose: bool, + ) -> Result { self.source() - .get_subtree_roots(pool, start_index, max_entries) + .get_block_header(hash, verbose) .await .map_err(ChainIndexError::backing_validator) } - // ********** Transparent address history methods ********** + // TODO(internal-first): `getblockdeltas` is buildable from the indexed chainblocks + // + finalised/non-finalised prevout resolution. Build it internally by default once + // an internal prevout resolver (spanning non-finalised + finalised, reconstructing + // addresses from `TxOutCompact`) exists, keeping this source call as the fallback. + async fn get_block_deltas(&self, hash: String) -> Result { + self.source() + .get_block_deltas(hash) + .await + .map_err(ChainIndexError::backing_validator) + } - /// Returns all changes for the given transparent addresses. - async fn get_address_deltas( - &self, - params: GetAddressDeltasParams, - ) -> Result { + // `getdifficulty` is the difficulty-adjusted expected difficulty (over a block + // window), not the tip block's stored bits, so it cannot be built from indexed data: + // always delegate to the backing validator. + async fn get_difficulty(&self) -> Result { self.source() - .get_address_deltas(params) + .get_difficulty() .await .map_err(ChainIndexError::backing_validator) } - /// Returns the total transparent balance for the given addresses. - async fn get_address_balance( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result { + async fn get_info(&self) -> Result { self.source() - .get_address_balance(address_strings) + .get_info() .await .map_err(ChainIndexError::backing_validator) } - /// Returns the transaction ids made by the given transparent addresses. - async fn get_address_txids( - &self, - request: GetAddressTxIdsRequest, - ) -> Result, Self::Error> { + // `getblockchaininfo` needs cumulative pool value balances (TipPoolValues) and on-disk + // size, which are not in the ChainIndex's indexed data, so it cannot be built + // internally: always delegate to the backing validator. + async fn get_blockchain_info(&self) -> Result { self.source() - .get_address_txids(request) + .get_blockchain_info() .await .map_err(ChainIndexError::backing_validator) } - /// Returns all unspent transparent outputs for the given addresses. - async fn get_address_utxos( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result, Self::Error> { + async fn get_peer_info(&self) -> Result { self.source() - .get_address_utxos(address_strings) + .get_peer_info() .await .map_err(ChainIndexError::backing_validator) } - async fn get_outpoint_spenders( - &self, - snapshot: &Self::Snapshot, - outpoints: Vec, - scope: types::ChainScope, - ) -> Result>, Self::Error> { - use std::collections::HashMap; + async fn get_block_subsidy(&self, height: u32) -> Result { + self.source() + .get_block_subsidy(height) + .await + .map_err(ChainIndexError::backing_validator) + } - let mut result: Vec> = vec![None; outpoints.len()]; + async fn get_mining_info(&self) -> Result { + self.source() + .get_mining_info() + .await + .map_err(ChainIndexError::backing_validator) + } - // 1) Non-finalised best chain (FullChain scope only). Scan only the blocks reachable - // via `heights_to_hashes` (the `blocks` map also holds reorged-away blocks, which - // must not count). One pass builds an outpoint -> spending-txid map regardless of - // how many outpoints we look up. Under `Finalised` scope this is skipped so results - // are reorg-stable. - if let ( - types::ChainScope::FullChain, - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - }, - ) = (scope, snapshot) - { - let mut nfs_spenders: HashMap = HashMap::new(); - for hash in non_finalized_snapshot.heights_to_hashes.values() { - let Some(block) = non_finalized_snapshot.blocks.get(hash) else { - continue; - }; - for tx in block.transactions() { - let txid = *tx.txid(); - // `spent_outpoints` already skips coinbase null prevouts and builds each - // `Outpoint`, keeping the construction in one place (see #1332). - for outpoint in tx.transparent().spent_outpoints() { - nfs_spenders.insert(outpoint, txid); - } - } - } - for (i, outpoint) in outpoints.iter().enumerate() { - if let Some(txid) = nfs_spenders.get(outpoint) { - result[i] = Some(*txid); - } - } - } + async fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> Result { + self.source() + .get_tx_out(txid, n, include_mempool) + .await + .map_err(ChainIndexError::backing_validator) + } - // 2) Finalised lookup for the still-unresolved outpoints, batched into one DB call. - let unresolved_indices: Vec = result - .iter() - .enumerate() - .filter_map(|(i, r)| r.is_none().then_some(i)) - .collect(); - if unresolved_indices.is_empty() { - return Ok(result); - } - let unresolved_outpoints: Vec = - unresolved_indices.iter().map(|&i| outpoints[i]).collect(); - let locations = self - .finalized_state - .get_outpoint_spenders(unresolved_outpoints) - .await?; + async fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> Result { + self.source() + .get_spent_info(request) + .await + .map_err(ChainIndexError::backing_validator) + } - // 3) Resolve each finalised `TxLocation` to a txid. Dedup identical locations so a - // block spending several queried outpoints is only fetched once. `get_txid` is a - // single keyed lookup, far cheaper than reconstructing the whole block. - let mut slots_by_location: HashMap> = HashMap::new(); - for (slot, location) in unresolved_indices.into_iter().zip(locations) { - if let Some(location) = location { - slots_by_location.entry(location).or_default().push(slot); - } - } - for (location, slots) in slots_by_location { - let txid = self.finalized_state.get_txid(location).await?; - for slot in slots { - result[slot] = Some(txid); - } - } + async fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> Result { + self.source() + .get_network_sol_ps(blocks, height) + .await + .map_err(ChainIndexError::backing_validator) + } - Ok(result) + async fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> Result { + validate_raw_transaction_hex(&raw_transaction_hex) + .map_err(ChainIndexError::internal_from)?; + self.source() + .send_raw_transaction(raw_transaction_hex) + .await + .map_err(ChainIndexError::backing_validator) } - // ********** Metadata methods ********** + async fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> Result { + self.source() + .get_treestate_by_id(hash_or_height) + .await + .map_err(ChainIndexError::backing_validator) + } + + /// Returns all changes for the given transparent addresses. + async fn get_address_deltas( + &self, + params: GetAddressDeltasParams, + ) -> Result { + self.source() + .get_address_deltas(params) + .await + .map_err(ChainIndexError::backing_validator) + } /// Returns Information about the mempool state: /// - size: Current tx count @@ -2555,39 +2928,6 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result { - Ok(match snapshot { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => non_finalized_snapshot.best_tip, - - ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height, - } => { - BlockIndex { - height: *validator_finalized_height, - hash: self - .source() - // TODO: do something more efficient than getting the whole block - .get_block(HashOrHeight::Height((*validator_finalized_height).into())) - .await - .map_err(|e| { - ChainIndexError::database_hole( - validator_finalized_height, - Some(Box::new(e)), - ) - })? - .ok_or(ChainIndexError::database_hole( - validator_finalized_height, - None, - ))? - .hash() - .into(), - } - } - }) - } - async fn get_tx_out_set_info(&self) -> Result { use crate::chain_index::types::db::metadata::{ is_unspendable_tx_out, ZAINO_TXOUTSET_ENTRY_LEN, @@ -2772,16 +3112,39 @@ impl ChainIndex for NodeBackedChainIndexSubscriber zebra_chain::parameters::NetworkUpgrade { + match self { + ShieldedPool::Sapling => zebra_chain::parameters::NetworkUpgrade::Sapling, + ShieldedPool::Orchard => zebra_chain::parameters::NetworkUpgrade::Nu5, + ShieldedPool::Ironwood => zebra_chain::parameters::NetworkUpgrade::Nu6_3, + } + } + + /// [`ShieldedPool::activation_upgrade`] in `zcash_protocol` terms, for call sites + /// gated through [`zcash_protocol::consensus::Parameters`]. + pub(crate) fn zcash_protocol_activation_upgrade( + &self, + ) -> zcash_protocol::consensus::NetworkUpgrade { + match self { + ShieldedPool::Sapling => zcash_protocol::consensus::NetworkUpgrade::Sapling, + ShieldedPool::Orchard => zcash_protocol::consensus::NetworkUpgrade::Nu5, + ShieldedPool::Ironwood => zcash_protocol::consensus::NetworkUpgrade::Nu6_3, + } + } + /// Returns the string representative of the given pool. /// /// Used for display purposes and in converting the strongly types `PoolType` @@ -2790,6 +3153,7 @@ impl ShieldedPool { match self { ShieldedPool::Sapling => "sapling".to_string(), ShieldedPool::Orchard => "orchard".to_string(), + ShieldedPool::Ironwood => "ironwood".to_string(), } } } diff --git a/packages/zaino-state/src/chain_index/encoding.rs b/packages/zaino-state/src/chain_index/encoding.rs index 9101f31dc..4f3bba6de 100644 --- a/packages/zaino-state/src/chain_index/encoding.rs +++ b/packages/zaino-state/src/chain_index/encoding.rs @@ -273,16 +273,72 @@ pub trait ZainoVersionedSerde: Sized { } } -/// Defines the fixed encoded length of a database record. +/// Defines the fixed encoded length metadata for a versioned database record. +/// +/// This trait is intentionally version-aware. +/// +/// A type can be fixed-length in one historical wire version and variable-length +/// in a later wire version. For that reason, fixed length must not be represented +/// as a single latest constant. Instead, callers must ask for the fixed body +/// length of a specific encoded version. +/// +/// Lengths returned by this trait are body lengths only: they exclude the +/// leading `ZainoVersionedSerde` version tag byte. +/// +/// Returning `None` means that the requested version is either: +/// - unsupported by this type, +/// - not fixed-length, +/// - or must be decoded by a variable-length wrapper instead. pub trait FixedEncodedLen { - /// the fixed encoded length of a database record *not* incuding the version byte. - const ENCODED_LEN: usize; - - /// Length of version tag in bytes. + /// Length of the version tag in bytes. const VERSION_TAG_LEN: usize = 1; - /// the fixed encoded length of a database record *incuding* the version byte. - const VERSIONED_LEN: usize = Self::ENCODED_LEN + Self::VERSION_TAG_LEN; + /// Returns the fixed encoded body length for `version`, excluding the + /// version tag. + /// + /// Implementations must return the historical length for the exact on-disk + /// version requested. They must not return the current/latest struct length + /// for all versions. + fn encoded_len(version: u8) -> Option; + + /// Returns the fixed encoded length for `version`, including the version tag. + fn versioned_len(version: u8) -> Option { + Self::encoded_len(version).map(|len| len + Self::VERSION_TAG_LEN) + } + + /// Returns the fixed body length of the current/latest wire version. + /// + /// Returns `None` if the latest version is variable-length. + fn latest_encoded_len() -> io::Result + where + Self: ZainoVersionedSerde, + { + Self::encoded_len(Self::VERSION).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("no fixed len available for version tag {}", Self::VERSION), + ) + }) + } + + /// Returns the fixed encoded length of the current/latest wire version, + /// including the version tag. + /// + /// Returns `None` if the latest version is variable-length. + fn latest_versioned_len() -> io::Result + where + Self: ZainoVersionedSerde, + { + Self::versioned_len(Self::VERSION).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "no fixed versioned len available for version tag {}", + Self::VERSION + ), + ) + }) + } } /* ──────────────────────────── CompactSize helpers ────────────────────────────── */ diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 456df8ddf..1a1915e82 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -2,8 +2,8 @@ //! //! This module provides `FinalisedState`, the *finalised* portion of the chain index. //! -//! “Finalised” in this context means: All but the top 100 blocks in the blockchain. This follows -//! Zebra's model where a reorg of depth greater than 100 would require a complete network restart. +//! “Finalised” in this context means: All but the top `OPERATIONAL_NFS_DEPTH` blocks in the blockchain. This +//! follows Zebra's model where a reorg deeper than `MAX_BLOCK_REORG_HEIGHT` would require a complete network restart. //! //! `FinalisedState` is a facade over a `FinalisedSource` — the //! backing implementation that actually serves finalised data. That backing is **not necessarily a @@ -243,6 +243,33 @@ use crate::{ use std::{sync::Arc, time::Duration}; use tokio::time::{interval, MissedTickBehavior}; +/// The activation heights of the three shielded pools whose data +/// [`build_indexed_block_from_source`] assembles, resolved once per run. +/// +/// Both the ingestion loop ([`capability::DbWrite::write_blocks_to_height`]) and the v1.2.1 → +/// v1.3.0 migration backfill need exactly this set of heights; resolving them in one place keeps +/// the two call sites from drifting apart. A `None` height means the pool's network upgrade has no +/// activation height on the given network. +struct PoolActivationHeights { + sapling: Option, + nu5: Option, + nu6_3: Option, +} + +impl PoolActivationHeights { + /// Resolves the Sapling, NU5 (Orchard), and NU6.3 (Ironwood) activation heights on + /// `zebra_network`. + fn resolve(zebra_network: &zebra_chain::parameters::Network) -> Self { + let activation_height = + |pool: super::ShieldedPool| pool.activation_upgrade().activation_height(zebra_network); + Self { + sapling: activation_height(super::ShieldedPool::Sapling), + nu5: activation_height(super::ShieldedPool::Orchard), + nu6_3: activation_height(super::ShieldedPool::Ironwood), + } + } +} + /// Fetches the block at `height_int` from `source` and builds its [`IndexedBlock`], threading /// `parent_chainwork` into the block metadata. /// @@ -251,9 +278,10 @@ use tokio::time::{interval, MissedTickBehavior}; /// owns the loop. pub(crate) async fn build_indexed_block_from_source( source: &S, - network: zaino_common::Network, + network: zebra_chain::parameters::Network, sapling_activation_height: zebra_chain::block::Height, nu5_activation_height: Option, + nu6_3_activation_height: Option, height_int: u32, parent_chainwork: Option, ) -> Result { @@ -276,38 +304,42 @@ pub(crate) async fn build_indexed_block_from_source( let block_hash = BlockHash::from(block.hash().0); // Fetch sapling / orchard commitment tree data if above the relevant network upgrade. - let (sapling_opt, orchard_opt) = source.get_commitment_tree_roots(block_hash).await?; + let (sapling_opt, orchard_opt, ironwood_opt) = + source.get_commitment_tree_roots(block_hash).await?; + let is_sapling_active = height_int >= sapling_activation_height.0; let is_orchard_active = nu5_activation_height .is_some_and(|nu5_activation_height| height_int >= nu5_activation_height.0); - - let (sapling_root, sapling_size) = if is_sapling_active { - sapling_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Sapling commitment tree root for block {block_hash}"), - )) - })? - } else { - (zebra_chain::sapling::tree::Root::default(), 0) - }; - - let (orchard_root, orchard_size) = if is_orchard_active { - orchard_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Orchard commitment tree root for block {block_hash}"), - )) - })? - } else { - (zebra_chain::orchard::tree::Root::default(), 0) - }; + let is_ironwood_active = nu6_3_activation_height + .is_some_and(|nu6_3_activation_height| height_int >= nu6_3_activation_height.0); + + let (sapling_root, sapling_size) = required_pool_root( + super::ShieldedPool::Sapling, + is_sapling_active, + sapling_opt, + || format!("block {block_hash}"), + )?; + let (orchard_root, orchard_size) = required_pool_root( + super::ShieldedPool::Orchard, + is_orchard_active, + orchard_opt, + || format!("block {block_hash}"), + )?; + let ironwood = optional_pool_root( + super::ShieldedPool::Ironwood, + is_ironwood_active, + ironwood_opt, + || format!("block {block_hash}"), + )?; let metadata = BlockMetadata::new( sapling_root, - sapling_size as u32, + sapling_size, orchard_root, - orchard_size as u32, + orchard_size, + ironwood, parent_chainwork, - network.to_zebra_network(), + network, ); let block_with_metadata = BlockWithMetadata::new(block.as_ref(), metadata); @@ -318,6 +350,55 @@ pub(crate) async fn build_indexed_block_from_source( }) } +/// Resolves a pool's optional commitment-tree root and size against the pool's activation +/// status at the block in question. +/// +/// From activation onward the root is required: a missing root is an unrecoverable source +/// error. Below activation (or on a network where the pool's upgrade has no activation +/// height) the pool has no tree yet, so the root defaults and the size is zero. +/// +/// `location` describes the block for the error message (e.g. `block ` or +/// `block at height `) and is only evaluated on failure. +fn required_pool_root( + pool: super::ShieldedPool, + is_active: bool, + root_and_size: Option<(R, u64)>, + location: impl FnOnce() -> String, +) -> Result<(R, u32), FinalisedStateError> { + optional_pool_root(pool, is_active, root_and_size, location) + .map(|resolved| resolved.unwrap_or_else(|| (R::default(), 0))) +} + +/// Like [`required_pool_root`], but keeps the below-activation case as `None` instead of +/// defaulting, for pools whose storage distinguishes "no treestate yet" from a +/// default-valued one (the stored ironwood root: `None` = no ironwood data, the encoding +/// the v1.2.1->v1.3.0 migration produces for pre-activation heights). +fn optional_pool_root( + pool: super::ShieldedPool, + is_active: bool, + root_and_size: Option<(R, u64)>, + location: impl FnOnce() -> String, +) -> Result, FinalisedStateError> { + let pool = pool.pool_string(); + match root_and_size { + Some((root, size)) => { + let size = u32::try_from(size).map_err(|error| { + FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( + format!("{pool} commitment tree size does not fit into u32: {error}"), + )) + })?; + Ok(Some((root, size))) + } + None if !is_active => Ok(None), + None => Err(FinalisedStateError::BlockchainSourceError( + BlockchainSourceError::Unrecoverable(format!( + "missing {pool} commitment tree root for {}", + location() + )), + )), + } +} + use super::source::BlockchainSource; /// A sync wider than this many blocks runs in the background (with ephemeral @@ -410,7 +491,7 @@ impl FinalisedState { return Ok(Self { db: Arc::new(Router::new(Arc::new(FinalisedSource::ephemeral( source, - cfg.network.into(), + cfg.network.clone(), None, )))), cfg, @@ -489,12 +570,25 @@ impl FinalisedState { source: migration_source, }; - if let Err(error) = migration_manager.migrate().await { - tracing::error!("FinalisedState migration failed: {error}"); + match migration_manager.migrate().await { + Ok(()) => { + // Start the background validator only now that every migration has + // finished: its initial scan reads tables a migration populates (e.g. + // `commitment_tree_data_1_3_0`), so starting it earlier would race the + // migration and fail on a not-yet-written row. + migration_router.primary_backend().start_validator(); + } + Err(error) => { + tracing::error!("FinalisedState migration failed: {error}"); - migration_router.store_primary_status(StatusType::CriticalError); + migration_router.store_primary_status(StatusType::CriticalError); + } } }); + } else { + // No migration to run, so the on-disk tables the validator scans are already at the + // current schema: start it immediately. + router.primary_backend().start_validator(); } Ok(Self { db: router, cfg }) @@ -604,7 +698,7 @@ impl FinalisedState { /// - `Some(version)` if a compatible database directory is found, /// - `None` if no database is detected (fresh DB creation case). async fn try_find_current_db_version(cfg: &ChainIndexConfig) -> Option { - let legacy_dir = match cfg.network.to_zebra_network().kind() { + let legacy_dir = match cfg.network.kind() { NetworkKind::Mainnet => "live", NetworkKind::Testnet => "test", NetworkKind::Regtest => "local", @@ -614,7 +708,7 @@ impl FinalisedState { return Some(0); } - let net_dir = match cfg.network.to_zebra_network().kind() { + let net_dir = match cfg.network.kind() { NetworkKind::Mainnet => "mainnet", NetworkKind::Testnet => "testnet", NetworkKind::Regtest => "regtest", @@ -732,7 +826,7 @@ impl FinalisedState { let ephemeral_reference = router .init_or_take_ephemeral( source.clone(), - cfg.network.to_zebra_network(), + cfg.network.clone(), EphemeralMode::ReadOnly, db_height_opt, ) @@ -999,6 +1093,14 @@ impl FinalisedState { migration_manager.migrate().await?; } + // This test helper builds a fixture at an arbitrary (often intermediate) version for + // inspection, so it deliberately does NOT start the validator: the validator only validates + // against the current schema and would fail on an intermediate-version database. The + // foreground migration is already complete, so mark the primary `Ready` directly (as + // `spawn_v1_0_0` does) to give callers a settled backend. Validation is exercised through + // the production `FinalisedState::spawn` path, which always targets the current schema. + router.store_primary_status(StatusType::Ready); + let metadata = router.get_metadata().await?; if metadata.version() != target_version { return Err(FinalisedStateError::Custom(format!( @@ -1030,7 +1132,6 @@ impl FinalisedState { source: T, ) -> Result, FinalisedStateError> { let db = FinalisedSource::spawn_v1_0_0(cfg).await?; - db.wait_until_ready().await; let tip = source.get_best_block_height().await?.ok_or_else(|| { FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( @@ -1039,6 +1140,13 @@ impl FinalisedState { })?; let tip = Height::from(tip); + // Ironwood (NU6.3) commitment tree data is only expected from activation. Below activation + // (or on a network with no NU6.3 activation height) the source has no ironwood root, so it + // defaults — mirroring `build_indexed_block_from_source`. + let nu6_3_activation_height = super::ShieldedPool::Ironwood + .activation_upgrade() + .activation_height(&cfg.network); + let mut parent_chainwork: Option = None; for height in crate::chain_index::types::GENESIS_HEIGHT.0..=tip.0 { @@ -1056,25 +1164,35 @@ impl FinalisedState { })?; let block_hash = BlockHash::from(block.hash().0); - let (sapling_opt, orchard_opt) = source.get_commitment_tree_roots(block_hash).await?; - let (sapling_root, sapling_size) = sapling_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Sapling commitment tree root for block {block_hash}"), - )) - })?; - let (orchard_root, orchard_size) = orchard_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Orchard commitment tree root for block {block_hash}"), - )) - })?; + let (sapling_opt, orchard_opt, ironwood_opt) = + source.get_commitment_tree_roots(block_hash).await?; + // Per this builder's contract, the fixture source provides Sapling and Orchard + // roots for every block, so those pools are unconditionally required. + let (sapling_root, sapling_size) = + required_pool_root(super::ShieldedPool::Sapling, true, sapling_opt, || { + format!("block {block_hash}") + })?; + let (orchard_root, orchard_size) = + required_pool_root(super::ShieldedPool::Orchard, true, orchard_opt, || { + format!("block {block_hash}") + })?; + let is_ironwood_active = + nu6_3_activation_height.is_some_and(|activation| height >= activation.0); + let ironwood = optional_pool_root( + super::ShieldedPool::Ironwood, + is_ironwood_active, + ironwood_opt, + || format!("block {block_hash}"), + )?; let metadata = BlockMetadata::new( sapling_root, - sapling_size as u32, + sapling_size, orchard_root, - orchard_size as u32, + orchard_size, + ironwood, parent_chainwork, - cfg.network.to_zebra_network(), + cfg.network.clone(), ); let block_with_metadata = BlockWithMetadata::new(block.as_ref(), metadata); let chain_block = IndexedBlock::try_from(block_with_metadata).map_err(|_| { diff --git a/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md b/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md index 78e0e92f2..52a137a74 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md +++ b/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md @@ -395,6 +395,88 @@ Bug Fixes / Optimisations version migrations run in the background while an ephemeral passthrough serves finalised reads from the backing source. +-------------------------------------------------------------------------------- +DB VERSION v1.3.0 (from v1.2.1) +Date: 2026-07-03 +-------------------------------------------------------------------------------- + +Summary +- Persist and serve the Ironwood (NU6.3) shielded pool from the finalised state. + Ironwood actions reuse the Orchard compact types (`OrchardCompactTx` / + `CompactOrchardAction`), matching zebra, which exposes ironwood actions as + `orchard::Action`. +- Add a new per-height `ironwood` table (`ironwood_1_3_0`) storing + `StoredEntryVar`, written for every block from v1.3.0 onward. +- Rebuild the `commitment_tree_data` table from the legacy fixed-length + `StoredEntryFixed` (V1) layout into the variable-length + `StoredEntryVar` (V2) layout, which carries the optional + Ironwood commitment-tree root and the Ironwood tree size. The rebuild moves the + data into a new table (`commitment_tree_data_1_3_0`) and drops the old one. + +On-disk schema +- Layout: + - No directory layout changes (still `//v1/`). +- Tables: + - Added: `ironwood` — per-height ironwood compact tx data + (LMDB database name: `ironwood_1_3_0`, `StoredEntryVar`). + - Renamed/rebuilt: `commitment_tree_data` moves from the fixed-length table + `commitment_tree_data_1_0_0` (`StoredEntryFixed`, V1) to + the variable-length table `commitment_tree_data_1_3_0` + (`StoredEntryVar`, V2). The legacy table is cleared once + the rebuild completes. + - Removed: legacy `commitment_tree_data_1_0_0` contents (table cleared). +- Encoding: + - Values: `CommitmentTreeData` gains a V2 body — `CommitmentTreeRoots` adds an + `Option<32-byte ironwood_root>` and `CommitmentTreeSizes` adds + `LE(u32) ironwood_total`. Because the optional root makes the record + variable-length, the table wrapper changes from `StoredEntryFixed` to + `StoredEntryVar`. + - Checksums / validation: `DB_SCHEMA_V1_HASH` updated to match the revised + schema text. Ironwood rows are checksum-protected by their height key. +- Invariants: + - Blocks written from v1.3.0 have an `ironwood` row aligned to the block's tx + list. Blocks written before v1.3.0 (all below NU6.3 activation) have no + ironwood row; readers and startup validation treat a missing row as + "no ironwood data at this height". + +API / capabilities +- Capability changes: + - No new capability bit: ironwood reads reuse `BLOCK_SHIELDED_EXT` and ironwood + compact-block/commitment data reuse `COMPACT_BLOCK_EXT`. +- Public surface changes: + - Added `BlockShieldedExt::{get_ironwood, get_block_ironwood, + get_block_range_ironwood}` (and the `DbReader` wrappers), mirroring the + orchard accessors. + - Compact blocks now populate `CompactTx.ironwood_actions` and + `ChainMetadata.ironwood_commitment_tree_size`; `IndexedBlock` materialisation + returns real ironwood tx data. + +Migration +- Strategy: in-place rebuild from existing on-disk data (no validator refetch), + single re-entrant step. +- Guard: refuses to run if the DB tip is at or above NU6.3 activation (such a + database was synced without ironwood data and must be re-indexed from the + validator). Below activation every stored block predates ironwood, so the + rebuild sets ironwood root/size to their defaults. +- Backfill: rebuilds each height's commitment row from the legacy fixed-length + row into the new `StoredEntryVar` (V2) table, then clears the legacy table. The + new `ironwood` table starts empty (no pre-NU6.3 block has ironwood data). +- Completion criteria: all heights through the tip rebuilt; `DbMetadata.version` + advanced to v1.3.0 with the refreshed schema hash; the temporary progress key + (`_migration_commitment_tree_data_progress_1_3_0_next_height`) removed. +- Failure handling: resumes from the temporary progress height; each rebuilt row + and the progress watermark commit together, and re-seen rows are accepted after + `NO_OVERWRITE` + byte-match verification. + +Bug Fixes / Optimisations +- Fixed `TreeRootData::extract_with_defaults` reading the orchard tree where it + should read ironwood, and `extract_ironwood_data` using the orchard value + balance amount instead of the ironwood amount. +- `to_compact_block`, the compact-block assembly, and the compact-block stream no + longer hardcode `ironwood_commitment_tree_size: 0` / empty ironwood actions. +- Compact-tx "omit empty transaction" filters now consider ironwood actions. +- The Fetch backend's `z_get_treestate` no longer discards the ironwood treestate. + -------------------------------------------------------------------------------- v0 SCHEMA SUPPORT REMOVED (no DB version change) Date: 2026-06-18 diff --git a/packages/zaino-state/src/chain_index/finalised_state/capability.rs b/packages/zaino-state/src/chain_index/finalised_state/capability.rs index 0193c5d69..c26b4ec8d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/capability.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/capability.rs @@ -372,12 +372,18 @@ impl ZainoVersionedSerde for DbMetadata { } } -/// `DbMetadata` has a fixed encoded body length. +/// Fixed-length encoding metadata for `DbMetadata`. /// +/// v1 consists of: /// Body length = `DbVersion::VERSIONED_LEN` (12 + 1) + 32-byte schema hash /// + `MigrationStatus::VERSIONED_LEN` (1 + 1) = 47 bytes. impl FixedEncodedLen for DbMetadata { - const ENCODED_LEN: usize = DbVersion::VERSIONED_LEN + 32 + MigrationStatus::VERSIONED_LEN; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(47), + _ => None, + } + } } /// Human-readable summary for logs. @@ -535,9 +541,16 @@ impl ZainoVersionedSerde for DbVersion { } } -// DbVersion: body = 3*(4-byte u32) - 12 bytes +/// Fixed-length encoding metadata for `DbVersion`. +/// +/// v1 consists of *(4-byte u32) = 12 bytes impl FixedEncodedLen for DbVersion { - const ENCODED_LEN: usize = 4 + 4 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(12), + _ => None, + } + } } /// Formats as `{major}.{minor}.{patch}` for logs and diagnostics. @@ -636,9 +649,16 @@ impl ZainoVersionedSerde for MigrationStatus { } } -/// `MigrationStatus` has a fixed 1-byte encoded body (discriminator). +/// Fixed-length encoding metadata for `MigrationStatus`. +/// +/// v1 consists of a single byte impl FixedEncodedLen for MigrationStatus { - const ENCODED_LEN: usize = 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(1), + _ => None, + } + } } // ***** Core Database functionality ***** @@ -898,6 +918,32 @@ pub trait BlockShieldedExt: Send + Sync { end: Height, ) -> impl SendFut, FinalisedStateError>>; + /// Fetch the serialized Ironwood (NU6.3) compact tx for the given TxLocation, if present. + /// + /// Ironwood actions are modelled with the Orchard compact types. Returns `None` when the block + /// has no ironwood row (any block below NU6.3 activation, or written before schema v1.3.0). + fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> impl SendFut, FinalisedStateError>>; + + /// Fetch block ironwood transaction data by height. + /// + /// Returns an empty [`OrchardTxList`] when the block has no ironwood row. + fn get_block_ironwood( + &self, + height: Height, + ) -> impl SendFut>; + + /// Fetches block ironwood tx data for the given (inclusive) height range. + /// + /// Heights with no ironwood row yield an empty [`OrchardTxList`]. + fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> impl SendFut, FinalisedStateError>>; + /// Fetch block commitment tree data by height. fn get_block_commitment_tree_data( &self, diff --git a/packages/zaino-state/src/chain_index/finalised_state/entry.rs b/packages/zaino-state/src/chain_index/finalised_state/entry.rs index 9877c80a5..f12b5d8f6 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/entry.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/entry.rs @@ -66,7 +66,8 @@ //! maintain a decode path (or bump the DB major version and migrate). use crate::{ - read_fixed_le, version, write_fixed_le, CompactSize, FixedEncodedLen, ZainoVersionedSerde, + read_fixed_le, read_u8, version, write_fixed_le, CompactSize, FixedEncodedLen, + ZainoVersionedSerde, }; use blake2::{ @@ -108,93 +109,83 @@ pub(crate) struct StoredEntryFixed { } impl StoredEntryFixed { - /// Constructs a new checksummed entry for `item` under `key`. + /// Constructs a new checksummed fixed-length entry for `item` under `key`. /// - /// The checksum is computed as: - /// `blake2b256(encoded_key || item.serialize())`. + /// The checksum is computed over: /// - /// # Key requirements - /// `key` must be the exact byte encoding used as the LMDB key for this record. If the caller - /// hashes a different key encoding than what is used for storage, verification will fail. + /// `encoded_key || item.serialize()` + /// + /// This constructor may only be used when the latest version of `T` is still + /// fixed-length. If the latest version of `T` has become variable-length, + /// callers must use the variable-length stored-entry wrapper for new writes. + /// + /// # Panics + /// + /// Panics if `T::serialize()` fails. Existing code already treated this path + /// as infallible. If desired, this can be made fallible later without changing + /// the on-disk format. pub(crate) fn new>(key: K, item: T) -> Self { let body = { - let mut v = Vec::with_capacity(T::VERSIONED_LEN); + let len = T::latest_versioned_len().unwrap_or(0); + let mut v = Vec::with_capacity(len); item.serialize(&mut v).unwrap(); v }; + let checksum = Self::blake2b256(&[key.as_ref(), &body].concat()); Self { item, checksum } } /// Verifies the checksum for this entry under `key`. /// - /// Returns `true` if and only if: - /// `self.checksum == blake2b256(encoded_key || item.serialize())`. - /// - /// # Key requirements - /// `key` must be the exact byte encoding used as the LMDB key for this record. - /// - /// # Usage - /// Callers should treat a checksum mismatch as a corruption or incompatibility signal and - /// return a hard error (or trigger a rebuild path), depending on context. + /// Verification tries every supported historical encoding from latest down + /// to v1. This is required because the decoded in-memory item no longer + /// remembers which exact version produced the stored checksum. pub(crate) fn verify>(&self, key: K) -> bool { - // Iterate from latest (T::VERSION) down to 1 (inclusive). let mut v = T::VERSION; + loop { - // Try to obtain the encoded bytes for this candidate version (tag + body). - match self.item.to_bytes_with_version(v) { - Ok(item_bytes) => { - // Compute the candidate checksum over (encoded_key || item_bytes). - let candidate = Self::blake2b256(&[key.as_ref(), &item_bytes].concat()); - if candidate == self.checksum { - return true; - } - } - Err(_) => { - // This version not supported by the type's encoder; try older version. + if let Ok(item_bytes) = self.item.to_bytes_with_version(v) { + let candidate = Self::blake2b256(&[key.as_ref(), &item_bytes].concat()); + if candidate == self.checksum { + return true; } } if v == 1 { break; } + v = v.saturating_sub(1); } false } - /// Returns a reference to the inner record. + /// Returns a reference to the inner decoded record. pub(crate) fn inner(&self) -> &T { &self.item } /// Computes a BLAKE2b-256 checksum over `data`. - /// - /// This is the hashing primitive used by both wrappers. The checksum is not keyed. pub(crate) fn blake2b256(data: &[u8]) -> [u8; 32] { let mut hasher = Blake2bVar::new(32).expect("Failed to create hasher"); hasher.update(data); + let mut output = [0u8; 32]; hasher .finalize_variable(&mut output) .expect("Failed to finalize hash"); + output } /// Builds serialized stored-entry bytes using a specific inner item version. /// - /// This is required when writing historical database values whose inner item must be encoded - /// with an older version than `T::VERSION`. - /// - /// The returned bytes are: - /// - StoredEntryFixed version tag - /// - inner item bytes encoded with `item_version` - /// - checksum over `encoded_key || inner_item_bytes` + /// This is used for tests and historical fixture construction. /// - /// This method returns serialized bytes directly because `StoredEntryFixed` does not store - /// the inner item version. Constructing `Self` alone would lose the requested item version - /// before the value is written. + /// The selected `item_version` must be fixed-length according to + /// `T::versioned_len(item_version)`. #[cfg(test)] pub(crate) fn to_bytes_with_item_version>( key: K, @@ -203,7 +194,14 @@ impl StoredEntryFixed { ) -> io::Result> { let item_bytes = item.to_bytes_with_version(item_version)?; - if item_bytes.len() != T::VERSIONED_LEN { + let expected_len = T::versioned_len(item_version).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "requested item version is not fixed-length", + ) + })?; + + if item_bytes.len() != expected_len { return Err(io::Error::new( io::ErrorKind::InvalidData, "encoded fixed-length item has an unexpected length", @@ -212,7 +210,7 @@ impl StoredEntryFixed { let checksum = Self::blake2b256(&[key.as_ref(), &item_bytes].concat()); - let mut stored_entry_bytes = Vec::with_capacity(1 + T::VERSIONED_LEN + 32); + let mut stored_entry_bytes = Vec::with_capacity(1 + expected_len + 32); stored_entry_bytes.push(Self::VERSION); stored_entry_bytes.extend_from_slice(&item_bytes); stored_entry_bytes.extend_from_slice(&checksum); @@ -221,44 +219,94 @@ impl StoredEntryFixed { } } -/// Versioned on-disk encoding for fixed-length checksummed entries. -/// -/// Body layout (after the `StoredEntryFixed` version tag): -/// 1. `T::serialize()` bytes (fixed length: `T::VERSIONED_LEN`) -/// 2. 32-byte checksum -/// -/// Note: `T::serialize()` includes `T`’s own version tag and body. impl ZainoVersionedSerde for StoredEntryFixed { const VERSION: u8 = version::V1; + /// Encodes the latest fixed stored-entry layout. fn encode_latest(&self, w: &mut W) -> io::Result<()> { Self::encode_v1(self, w) } + /// Decodes the latest fixed stored-entry layout. fn decode_latest(r: &mut R) -> io::Result { Self::decode_v1(r) } + /// Encodes a fixed stored entry. + /// + /// The inner item is serialized with its own latest version tag and body, + /// followed by the 32-byte checksum. + /// + /// This method requires the latest version of `T` to still be fixed-length. fn encode_v1(&self, w: &mut W) -> io::Result<()> { - self.item.serialize(&mut *w)?; + let expected_len = T::latest_versioned_len().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "latest item version is not fixed-length", + ) + })?; + + let mut item_bytes = Vec::with_capacity(expected_len); + self.item.serialize(&mut item_bytes)?; + + if item_bytes.len() != expected_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "encoded fixed-length item has an unexpected length", + )); + } + + w.write_all(&item_bytes)?; write_fixed_le::<32, _>(&mut *w, &self.checksum) } + /// Decodes a fixed stored entry. + /// + /// This method reads the inner item version tag first, then uses + /// `T::encoded_len(version)` to determine how many body bytes to read. + /// + /// This is what keeps old fixed-length rows readable after the latest `T` + /// changes size or becomes variable-length. fn decode_v1(r: &mut R) -> io::Result { - let mut body = vec![0u8; T::VERSIONED_LEN]; - r.read_exact(&mut body)?; - let item = T::deserialize(&body[..])?; + let item_version = read_u8(&mut *r)?; + + let body_len = T::encoded_len(item_version).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "stored item version is not fixed-length or is unsupported", + ) + })?; + let mut item_bytes = Vec::with_capacity(1 + body_len); + item_bytes.push(item_version); + item_bytes.resize(1 + body_len, 0); + + r.read_exact(&mut item_bytes[1..])?; + + let item = T::deserialize(&item_bytes[..])?; let checksum = read_fixed_le::<32, _>(r)?; + Ok(Self { item, checksum }) } } -/// `StoredEntryFixed` has a fixed encoded body length. -/// -/// Body length = `T::VERSIONED_LEN` + 32 bytes checksum. +/// Fixed stored entries are only fixed-length when the latest inner item version +/// is fixed-length. impl FixedEncodedLen for StoredEntryFixed { - const ENCODED_LEN: usize = T::VERSIONED_LEN + 32; + /// Returns the fixed encoded body length for a stored fixed entry. + /// + /// `StoredEntryFixed` v1 consists of: + /// - the latest fixed-length versioned encoding of the inner item `T` + /// - a 32-byte checksum + /// + /// If the latest version of `T` is not fixed-length, this returns `None` + /// because `StoredEntryFixed` cannot have a fixed latest encoded length. + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => T::latest_versioned_len().ok().map(|item_len| item_len + 32), + _ => None, + } + } } /// Variable-length checksummed database value wrapper. @@ -439,105 +487,177 @@ mod tests { pub x: u32, } - // TestInner: versioned type with two encodings: - // - v1: x as little-endian (body only) - // - v2: x as big-endian (body only) and v2 is the current version impl ZainoVersionedSerde for TestInner { const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { self.encode_v2(w) } + fn decode_latest(r: &mut R) -> io::Result { Self::decode_v2(r) } + /// v1 body: + /// - 4 bytes: `x` as little-endian `u32` fn encode_v1(&self, w: &mut W) -> io::Result<()> { write_u32_le(w, self.x) } + + /// v2 body: + /// - 4 bytes: `x` as big-endian `u32` + /// - 4 bytes: duplicate/check field as big-endian `u32` + /// + /// This intentionally makes v2 a different fixed length from v1 so the + /// `StoredEntryFixed` decoder must use the version tag to select the + /// correct body length. fn encode_v2(&self, w: &mut W) -> io::Result<()> { - write_u32_be(w, self.x) + write_u32_be(&mut *w, self.x)?; + write_u32_be(&mut *w, !self.x) } fn decode_v1(r: &mut R) -> io::Result { let x = read_u32_le(r)?; Ok(TestInner { x }) } + fn decode_v2(r: &mut R) -> io::Result { - let x = read_u32_be(r)?; + let x = read_u32_be(&mut *r)?; + let check = read_u32_be(&mut *r)?; + + if check != !x { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid TestInner v2 check field", + )); + } + Ok(TestInner { x }) } } - // Make TestInner fixed-length for StoredEntryFixed tests. impl FixedEncodedLen for TestInner { - // body length (no version tag): 4 bytes (u32) - const ENCODED_LEN: usize = 4; + /// Historical fixed body lengths, excluding the version tag. + /// + /// v1 is 4 bytes. + /// v2 is 8 bytes. + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + version::V2 => Some(8), + _ => None, + } + } } - // Helper: simple key bytes for tests fn key_bytes() -> Vec { b"test-key".to_vec() } - // StoredEntryFixed: latest roundtrip (new -> to_bytes -> from_bytes -> verify) + #[test] + fn test_inner_versions_have_different_lengths() { + let inner = TestInner { x: 0x1122_3344 }; + + let v1 = inner + .to_bytes_with_version(version::V1) + .expect("inner v1 bytes"); + let v2 = inner + .to_bytes_with_version(version::V2) + .expect("inner v2 bytes"); + + assert_eq!(v1.len(), 1 + 4); + assert_eq!(v2.len(), 1 + 8); + assert_ne!(v1.len(), v2.len()); + + assert_eq!(TestInner::versioned_len(version::V1), Some(1 + 4)); + assert_eq!(TestInner::versioned_len(version::V2), Some(1 + 8)); + } + #[test] fn stored_entry_fixed_roundtrip_latest() { let inner = TestInner { x: 0x1122_3344 }; let key = key_bytes(); - // Construct wrapper using current serializer (StoredEntryFixed::new uses latest encoding) let wrapper = StoredEntryFixed::new(&key, inner.clone()); - // Verify should succeed for the same key - assert!( - wrapper.verify(&key), - "wrapper verify (latest) should succeed" - ); + assert!(wrapper.verify(&key), "wrapper verify latest should succeed"); - // Encode wrapper to bytes and decode back let bytes = wrapper.to_bytes().expect("wrapper to_bytes"); let parsed = StoredEntryFixed::::from_bytes(&bytes).expect("from_bytes"); + assert_eq!(parsed.item, inner); assert_eq!(parsed.checksum, wrapper.checksum); - - // parsed wrapper should verify with same key assert!(parsed.verify(&key)); } #[test] - // StoredEntryFixed: historical v1 body present on disk -> from_bytes + verify must succeed - fn stored_entry_fixed_verify_old_v1() { + fn stored_entry_fixed_decodes_historical_v1_with_shorter_length() { let inner = TestInner { x: 0xAABB_CCDD }; let key = key_bytes(); - // Produce item bytes according to historical v1 (tag + body) let item_bytes_v1 = inner .to_bytes_with_version(version::V1) .expect("inner v1 bytes"); - // Compute checksum over (key || item_bytes_v1) + assert_eq!( + item_bytes_v1.len(), + TestInner::versioned_len(version::V1).unwrap() + ); + assert_ne!( + item_bytes_v1.len(), + TestInner::latest_versioned_len().unwrap() + ); + let mut digest_input = Vec::with_capacity(key.len() + item_bytes_v1.len()); digest_input.extend_from_slice(&key); digest_input.extend_from_slice(&item_bytes_v1); + let checksum = StoredEntryFixed::::blake2b256(&digest_input); - // Manually build on-disk raw bytes for StoredEntryFixed: - // [StoredEntryFixed::VERSION] + item_bytes_v1 (should have length T::VERSIONED_LEN) + checksum let mut raw = Vec::new(); raw.push(StoredEntryFixed::::VERSION); raw.extend_from_slice(&item_bytes_v1); raw.extend_from_slice(&checksum); - // Parse using from_bytes (which will call decode_v1 and reconstruct wrapper) - let parsed = StoredEntryFixed::::from_bytes(&raw).expect("from_bytes v1"); - // parsed.item should equal the decoded inner + let parsed = StoredEntryFixed::::from_bytes(&raw) + .expect("fixed wrapper should decode historical shorter v1 item"); + assert_eq!(parsed.item, inner); - // verify should succeed using the same key (it will try v2 then v1 candidate encodings) + assert_eq!(parsed.checksum, checksum); + assert!(parsed.verify(&key)); + } + + #[test] + fn stored_entry_fixed_decodes_latest_v2_with_longer_length() { + let inner = TestInner { x: 0x5566_7788 }; + let key = key_bytes(); + + let item_bytes_v2 = inner + .to_bytes_with_version(version::V2) + .expect("inner v2 bytes"); + + assert_eq!( + item_bytes_v2.len(), + TestInner::versioned_len(version::V2).unwrap() + ); + + let checksum = StoredEntryFixed::::blake2b256( + &[key.as_slice(), item_bytes_v2.as_slice()].concat(), + ); + + let mut raw = Vec::new(); + raw.push(StoredEntryFixed::::VERSION); + raw.extend_from_slice(&item_bytes_v2); + raw.extend_from_slice(&checksum); + + let parsed = StoredEntryFixed::::from_bytes(&raw) + .expect("fixed wrapper should decode latest longer v2 item"); + + assert_eq!(parsed.item, inner); + assert_eq!(parsed.checksum, checksum); assert!(parsed.verify(&key)); } - // StoredEntryFixed: verify fails on tampered checksum or key #[test] fn stored_entry_fixed_verify_tamper() { let inner = TestInner { x: 0x0102_0304 }; @@ -546,16 +666,15 @@ mod tests { let mut wrapper = StoredEntryFixed::new(&key, inner.clone()); assert!(wrapper.verify(&key)); - // Tamper checksum (flip a byte) and ensure verify fails wrapper.checksum[0] ^= 0xff; assert!( !wrapper.verify(&key), "verify should fail with tampered checksum" ); - // Restore checksum and verify ok, then check wrong key fails wrapper = StoredEntryFixed::new(&key, inner.clone()); assert!(wrapper.verify(&key)); + let wrong_key = b"other-key".to_vec(); assert!( !wrapper.verify(&wrong_key), @@ -563,22 +682,21 @@ mod tests { ); } - // -------------------- StoredEntryVar tests -------------------- - #[test] fn stored_entry_var_roundtrip_latest() { let inner = TestInner { x: 0x5566_7788 }; let key = key_bytes(); let wrapper = StoredEntryVar::new(&key, inner.clone()); + assert!( wrapper.verify(&key), - "var wrapper verify (latest) should succeed" + "var wrapper verify latest should succeed" ); - // Encode wrapper to bytes and decode back via From/To bytes let bytes = wrapper.to_bytes().expect("var to_bytes"); let parsed = StoredEntryVar::::from_bytes(&bytes).expect("var from_bytes"); + assert_eq!(parsed.item, inner); assert_eq!(parsed.checksum, wrapper.checksum); assert!(parsed.verify(&key)); @@ -589,30 +707,26 @@ mod tests { let inner = TestInner { x: 0xDEAD_BEEF }; let key = key_bytes(); - // item serialized as v1 (tag + body) let item_bytes_v1 = inner .to_bytes_with_version(version::V1) .expect("inner v1 bytes"); - // checksum computed over (key || item_bytes_v1) let mut digest_input = Vec::with_capacity(key.len() + item_bytes_v1.len()); digest_input.extend_from_slice(&key); digest_input.extend_from_slice(&item_bytes_v1); + let checksum = StoredEntryVar::::blake2b256(&digest_input); - // Build raw stored value for StoredEntryVar: - // [StoredEntryVar::VERSION] + CompactSize(len) + item_bytes_v1 + checksum let mut raw = Vec::new(); raw.push(StoredEntryVar::::VERSION); CompactSize::write(&mut raw, item_bytes_v1.len()).expect("write compactsize"); raw.extend_from_slice(&item_bytes_v1); - // write checksum as fixed 32 bytes write_fixed_le::<32, _>(&mut raw, &checksum).expect("write checksum"); - // from_bytes should parse the body and return wrapper let parsed = StoredEntryVar::::from_bytes(&raw).expect("var from_bytes v1"); + assert_eq!(parsed.item, inner); - // verify must succeed using same key (it will try v2 then v1) + assert_eq!(parsed.checksum, checksum); assert!(parsed.verify(&key)); } @@ -624,13 +738,12 @@ mod tests { let mut wrapper = StoredEntryVar::new(&key, inner); assert!(wrapper.verify(&key)); - // tamper checksum wrapper.checksum[31] ^= 0xff; assert!(!wrapper.verify(&key)); - // restore and test wrong key fails let wrapper = StoredEntryVar::new(&key, TestInner { x: 0xCAFEBABE }); let wrong_key = b"bad-key".to_vec(); + assert!(!wrapper.verify(&wrong_key)); } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index 6697cf45d..d2aeb1cce 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -283,6 +283,18 @@ impl FinalisedSource { } } + /// Start the background validator on the primary v1 backend (no-op for the ephemeral + /// passthrough). + /// + /// Called by the orchestrator only once all pending migrations have completed, so the + /// validator never races a migration that populates the tables its initial scan reads. + pub(super) fn start_validator(&self) { + match self { + Self::V1(db) => db.start_validator(), + Self::Ephemeral(_) => {} + } + } + /// Stores a new runtime status in the concrete backend. /// /// This is used by router-level background orchestration, for example to report an asynchronous @@ -360,6 +372,20 @@ impl FinalisedSource { pub(crate) fn transparent_db(&self) -> Result { Ok(self.require_v1("v1 transparent db")?.transparent_db()) } + + /// Provides access to the (v1.3.0) `StoredEntryVar` commitment-tree-data table, required for + /// Migration1_2_1To1_3_0 to write the rebuilt commitment rows. + pub(crate) fn commitment_tree_data_db(&self) -> Result { + Ok(self + .require_v1("v1 commitment_tree_data db")? + .commitment_tree_data_db()) + } + + /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to + /// backfill ironwood rows from validator-fetched block data. + pub(super) fn ironwood_db(&self) -> Result { + Ok(self.require_v1("v1 ironwood db")?.ironwood_db()) + } } impl From for FinalisedSource { @@ -675,6 +701,34 @@ impl BlockShieldedExt for FinalisedSource { } } + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + match self { + Self::V1(db) => db.get_ironwood(tx_location).await, + Self::Ephemeral(db) => db.get_ironwood(tx_location).await, + } + } + + async fn get_block_ironwood(&self, h: Height) -> Result { + match self { + Self::V1(db) => db.get_block_ironwood(h).await, + Self::Ephemeral(db) => db.get_block_ironwood(h).await, + } + } + + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + match self { + Self::V1(db) => db.get_block_range_ironwood(start, end).await, + Self::Ephemeral(db) => db.get_block_range_ironwood(start, end).await, + } + } + async fn get_block_commitment_tree_data( &self, height: Height, diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt index 5c001e474..80f3f1d3c 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt @@ -1,5 +1,5 @@ # ──────────────────────────────────────────────────────────────────────────────── -# Zaino – Finalised State Database, on-disk layout (Schema v1.2, 2026-13-05) +# Zaino – Finalised State Database, on-disk layout (Schema v1.3, 2026-07-03) # ──────────────────────────────────────────────────────────────────────────────── # # Any change to this file is a **breaking** change. Bump the schema version, @@ -71,13 +71,33 @@ # actions = CS n + n × CompactOrchardAction # CompactOrchardAction = 32-byte nullifier 32-byte cmx 32-byte epk 52-byte ciphertext # -# 6. commitment_tree_data ― H -> StoredEntryFixed> +# 6. commitment_tree_data ― H -> StoredEntryVar # Key : BE height -# Val : 0x01 + CS n + n × CommitmentTreeData -# CommitmentTreeData V1 body = +# Val : 0x01 + CS len + CommitmentTreeData + [32] check_hash +# CommitmentTreeData V2 body = # CommitmentTreeRoots + CommitmentTreeSizes # CommitmentTreeRoots = 32-byte sapling_root 32-byte orchard_root +# Option<32-byte ironwood_root> # CommitmentTreeSizes = LE(u32) sapling_total LE(u32) orchard_total +# LE(u32) ironwood_total +# +# (Legacy CommitmentTreeData V1 body = +# 32-byte sapling_root 32-byte orchard_root +# LE(u32) sapling_total LE(u32) orchard_total ; +# fixed-length, previously wrapped in StoredEntryFixed. Rebuilt into the V2 +# StoredEntryVar layout above by the v1.2.1 → v1.3.0 migration, then dropped.) +# +# 13. ironwood ― H -> StoredEntryVar (schema v1.3.0) +# Key : BE height +# Val : 0x01 + CS tx_count + tx_count × Option +# OrchardCompactTx = +# Option value_balance +# actions = CS n + n × CompactOrchardAction +# CompactOrchardAction = 32-byte nullifier 32-byte cmx 32-byte epk 52-byte ciphertext +# +# Ironwood (NU6.3) actions reuse the Orchard compact types. Rows are written for +# every block from schema v1.3.0 onward; blocks written before v1.3.0 (all below +# NU6.3 activation) have no ironwood row, which readers treat as "no ironwood data". # # 7. heights ― B (block hash) -> StoredEntryFixed # Key : 32-byte block hash (internal byte order) @@ -132,7 +152,7 @@ # # ─────────────────────────── Environment settings ───────────────────────────── # LMDB page-size: platform default -# max_dbs: 12 (see list above) +# max_dbs: 16 (see list above) # Flags: MDB_NOTLS | MDB_NORDAHEAD # # All Databases are append-only and indexed by height -> LMDB default diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index ee06e8d5a..59bfe057e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -13,6 +13,9 @@ use zebra_state::HashOrHeight; use crate::chain_index::finalised_state::capability::{DbCore, DbWrite}; use crate::chain_index::finalised_state::DbMetadata; +use crate::chain_index::ShieldedPool; + +use super::super::{optional_pool_root, required_pool_root}; use crate::chain_index::source::BlockchainSourceError; use crate::chain_index::{ finalised_state::capability::{ @@ -33,6 +36,31 @@ use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; const EPHEMERAL_FINALISED_STATE_STATUS_POLL_INTERVAL: Duration = Duration::from_secs(5); +/// Converts a raw `u32` block height (a stored [`TxLocation`] height) into a [`Height`], +/// surfacing an out-of-range value as a [`FinalisedStateError`] instead of panicking. +fn height_from_u32(height: u32) -> Result { + Height::try_from(height).map_err(|error| { + FinalisedStateError::Custom(format!("invalid block height {height}: {error}")) + }) +} + +/// Collects one item per height across the inclusive `start..=end` range, in ascending +/// order, by calling `get_at` for each height. +async fn collect_block_range( + start: Height, + end: Height, + mut get_at: impl FnMut(Height) -> Fut, +) -> Result, FinalisedStateError> +where + Fut: std::future::Future>, +{ + let mut items = Vec::new(); + for height in Height::range_inclusive(start, end) { + items.push(get_at(height).await?); + } + Ok(items) +} + /// Source-backed finalised-state backend used when persistent finalised-state storage is not /// serving normal requests. /// @@ -244,53 +272,41 @@ impl EphemeralFinalisedState { let block_hash = BlockHash::from(block.hash()); let block_height = zebra_chain::block::Height(height.0); - let (sapling, orchard) = self.source.get_commitment_tree_roots(block_hash).await?; + let (sapling, orchard, ironwood) = + self.source.get_commitment_tree_roots(block_hash).await?; + let sapling_is_active = self.network.is_nu_active( - zcash_protocol::consensus::NetworkUpgrade::Sapling, + ShieldedPool::Sapling.zcash_protocol_activation_upgrade(), block_height.into(), ); let orchard_is_active = self.network.is_nu_active( - zcash_protocol::consensus::NetworkUpgrade::Nu5, + ShieldedPool::Orchard.zcash_protocol_activation_upgrade(), block_height.into(), ); - let (sapling_root, sapling_size) = match sapling { - Some((root, size)) => (root, size), - None if !sapling_is_active => Default::default(), - None => { - return Err(FinalisedStateError::BlockchainSourceError( - BlockchainSourceError::Unrecoverable(format!( - "missing Sapling commitment tree root for active Sapling block at height {height}" - )), - )); - } - }; - let (orchard_root, orchard_size) = match orchard { - Some((root, size)) => (root, size), - None if !orchard_is_active => Default::default(), - None => { - return Err(FinalisedStateError::BlockchainSourceError( - BlockchainSourceError::Unrecoverable(format!( - "missing Orchard commitment tree root for active NU5 block at height {height}" - )), - )); - } - }; - let sapling_size = u32::try_from(sapling_size).map_err(|error| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("sapling commitment tree size does not fit into u32: {error}"), - )) - })?; - let orchard_size = u32::try_from(orchard_size).map_err(|error| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("orchard commitment tree size does not fit into u32: {error}"), - )) - })?; + let ironwood_is_active = self.network.is_nu_active( + ShieldedPool::Ironwood.zcash_protocol_activation_upgrade(), + block_height.into(), + ); + + let (sapling_root, sapling_size) = + required_pool_root(ShieldedPool::Sapling, sapling_is_active, sapling, || { + format!("block at height {height}") + })?; + let (orchard_root, orchard_size) = + required_pool_root(ShieldedPool::Orchard, orchard_is_active, orchard, || { + format!("block at height {height}") + })?; + let ironwood = + optional_pool_root(ShieldedPool::Ironwood, ironwood_is_active, ironwood, || { + format!("block at height {height}") + })?; let block_metadata = BlockMetadata::new( sapling_root, sapling_size, orchard_root, orchard_size, + ironwood, None, // ephemeral store does not track chainwork self.network.clone(), ); @@ -454,16 +470,7 @@ impl BlockCoreExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut headers = Vec::new(); - - for height in u32::from(start)..=u32::from(end) { - headers.push( - self.get_block_header(Height::try_from(height).unwrap()) - .await?, - ); - } - - Ok(headers) + collect_block_range(start, end, |height| self.get_block_header(height)).await } async fn get_block_txids(&self, height: Height) -> Result { @@ -483,16 +490,7 @@ impl BlockCoreExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut txid_lists = Vec::new(); - - for height in u32::from(start)..=u32::from(end) { - txid_lists.push( - self.get_block_txids(Height::try_from(height).unwrap()) - .await?, - ); - } - - Ok(txid_lists) + collect_block_range(start, end, |height| self.get_block_txids(height)).await } async fn get_txid( @@ -547,7 +545,7 @@ impl BlockTransparentExt for EphemeralFinalisedState { tx_location: TxLocation, ) -> Result, FinalisedStateError> { let chain_block = self - .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .get_required_chain_block(height_from_u32(tx_location.block_height())?) .await?; Ok(chain_block @@ -576,16 +574,7 @@ impl BlockTransparentExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut transparent_lists = Vec::new(); - - for height in u32::from(start)..=u32::from(end) { - transparent_lists.push( - self.get_block_transparent(Height::try_from(height).unwrap()) - .await?, - ); - } - - Ok(transparent_lists) + collect_block_range(start, end, |height| self.get_block_transparent(height)).await } async fn get_previous_output( @@ -634,7 +623,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { tx_location: TxLocation, ) -> Result, FinalisedStateError> { let chain_block = self - .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .get_required_chain_block(height_from_u32(tx_location.block_height())?) .await?; Ok(chain_block @@ -663,16 +652,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut sapling_lists = Vec::new(); - - for height in u32::from(start)..=u32::from(end) { - sapling_lists.push( - self.get_block_sapling(Height::try_from(height).unwrap()) - .await?, - ); - } - - Ok(sapling_lists) + collect_block_range(start, end, |height| self.get_block_sapling(height)).await } async fn get_orchard( @@ -680,7 +660,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { tx_location: TxLocation, ) -> Result, FinalisedStateError> { let chain_block = self - .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .get_required_chain_block(height_from_u32(tx_location.block_height())?) .await?; Ok(chain_block @@ -709,16 +689,44 @@ impl BlockShieldedExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut orchard_lists = Vec::new(); + collect_block_range(start, end, |height| self.get_block_orchard(height)).await + } - for height in u32::from(start)..=u32::from(end) { - orchard_lists.push( - self.get_block_orchard(Height::try_from(height).unwrap()) - .await?, - ); - } + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + let chain_block = self + .get_required_chain_block(height_from_u32(tx_location.block_height())?) + .await?; - Ok(orchard_lists) + Ok(chain_block + .transactions() + .get(usize::from(tx_location.tx_index())) + .map(|transaction| transaction.ironwood().clone())) + } + + async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + let chain_block = self.get_required_chain_block(height).await?; + + Ok(OrchardTxList::new( + chain_block + .transactions() + .iter() + .map(|transaction| Some(transaction.ironwood().clone())) + .collect(), + )) + } + + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + collect_block_range(start, end, |height| self.get_block_ironwood(height)).await } async fn get_block_commitment_tree_data( @@ -734,16 +742,10 @@ impl BlockShieldedExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut commitment_tree_data = Vec::new(); - - for height in u32::from(start)..=u32::from(end) { - commitment_tree_data.push( - self.get_block_commitment_tree_data(Height::try_from(height).unwrap()) - .await?, - ); - } - - Ok(commitment_tree_data) + collect_block_range(start, end, |height| { + self.get_block_commitment_tree_data(height) + }) + .await } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 6c2bce199..bd241b01b 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -39,10 +39,10 @@ use crate::{ config::ChainIndexConfig, error::FinalisedStateError, BlockHash, BlockHeaderData, CommitmentTreeData, CompactBlockStream, CompactOrchardAction, - CompactSaplingOutput, CompactSaplingSpend, CompactSize, CompactTxData, FixedEncodedLen as _, - Height, IndexedBlock, NamedAtomicStatus, OrchardCompactTx, OrchardTxList, Outpoint, - SaplingCompactTx, SaplingTxList, StatusType, TransparentCompactTx, TransparentTxList, - TxInCompact, TxLocation, TxOutCompact, TxidList, ZainoVersionedSerde as _, + CompactSaplingSpend, CompactSize, CompactTxData, FixedEncodedLen as _, Height, IndexedBlock, + NamedAtomicStatus, OrchardCompactTx, OrchardTxList, Outpoint, SaplingCompactTx, SaplingTxList, + StatusType, TransparentCompactTx, TransparentTxList, TxInCompact, TxLocation, TxOutCompact, + TxidList, ZainoVersionedSerde as _, }; #[cfg(feature = "transparent_address_history_experimental")] @@ -126,15 +126,15 @@ pub(crate) const DB_SCHEMA_V1_TEXT: &str = include_str!("db_schema_v1.txt"); /// This value is compared against the schema hash stored in the metadata record to detect schema /// drift without a corresponding version bump. pub(crate) const DB_SCHEMA_V1_HASH: [u8; 32] = [ - 0x11, 0xb2, 0x6a, 0x12, 0x08, 0x67, 0xf0, 0x42, 0xf6, 0x31, 0x45, 0xea, 0x87, 0xe7, 0x23, 0x75, - 0x40, 0x3b, 0xf2, 0x14, 0xaa, 0x2b, 0x00, 0x12, 0xec, 0xa4, 0x4d, 0x00, 0xe9, 0x0b, 0x07, 0x9b, + 0xaf, 0xcb, 0x80, 0xfe, 0x89, 0x2b, 0xc5, 0xba, 0x8e, 0x5d, 0x20, 0xfe, 0x56, 0x72, 0x81, 0x75, + 0x58, 0x8a, 0xb6, 0x49, 0xf7, 0xc4, 0x45, 0xcd, 0xa2, 0x8f, 0xaf, 0xb9, 0x6a, 0x95, 0xc8, 0x75, ]; /// *Current* database V1 version. pub(crate) const DB_VERSION_V1: DbVersion = DbVersion { major: 1, - minor: 2, - patch: 1, + minor: 3, + patch: 0, }; /// LMDB table name for the finalised txout-set accumulator. @@ -314,9 +314,17 @@ pub(crate) struct DbV1 { /// Stored per-block, in order. orchard: Database, - /// Block commitment tree data: `Height` -> `StoredEntryFixed>` + /// Ironwood: `Height` -> `StoredEntryVar` /// - /// Stored per-block, in order. + /// Ironwood (NU6.3) shielded-pool actions, modelled with the Orchard compact types. Stored + /// per-block, in order. Introduced in schema v1.3.0. + ironwood: Database, + + /// Block commitment tree data: `Height` -> `StoredEntryVar` + /// + /// Stored per-block, in order. The value is a `StoredEntryVar` (not `StoredEntryFixed`) from + /// schema v1.3.0 onward, because `CommitmentTreeData` V2 carries an optional Ironwood root and + /// is therefore variable-length. commitment_tree_data: Database, /// Heights: `Hash` -> `StoredEntryFixed` @@ -397,16 +405,22 @@ pub(crate) struct DbV1 { /// - validated read fetchers used by the capability trait implementations, and /// - internal validation / indexing helpers. impl DbV1 { - /// Spawns a new [`DbV1`] and opens (or creates) the LMDB environment for the configured network. + /// Opens (and heals) the v1 database for the configured network **without** starting the + /// background validator. /// /// This method: /// - chooses a versioned path suffix (`...//v1`), /// - configures LMDB map size and reader slots, - /// - opens or creates all V1 named databases, - /// - validates or initializes the `"metadata"` record (schema hash + version), and - /// - spawns the background validator / maintenance task. + /// - opens or creates all V1 named databases, and + /// - validates or initializes the `"metadata"` record (schema hash + version). + /// + /// The validator is started separately via [`DbV1::start_validator`]. This split exists so the + /// orchestrator can guarantee that any pending schema migration finishes *before* validation + /// runs: the validator's `initial_block_scan` reads tables (e.g. `commitment_tree_data_1_3_0`) + /// that a migration populates, so starting it concurrently with a migration races the migration + /// and can fail on a not-yet-written row. pub(crate) async fn spawn(config: &ChainIndexConfig) -> Result { - let mut zaino_db = Self::open_env_and_dbs(config).await?; + let zaino_db = Self::open_env_and_dbs(config).await?; // Validate (or initialise) the metadata entry before we touch any tables. zaino_db.check_schema_version().await?; @@ -416,21 +430,33 @@ impl DbV1 { // on a quiescent database. zaino_db.reconcile_alpha_txid_location_index().await?; - // Spawn handler task to perform background validation and trailing tx cleanup. - zaino_db.spawn_handler().await?; - Ok(zaino_db) } /// Opens the LMDB environment and every V1 named database, returning an *unstarted* [`DbV1`] /// (status `Spawning`, `db_handler` = `None`, fresh atomics). Performs no metadata validation /// and starts no background task — each caller (`spawn`, `spawn_v1_0_0`) adds its own tail. + /// + /// The `commitment_tree_data` handle is the up-to-date `commitment_tree_data_1_3_0` table + /// (`StoredEntryVar`). The v1.0.0 fixture builder ([`DbV1::spawn_v1_0_0`]) opens the legacy + /// `commitment_tree_data_1_0_0` table (`StoredEntryFixed`) instead, via + /// [`DbV1::open_env_and_dbs_with_commitment_table`]. async fn open_env_and_dbs(config: &ChainIndexConfig) -> Result { + Self::open_env_and_dbs_with_commitment_table(config, "commitment_tree_data_1_3_0").await + } + + /// [`DbV1::open_env_and_dbs`] with the commitment-tree table name as a parameter, so the + /// v1.0.0 fixture opener can select the legacy table without creating the v1.3.0 one + /// (on-disk version detection keys off which tables exist). + async fn open_env_and_dbs_with_commitment_table( + config: &ChainIndexConfig, + commitment_table: &str, + ) -> Result { info!("Launching FinalisedState"); // Prepare database details and path. let db_size_bytes = config.storage.database.size.to_byte_count(); - let db_path_dir = match config.network.to_zebra_network().kind() { + let db_path_dir = match config.network.kind() { NetworkKind::Mainnet => "mainnet", NetworkKind::Testnet => "testnet", NetworkKind::Regtest => "regtest", @@ -464,7 +490,7 @@ impl DbV1 { // drops the unflushed page cache, write order is not guaranteed and a crash *can* leave torn // pages; the recovery there is to wipe and re-index. See `SYNC_CHECKPOINT_INTERVAL`.) let env = Environment::new() - .set_max_dbs(15) + .set_max_dbs(16) .set_map_size(db_size_bytes) .set_max_readers(max_readers) .set_flags( @@ -484,9 +510,10 @@ impl DbV1 { super::open_or_create_db(&env, "sapling_1_0_0", DatabaseFlags::empty()).await?; let orchard = super::open_or_create_db(&env, "orchard_1_0_0", DatabaseFlags::empty()).await?; + let ironwood = + super::open_or_create_db(&env, "ironwood_1_3_0", DatabaseFlags::empty()).await?; let commitment_tree_data = - super::open_or_create_db(&env, "commitment_tree_data_1_0_0", DatabaseFlags::empty()) - .await?; + super::open_or_create_db(&env, commitment_table, DatabaseFlags::empty()).await?; let hashes = super::open_or_create_db(&env, "hashes_1_0_0", DatabaseFlags::empty()).await?; let spent = super::open_or_create_db(&env, "spent_1_0_0", DatabaseFlags::empty()).await?; @@ -519,6 +546,7 @@ impl DbV1 { transparent, sapling, orchard, + ironwood, commitment_tree_data, heights: hashes, spent, @@ -546,6 +574,7 @@ impl DbV1 { transparent: self.transparent, sapling: self.sapling, orchard: self.orchard, + ironwood: self.ironwood, commitment_tree_data: self.commitment_tree_data, heights: self.heights, spent: self.spent, @@ -572,7 +601,12 @@ impl DbV1 { /// `initial_block_scan`). /// - **Steady state:** periodically attempts to validate the next height after `validated_tip`. /// Separately, it performs periodic trailing-reader cleanup via `clean_trailing()`. - async fn spawn_handler(&mut self) -> Result<(), FinalisedStateError> { + /// + /// Kept separate from [`DbV1::spawn`] so the orchestrator starts it only once all pending + /// migrations have finished (the validator reads tables a migration populates). Takes `&self` + /// (the join handle lives behind a `Mutex`) so it can be driven through the shared + /// `Arc` the router holds after spawn. + pub(super) fn start_validator(&self) { // Clone everything the task needs so we can move it into the async block. let zaino_db = self.detached_handle(); @@ -674,7 +708,6 @@ impl DbV1 { }); *self.db_handler.lock().expect("db_handler mutex poisoned") = Some(handle); - Ok(()) } /// Validates every stored spent-outpoint entry (`Outpoint` -> `TxLocation`) by checksum. @@ -811,6 +844,18 @@ impl DbV1 { self.metadata } + /// Provides access to the (v1.3.0) `StoredEntryVar` commitment-tree-data table, required for + /// Migration1_2_1To1_3_0 to write the rebuilt commitment rows. + pub(crate) fn commitment_tree_data_db(&self) -> Database { + self.commitment_tree_data + } + + /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to + /// backfill ironwood rows for post-NU6.3 blocks from validator-fetched block data. + pub(super) fn ironwood_db(&self) -> Database { + self.ironwood + } + /// Provudes access to the spent DB table, required for Migration1_1_0To1_2_0. pub(crate) fn spent_db(&self) -> Database { self.spent @@ -971,14 +1016,29 @@ impl DbV1 { pub(crate) async fn spawn_v1_0_0( config: &ChainIndexConfig, ) -> Result { - let mut zaino_db = Self::open_env_and_dbs(config).await?; + // The v1.0.0 fixture reproduces the legacy on-disk layout: its commitment tree data lives in + // `commitment_tree_data_1_0_0` as a `StoredEntryFixed` (see `write_block_v1_0_0`), which the + // v1.2.1 -> v1.3.0 migration later rebuilds into the `commitment_tree_data_1_3_0` + // `StoredEntryVar` table. This opener therefore opens the legacy commitment table and never + // creates the v1.3.0 table. + let zaino_db = + Self::open_env_and_dbs_with_commitment_table(config, "commitment_tree_data_1_0_0") + .await?; // Write the historical v1.0.0 metadata record. Intentionally skips `check_schema_version` // (see the method doc) — that is the behavioural difference from `spawn`. zaino_db.write_v1_0_0_metadata()?; - // Spawn handler task to perform background validation and trailing tx cleanup. - zaino_db.spawn_handler().await?; + // Deliberately does NOT start the background validator. This builds a *pre-migration* + // v1.0.0 fixture; the validator validates against the current (v1.3.0) schema — it reads + // `commitment_tree_data_1_3_0` and the `ironwood` table — so it must run only after the + // database has been migrated to the newest schema. Callers build the fixture with direct + // v1.0.0 writes, shut it down, then reopen through `FinalisedState::spawn`, which migrates + // first and starts the validator afterwards. + // + // With no validator to advance it, mark the empty fixture `Ready` directly so callers see a + // settled backend. + zaino_db.status.store(StatusType::Ready); Ok(zaino_db) } @@ -1068,7 +1128,7 @@ mod tests { async fn initial_spent_scan_reports_corrupt_value() { use lmdb::{Transaction as _, WriteFlags}; use zaino_common::network::ActivationHeights; - use zaino_common::{DatabaseConfig, Network, StorageConfig}; + use zaino_common::{DatabaseConfig, StorageConfig}; let temp_dir = tempfile::tempdir().expect("tempdir"); let config = ChainIndexConfig { @@ -1081,7 +1141,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let db = DbV1::spawn(&config).await.expect("spawn empty v1 db"); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs index 5a82c9b13..20621b02d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs @@ -56,27 +56,11 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.headers, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "header data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - let entry = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("header decode error: {e}")))?; - - Ok(*entry.inner()) - }) + self.read_row_at_height(self.headers, "header", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("header data missing from db".into()) + }) } /// Fetches block headers for the given height range. @@ -91,45 +75,7 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.headers) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "header data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| *e.inner()) - .map_err(|e| FinalisedStateError::Custom(format!("header decode error: {e}"))) - }) - .collect() + self.scan_rows(self.headers, "header", start, end).await } /// Fetch the txid bytes for a given TxLocation. @@ -188,14 +134,26 @@ impl DbV1 { // Each txid entry is: [0] version tag + [1..32] txid // So we skip idx * 33 bytes to reach the start of the correct Hash - let offset = cursor.position() + (idx as u64) * TransactionHash::VERSIONED_LEN as u64; + let transaction_versioned_len = TransactionHash::latest_versioned_len()?; + let offset = cursor.position() + (idx as u64) * transaction_versioned_len as u64; cursor.set_position(offset); // Read [0] Txid Record version (skip 1 byte) cursor.set_position(cursor.position() + 1); // Then read 32 bytes for the txid - let mut txid_bytes = [0u8; TransactionHash::ENCODED_LEN]; + let transaction_encoded_len = TransactionHash::latest_encoded_len()?; + if transaction_encoded_len != 32 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "TransactionHash latest encoded length must be 32 bytes, got {}", + transaction_encoded_len + ), + ))?; + } + + let mut txid_bytes = [0u8; 32]; cursor .read_exact(&mut txid_bytes) .map_err(|e| FinalisedStateError::Custom(format!("txid read error: {e}")))?; @@ -206,28 +164,9 @@ impl DbV1 { /// Fetch block txids by height. async fn get_block_txids(&self, height: Height) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.txids, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "txid data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("txids decode error: {e}")))?; - - Ok(entry.inner().clone()) - }) + self.read_row_at_height(self.txids, "txids", height) + .await? + .ok_or_else(|| FinalisedStateError::DataUnavailable("txid data missing from db".into())) } /// Fetches block txids for the given height range. @@ -242,45 +181,7 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.txids) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "txid data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| FinalisedStateError::Custom(format!("txids decode error: {e}"))) - }) - .collect() + self.scan_rows(self.txids, "txids", start, end).await } // Fetch the TxLocation for the given txid, transaction data is indexed by TxLocation internally. diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs index 087d679a1..61cf42644 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs @@ -2,6 +2,28 @@ use super::*; +use crate::chain_index::ShieldedPool; +use crate::{FixedEncodedLen, ZainoVersionedSerde}; + +/// How a pool point lookup treats a block height with no row in the pool's table. +enum MissingRow { + /// Every indexed block has a row in this pool's table; an absent row is an error. + Error, + /// The table is sparse: an absent row means the block has no data for this pool. + NoPoolData, +} + +impl MissingRow { + /// The missing-row policy of `pool`'s table: sapling and orchard rows exist for + /// every indexed block; pools introduced after schema v1.3.0 use sparse tables. + fn for_pool(pool: ShieldedPool) -> MissingRow { + match pool { + ShieldedPool::Sapling | ShieldedPool::Orchard => MissingRow::Error, + ShieldedPool::Ironwood => MissingRow::NoPoolData, + } + } +} + /// [`BlockShieldedExt`] capability implementation for [`DbV1`]. /// /// Provides access to Sapling / Orchard compact transaction data and per-block commitment tree @@ -51,6 +73,28 @@ impl BlockShieldedExt for DbV1 { self.get_block_range_orchard(start, end).await } + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + self.get_ironwood(tx_location).await + } + + async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + self.get_block_ironwood(height).await + } + + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + self.get_block_range_ironwood(start, end).await + } + async fn get_block_commitment_tree_data( &self, height: Height, @@ -77,177 +121,240 @@ impl DbV1 { &self, tx_location: TxLocation, ) -> Result, FinalisedStateError> { - use std::io::{Cursor, Read}; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - - let height = Height::try_from(tx_location.block_height()) - .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; - let height_bytes = height.to_bytes()?; - - let raw = match txn.get(self.sapling, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "sapling data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - let mut cursor = Cursor::new(raw); - - // Skip [0] StoredEntry version - cursor.set_position(1); - - // Read CompactSize: length of serialized body - CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("compact size read error: {e}")) - })?; - - // Skip SaplingTxList version byte - cursor.set_position(cursor.position() + 1); - - // Read CompactSize: number of entries - let list_len = CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("sapling tx list len error: {e}")) - })?; + self.get_pool_tx(ShieldedPool::Sapling, tx_location) + } - let idx = tx_location.tx_index() as usize; - if idx >= list_len as usize { - return Err(FinalisedStateError::Custom( - "tx_index out of range in sapling tx list".to_string(), - )); - } + /// Fetch block sapling transaction data by height. + async fn get_block_sapling( + &self, + height: Height, + ) -> Result { + self.get_block_pool_tx_list(ShieldedPool::Sapling, height, || { + SaplingTxList::new(Vec::new()) + }) + .await + } - // Skip preceding entries - for _ in 0..idx { - Self::skip_opt_sapling_entry(&mut cursor) - .map_err(|e| FinalisedStateError::Custom(format!("skip entry error: {e}")))?; - } + /// Fetches block sapling tx data for the given height range. + /// + /// Uses cursor based fetch. + /// + /// NOTE: Currently this method only fetches ranges where start_height <= end_height, + /// This could be updated by following the cursor step example in + /// get_compact_block_streamer. + async fn get_block_range_sapling( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + self.get_block_range_pool_tx_list(ShieldedPool::Sapling, start, end, || { + SaplingTxList::new(Vec::new()) + }) + .await + } - let start = cursor.position(); + /// Fetch the serialized OrchardCompactTx for the given TxLocation, if present. + /// + /// This uses an optimized lookup without decoding the full TxidList. + async fn get_orchard( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + self.get_pool_tx(ShieldedPool::Orchard, tx_location) + } - // Peek presence flag - let mut presence = [0u8; 1]; - cursor.read_exact(&mut presence).map_err(|e| { - FinalisedStateError::Custom(format!("failed to read Option tag: {e}")) - })?; + /// Fetch block orchard transaction data by height. + async fn get_block_orchard( + &self, + height: Height, + ) -> Result { + self.get_block_pool_tx_list(ShieldedPool::Orchard, height, || { + OrchardTxList::new(Vec::new()) + }) + .await + } - if presence[0] == 0 { - return Ok(None); - } else if presence[0] != 1 { - return Err(FinalisedStateError::Custom(format!( - "invalid Option tag: {}", - presence[0] - ))); - } + /// Fetches block orchard tx data for the given height range. + /// + /// Uses cursor based fetch. + /// + /// NOTE: Currently this method only fetches ranges where start_height <= end_height, + /// This could be updated by following the cursor step example in + /// get_compact_block_streamer. + async fn get_block_range_orchard( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + self.get_block_range_pool_tx_list(ShieldedPool::Orchard, start, end, || { + OrchardTxList::new(Vec::new()) + }) + .await + } - // Rewind to include tag in returned bytes - cursor.set_position(start); - Self::skip_opt_sapling_entry(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("skip entry error (second pass): {e}")) - })?; + /// Fetch the serialized `OrchardCompactTx` for the given TxLocation from the ironwood table. + /// + /// Mirrors [`DbV1::get_orchard`] against the ironwood table (ironwood reuses the Orchard compact + /// types). A missing ironwood row — any block below NU6.3 activation, or one written before + /// schema v1.3.0 — yields `Ok(None)` rather than an error. + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + self.get_pool_tx(ShieldedPool::Ironwood, tx_location) + } - let end = cursor.position(); + /// Fetch block ironwood transaction data by height. + /// + /// A missing ironwood row yields an empty [`OrchardTxList`]. + async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + self.get_block_pool_tx_list(ShieldedPool::Ironwood, height, || { + OrchardTxList::new(Vec::new()) + }) + .await + } - Ok(Some(SaplingCompactTx::from_bytes( - &raw[start as usize..end as usize], - )?)) + /// Fetches block ironwood tx data for the given (inclusive) height range. + /// + /// Unlike the orchard range fetch this resolves each height individually so that heights with no + /// ironwood row (pre-v1.3.0 / pre-NU6.3) yield an empty [`OrchardTxList`], keeping the result + /// aligned one-entry-per-height with the requested range. + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + self.get_block_range_pool_tx_list(ShieldedPool::Ironwood, start, end, || { + OrchardTxList::new(Vec::new()) }) + .await } - /// Fetch block sapling transaction data by height. - async fn get_block_sapling( + /// Fetch block commitment tree data by height. + async fn get_block_commitment_tree_data( &self, height: Height, - ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.sapling, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "sapling data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("sapling decode error: {e}")))?; - - Ok(entry.inner().clone()) - }) + ) -> Result { + self.read_row_at_height(self.commitment_tree_data, "commitment_tree", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("commitment tree data missing from db".into()) + }) } - /// Fetches block sapling tx data for the given height range. + /// Fetches block commitment tree data for the given height range. /// /// Uses cursor based fetch. /// /// NOTE: Currently this method only fetches ranges where start_height <= end_height, /// This could be updated by following the cursor step example in /// get_compact_block_streamer. - async fn get_block_range_sapling( + async fn get_block_range_commitment_tree_data( &self, start: Height, end: Height, - ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); + ) -> Result, FinalisedStateError> { + self.scan_rows(self.commitment_tree_data, "commitment_tree", start, end) + .await + } + + // *** Internal DB methods *** + + /// The LMDB table holding `pool`'s per-block compact transaction lists. + fn pool_table(&self, pool: ShieldedPool) -> lmdb::Database { + match pool { + ShieldedPool::Sapling => self.sapling, + ShieldedPool::Orchard => self.orchard, + ShieldedPool::Ironwood => self.ironwood, } + } - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; + /// The entry-skip function for `pool`'s tx-list encoding (orchard and ironwood + /// share the Orchard compact layout). + fn pool_skip_entry(pool: ShieldedPool) -> fn(&mut std::io::Cursor<&[u8]>) -> io::Result<()> { + match pool { + ShieldedPool::Sapling => Self::skip_opt_sapling_entry, + ShieldedPool::Orchard | ShieldedPool::Ironwood => Self::skip_opt_orchard_entry, + } + } - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.sapling) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "sapling data missing from db".into(), + /// Fetch one pool's whole-block tx list by height. Dense pools error on a missing + /// row; sparse pools yield `empty()`. + async fn get_block_pool_tx_list( + &self, + pool: ShieldedPool, + height: Height, + empty: impl FnOnce() -> T, + ) -> Result { + let label = pool.pool_string(); + match self + .read_row_at_height(self.pool_table(pool), &label, height) + .await? + { + Some(list) => Ok(list), + None => match MissingRow::for_pool(pool) { + MissingRow::Error => Err(FinalisedStateError::DataUnavailable(format!( + "{label} data missing from db" + ))), + MissingRow::NoPoolData => Ok(empty()), + }, + } + } + + /// Fetch one pool's tx lists for the inclusive height range, one entry per height. + /// + /// Dense pools cursor-scan the range; sparse pools resolve each height individually + /// so absent rows yield `empty()` and the result stays aligned one-entry-per-height + /// with the requested range. + async fn get_block_range_pool_tx_list( + &self, + pool: ShieldedPool, + start: Height, + end: Height, + empty: impl Fn() -> T, + ) -> Result, FinalisedStateError> { + match MissingRow::for_pool(pool) { + MissingRow::Error => { + self.scan_rows(self.pool_table(pool), &pool.pool_string(), start, end) + .await + } + MissingRow::NoPoolData => { + if end.0 < start.0 { + return Err(FinalisedStateError::Custom( + "invalid block range: end < start".to_string(), )); } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; + self.validate_block_range(start, end).await?; + + let mut out = Vec::with_capacity((end.0 - start.0 + 1) as usize); + for height in Height::range_inclusive(start, end) { + out.push(self.get_block_pool_tx_list(pool, height, &empty).await?); } - raw_entries.push(v.to_vec()); + Ok(out) } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| FinalisedStateError::Custom(format!("sapling decode error: {e}"))) - }) - .collect() + } } - /// Fetch the serialized OrchardCompactTx for the given TxLocation, if present. + /// Point lookup for one transaction's compact data in `pool`'s per-block table, + /// without decoding the whole block row. /// - /// This uses an optimized lookup without decoding the full TxidList. - async fn get_orchard( + /// Walks the `StoredEntryVar` tx-list bytes entry-by-entry with the pool's skip + /// function, then decodes only the requested entry. + fn get_pool_tx( &self, + pool: ShieldedPool, tx_location: TxLocation, - ) -> Result, FinalisedStateError> { + ) -> Result, FinalisedStateError> { use std::io::{Cursor, Read}; + let table = self.pool_table(pool); + let label = pool.pool_string(); + let missing_row = MissingRow::for_pool(pool); + let skip_entry = Self::pool_skip_entry(pool); + tokio::task::block_in_place(|| { let txn = self.env.begin_ro_txn()?; @@ -255,12 +362,15 @@ impl DbV1 { .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; let height_bytes = height.to_bytes()?; - let raw = match txn.get(self.orchard, &height_bytes) { + let raw = match txn.get(table, &height_bytes) { Ok(val) => val, Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "orchard data missing from db".into(), - )); + return match missing_row { + MissingRow::NoPoolData => Ok(None), + MissingRow::Error => Err(FinalisedStateError::DataUnavailable(format!( + "{label} data missing from db" + ))), + }; } Err(e) => return Err(FinalisedStateError::LmdbError(e)), }; @@ -275,24 +385,24 @@ impl DbV1 { FinalisedStateError::Custom(format!("compact size read error: {e}")) })?; - // Skip OrchardTxList version byte + // Skip the tx-list version byte cursor.set_position(cursor.position() + 1); // Read CompactSize: number of entries let list_len = CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("orchard tx list len error: {e}")) + FinalisedStateError::Custom(format!("{label} tx list len error: {e}")) })?; let idx = tx_location.tx_index() as usize; if idx >= list_len as usize { - return Err(FinalisedStateError::Custom( - "tx_index out of range in orchard tx list".to_string(), - )); + return Err(FinalisedStateError::Custom(format!( + "tx_index out of range in {label} tx list" + ))); } // Skip preceding entries for _ in 0..idx { - Self::skip_opt_orchard_entry(&mut cursor) + skip_entry(&mut cursor) .map_err(|e| FinalisedStateError::Custom(format!("skip entry error: {e}")))?; } @@ -313,207 +423,29 @@ impl DbV1 { ))); } - // Rewind to include presence flag in output + // Rewind to include the presence flag in the returned bytes cursor.set_position(start); - Self::skip_opt_orchard_entry(&mut cursor).map_err(|e| { + skip_entry(&mut cursor).map_err(|e| { FinalisedStateError::Custom(format!("skip entry error (second pass): {e}")) })?; let end = cursor.position(); - Ok(Some(OrchardCompactTx::from_bytes( - &raw[start as usize..end as usize], - )?)) - }) - } - - /// Fetch block orchard transaction data by height. - async fn get_block_orchard( - &self, - height: Height, - ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.orchard, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "orchard data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("orchard decode error: {e}")))?; - - Ok(entry.inner().clone()) - }) - } - - /// Fetches block orchard tx data for the given height range. - /// - /// Uses cursor based fetch. - /// - /// NOTE: Currently this method only fetches ranges where start_height <= end_height, - /// This could be updated by following the cursor step example in - /// get_compact_block_streamer. - async fn get_block_range_orchard( - &self, - start: Height, - end: Height, - ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.orchard) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "orchard data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| FinalisedStateError::Custom(format!("orchard decode error: {e}"))) - }) - .collect() - } - - /// Fetch block commitment tree data by height. - async fn get_block_commitment_tree_data( - &self, - height: Height, - ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.commitment_tree_data, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "commitment tree data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry = StoredEntryFixed::from_bytes(raw).map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - })?; - - Ok(entry.item) + Ok(Some(T::from_bytes(&raw[start as usize..end as usize])?)) }) } - /// Fetches block commitment tree data for the given height range. - /// - /// Uses cursor based fetch. - /// - /// NOTE: Currently this method only fetches ranges where start_height <= end_height, - /// This could be updated by following the cursor step example in - /// get_compact_block_streamer. - async fn get_block_range_commitment_tree_data( - &self, - start: Height, - end: Height, - ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.commitment_tree_data) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "commitment tree data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryFixed::::from_bytes(&bytes) - .map(|e| e.item) - .map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - }) - }) - .collect() - } - - // *** Internal DB methods *** - - /// Skips one `Option` from the current cursor position. - /// - /// The input should be a cursor over just the inner item "list" bytes of a: - /// - `StoredEntryVar` - /// - /// Advances past: - /// - 1 byte `0x00` if None, or - /// - 1 + 1 + value + spends + outputs if Some (presence + version + body) - /// - /// This is faster than deserialising the whole struct as we only read the compact sizes. + /// Skips the shared prelude of one `Option` entry: the presence + /// byte, and — when the entry is present — the version byte and the `Option` + /// value balance. Returns `false` when the entry was `None` (nothing more to skip). #[inline] - fn skip_opt_sapling_entry(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result<()> { + fn skip_opt_tx_prelude(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result { // Read presence byte let mut presence = [0u8; 1]; cursor.read_exact(&mut presence)?; if presence[0] == 0 { - return Ok(()); + return Ok(false); } else if presence[0] != 1 { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -537,23 +469,46 @@ impl DbV1 { )); } - // Read number of spends (CompactSize) - let spend_len = CompactSize::read(&mut *cursor)? as usize; - let spend_skip = spend_len * CompactSaplingSpend::VERSIONED_LEN; - cursor.set_position(cursor.position() + spend_skip as u64); - - // Read number of outputs (CompactSize) - let output_len = CompactSize::read(&mut *cursor)? as usize; - let output_skip = output_len * CompactSaplingOutput::VERSIONED_LEN; - cursor.set_position(cursor.position() + output_skip as u64); + Ok(true) + } + /// Advances the cursor past a CompactSize entry count followed by that many + /// fixed-length entries of type `T`. Taking the width from the type being + /// skipped makes a wrong-width skip unrepresentable. + #[inline] + fn skip_counted_fixed_entries( + cursor: &mut std::io::Cursor<&[u8]>, + ) -> io::Result<()> { + let count = CompactSize::read(&mut *cursor)? as usize; + let entry_len = T::latest_versioned_len()?; + cursor.set_position(cursor.position() + (count * entry_len) as u64); Ok(()) } + /// Skips one `Option` from the current cursor position. + /// + /// The input should be a cursor over just the inner item "list" bytes of a: + /// - `StoredEntryVar` + /// + /// Advances past: + /// - 1 byte `0x00` if None, or + /// - 1 + 1 + value + spends + outputs if Some (presence + version + body) + /// + /// This is faster than deserialising the whole struct as we only read the compact sizes. + #[inline] + fn skip_opt_sapling_entry(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result<()> { + if !Self::skip_opt_tx_prelude(cursor)? { + return Ok(()); + } + Self::skip_counted_fixed_entries::(cursor)?; + Self::skip_counted_fixed_entries::(cursor) + } + /// Skips one `Option` from the current cursor position. /// /// The input should be a cursor over just the inner item "list" bytes of a: - /// - `StoredEntryVar` + /// - `StoredEntryVar` (the orchard and ironwood tables share this + /// layout) /// /// Advances past: /// - 1 byte `0x00` if None, or @@ -562,42 +517,49 @@ impl DbV1 { /// This is faster than deserialising the whole struct as we only read the compact sizes. #[inline] fn skip_opt_orchard_entry(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result<()> { - // Read presence byte - let mut presence = [0u8; 1]; - cursor.read_exact(&mut presence)?; - - if presence[0] == 0 { + if !Self::skip_opt_tx_prelude(cursor)? { return Ok(()); - } else if presence[0] != 1 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid Option tag: {}", presence[0]), - )); - } - - // Read version - cursor.read_exact(&mut [0u8; 1])?; - - // Read value: Option - let mut value_tag = [0u8; 1]; - cursor.read_exact(&mut value_tag)?; - if value_tag[0] == 1 { - // Some(i64): read 8 bytes - cursor.set_position(cursor.position() + 8); - } else if value_tag[0] != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid Option tag: {}", value_tag[0]), - )); } + Self::skip_counted_fixed_entries::(cursor) + } +} - // Read number of actions (CompactSize) - let action_len = CompactSize::read(&mut *cursor)? as usize; - - // Skip actions: each is 1-byte version + 148-byte body - let action_skip = action_len * CompactOrchardAction::VERSIONED_LEN; - cursor.set_position(cursor.position() + action_skip as u64); - - Ok(()) +#[cfg(test)] +mod skip_opt_sapling_entry { + use super::*; + use crate::CompactSaplingOutput; + + /// Regression test for the sapling point-lookup skip width: the skip must advance + /// exactly one encoded `Option` entry. A refactor had it skipping + /// sapling *outputs* at the spend width (33 bytes instead of 117), landing the cursor + /// mid-record for any entry with outputs and corrupting every `get_sapling` read of a + /// transaction at or after such an entry. + #[test] + fn advances_exactly_one_encoded_entry() { + let tx = SaplingCompactTx::new( + Some(7), + vec![CompactSaplingSpend::new([1u8; 32])], + vec![ + CompactSaplingOutput::new([2u8; 32], [3u8; 32], [4u8; 52]), + CompactSaplingOutput::new([5u8; 32], [6u8; 32], [7u8; 52]), + ], + ); + let list = SaplingTxList::new(vec![Some(tx)]); + let bytes = list + .to_bytes() + .expect("encoding an in-memory list cannot fail"); + + let mut cursor = std::io::Cursor::new(bytes.as_slice()); + // Mirror `get_sapling`: skip the SaplingTxList version byte, then the entry count. + cursor.set_position(1); + CompactSize::read(&mut cursor).expect("entry count is present"); + + DbV1::skip_opt_sapling_entry(&mut cursor).expect("entry is well-formed"); + + assert_eq!( + cursor.position() as usize, + bytes.len(), + "skip_opt_sapling_entry must land exactly on the end of the single encoded entry" + ); } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs index 0339ef486..93eadd93e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs @@ -132,30 +132,11 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.transparent, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "transparent data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| { - FinalisedStateError::Custom(format!("transparent decode error: {e}")) - })?; - - Ok(entry.inner().clone()) - }) + self.read_row_at_height(self.transparent, "transparent", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("transparent data missing from db".into()) + }) } /// Fetches block transparent tx data for the given height range. @@ -170,47 +151,8 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.transparent) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "transparent data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| { - FinalisedStateError::Custom(format!("transparent decode error: {e}")) - }) - }) - .collect() + self.scan_rows(self.transparent, "transparent", start, end) + .await } // *** Internal DB methods *** @@ -250,14 +192,16 @@ impl DbV1 { let vin_len = CompactSize::read(&mut *cursor)? as usize; // Skip vin entries: each is 1-byte version + 36-byte body - let vin_skip = vin_len * TxInCompact::VERSIONED_LEN; + let tx_in_len = TxInCompact::latest_versioned_len()?; + let vin_skip = vin_len * tx_in_len; cursor.set_position(cursor.position() + vin_skip as u64); // Read vout_len (CompactSize) let vout_len = CompactSize::read(&mut *cursor)? as usize; // Skip vout entries: each is 1-byte version + 29-byte body - let vout_skip = vout_len * TxOutCompact::VERSIONED_LEN; + let tx_out_len = TxOutCompact::latest_versioned_len()?; + let vout_skip = vout_len * tx_out_len; cursor.set_position(cursor.position() + vout_skip as u64); Ok(()) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs index 9dc42aadf..eaf484dbe 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs @@ -146,6 +146,26 @@ impl DbV1 { None => &[], }; + // ----- Fetch Ironwood Tx Data ----- + // + // Ironwood rows exist only from schema v1.3.0 (NU6.3) onward, so a missing row is not an + // error — it just means the block has no ironwood data. + let ironwood_stored_entry_var = if pool_types.includes_ironwood() { + match txn.get(self.ironwood, &height_bytes) { + Ok(raw) => Some(StoredEntryVar::::from_bytes(raw).map_err( + |e| FinalisedStateError::Custom(format!("ironwood decode error: {e}")), + )?), + Err(lmdb::Error::NotFound) => None, + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + } + } else { + None + }; + let ironwood = match ironwood_stored_entry_var.as_ref() { + Some(stored_entry_var) => stored_entry_var.inner().tx(), + None => &[], + }; + // ----- Construct CompactTx ----- let vtx: Vec = txids .iter() @@ -184,6 +204,17 @@ impl DbV1 { }) .unwrap_or_default(); + let ironwood_actions = ironwood + .get(i) + .and_then(|opt| opt.as_ref()) + .map(|o| { + o.actions() + .iter() + .map(|a| a.into_compact()) + .collect::>() + }) + .unwrap_or_default(); + let (vin, vout) = transparent .get(i) .and_then(|opt| opt.as_ref()) @@ -205,6 +236,7 @@ impl DbV1 { if spends.is_empty() && outputs.is_empty() && actions.is_empty() + && ironwood_actions.is_empty() && vin.is_empty() && vout.is_empty() { @@ -218,6 +250,7 @@ impl DbV1 { spends, outputs, actions, + ironwood_actions, vin, vout, }) @@ -234,15 +267,17 @@ impl DbV1 { } Err(e) => return Err(FinalisedStateError::LmdbError(e)), }; - let commitment_tree_data: CommitmentTreeData = *StoredEntryFixed::from_bytes(raw) - .map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - })? - .inner(); + let commitment_tree_data: CommitmentTreeData = + *StoredEntryVar::::from_bytes(raw) + .map_err(|e| { + FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) + })? + .inner(); let chain_metadata = zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size: commitment_tree_data.sizes().sapling(), orchard_commitment_tree_size: commitment_tree_data.sizes().orchard(), + ironwood_commitment_tree_size: commitment_tree_data.sizes().ironwood(), }; // ----- Construct CompactBlock ----- @@ -947,6 +982,52 @@ impl DbV1 { None => &[], }; + // Ironwood is fetched with a tolerant point lookup rather than a lockstep cursor: + // rows are sparse (present only from schema v1.3.0 / NU6.3), so a missing row must + // not desync a height-aligned cursor. `NotFound` simply means "no ironwood here". + let ironwood_entries: Option> = + if pool_types.includes_ironwood() { + let ironwood_key = match current_height.to_bytes() { + Ok(key) => key, + Err(error) => { + send_status( + &sender, + tonic::Status::internal(format!( + "ironwood height to_bytes failed: {error}" + )), + ); + return; + } + }; + match txn.get(zaino_db.ironwood, &ironwood_key) { + Ok(raw) => match StoredEntryVar::::from_bytes(raw) + .map_err(|error| format!("ironwood decode error: {error}")) + { + Ok(entry) => Some(entry), + Err(message) => { + send_status(&sender, tonic::Status::internal(message)); + return; + } + }, + Err(lmdb::Error::NotFound) => None, + Err(error) => { + send_status( + &sender, + tonic::Status::internal(format!( + "lmdb get(ironwood) failed: {error}" + )), + ); + return; + } + } + } else { + None + }; + let ironwood = match ironwood_entries.as_ref() { + Some(entry) => entry.inner().tx(), + None => &[], + }; + // Invariant: if a pool is requested, its per-height vector length must match txids. if pool_types.includes_transparent() && transparent.len() != txids.len() { send_status( @@ -984,6 +1065,22 @@ impl DbV1 { ); return; } + // Ironwood rows are optional; only enforce alignment when a row is present. + if pool_types.includes_ironwood() + && !ironwood.is_empty() + && ironwood.len() != txids.len() + { + send_status( + &sender, + tonic::Status::internal(format!( + "ironwood list length mismatch at height {}: txids={}, ironwood={}", + current_height.0, + txids.len(), + ironwood.len(), + )), + ); + return; + } // ----- Build CompactTx list ----- // @@ -1034,6 +1131,17 @@ impl DbV1 { }) .unwrap_or_default(); + let ironwood_actions = ironwood + .get(i) + .and_then(|opt| opt.as_ref()) + .map(|o| { + o.actions() + .iter() + .map(|a| a.into_compact()) + .collect::>() + }) + .unwrap_or_default(); + let (vin, vout) = transparent .get(i) .and_then(|opt| opt.as_ref()) @@ -1049,6 +1157,7 @@ impl DbV1 { if spends.is_empty() && outputs.is_empty() && actions.is_empty() + && ironwood_actions.is_empty() && vin.is_empty() && vout.is_empty() { @@ -1062,6 +1171,7 @@ impl DbV1 { spends, outputs, actions, + ironwood_actions, vin, vout, }); @@ -1069,8 +1179,10 @@ impl DbV1 { // ----- Decode commitment tree data and construct block ----- let commitment_tree_data: CommitmentTreeData = - match StoredEntryFixed::from_bytes(raw_commitment_tree_bytes) - .map_err(|error| format!("commitment_tree decode error: {error}")) + match StoredEntryVar::::from_bytes( + raw_commitment_tree_bytes, + ) + .map_err(|error| format!("commitment_tree decode error: {error}")) { Ok(entry) => *entry.inner(), Err(message) => { @@ -1082,6 +1194,7 @@ impl DbV1 { let chain_metadata = zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size: commitment_tree_data.sizes().sapling(), orchard_commitment_tree_size: commitment_tree_data.sizes().orchard(), + ironwood_commitment_tree_size: commitment_tree_data.sizes().ironwood(), }; let compact_block = zaino_proto::proto::compact_formats::CompactBlock { diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs index e6727bcf7..a70efb0ff 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs @@ -112,9 +112,31 @@ impl DbV1 { .clone(); let orchard = orchard_list.tx(); + // Ironwood (NU6.3): rows only exist from schema v1.3.0 onward, and only for blocks at or + // above NU6.3 activation. A missing row means the block predates ironwood, so every + // transaction has an empty ironwood component. + let ironwood_list = match txn.get(self.ironwood, &height_bytes) { + Ok(raw) => Some( + StoredEntryVar::::from_bytes(raw) + .map_err(|e| { + FinalisedStateError::Custom(format!("ironwood decode error: {e}")) + })? + .inner() + .clone(), + ), + Err(lmdb::Error::NotFound) => None, + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + let ironwood: &[Option] = + ironwood_list.as_ref().map(|list| list.tx()).unwrap_or(&[]); + // Build CompactTxData let len = txids.len(); - if transparent.len() != len || sapling.len() != len || orchard.len() != len { + if transparent.len() != len + || sapling.len() != len + || orchard.len() != len + || (!ironwood.is_empty() && ironwood.len() != len) + { return Err(FinalisedStateError::Custom( "mismatched tx list lengths in block data".to_string(), )); @@ -132,8 +154,20 @@ impl DbV1 { let orchard_tx = orchard[i] .clone() .unwrap_or_else(|| OrchardCompactTx::new(None, vec![])); - - CompactTxData::new(i as u64, txid, transparent_tx, sapling_tx, orchard_tx) + let ironwood_tx = ironwood + .get(i) + .cloned() + .flatten() + .unwrap_or_else(OrchardCompactTx::empty); + + CompactTxData::new( + i as u64, + txid, + transparent_tx, + sapling_tx, + orchard_tx, + ironwood_tx, + ) }) .collect(); @@ -148,11 +182,12 @@ impl DbV1 { Err(e) => return Err(FinalisedStateError::LmdbError(e)), }; - let commitment_tree_data: CommitmentTreeData = *StoredEntryFixed::from_bytes(raw) - .map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - })? - .inner(); + let commitment_tree_data: CommitmentTreeData = + *StoredEntryVar::::from_bytes(raw) + .map_err(|e| { + FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) + })? + .inner(); // Construct IndexedBlock Ok(Some(IndexedBlock::new( diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs index d10e2b31f..67e21468d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs @@ -2,6 +2,8 @@ use super::*; +use crate::ZainoVersionedSerde; + /// [`DbRead`] capability implementation for [`DbV1`]. /// /// This trait is the read-only surface used by higher layers. Methods typically delegate to @@ -122,3 +124,91 @@ impl DbV1 { // *** Internal DB methods *** } + +impl DbV1 { + /// Fetches and decodes one `StoredEntryVar` row keyed by an already-validated + /// height. Returns `Ok(None)` when the table has no row for the height; `label` + /// names the table in decode errors. + fn read_row( + &self, + table: lmdb::Database, + label: &str, + height_bytes: &[u8], + ) -> Result, FinalisedStateError> { + tokio::task::block_in_place(|| { + let txn = self.env.begin_ro_txn()?; + let raw = match txn.get(table, &height_bytes) { + Ok(val) => val, + Err(lmdb::Error::NotFound) => return Ok(None), + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) + .map_err(|e| FinalisedStateError::Custom(format!("{label} decode error: {e}")))?; + Ok(Some(entry.item)) + }) + } + + /// [`DbV1::read_row`] at a height that is first validated against the index. + pub(super) async fn read_row_at_height( + &self, + table: lmdb::Database, + label: &str, + height: Height, + ) -> Result, FinalisedStateError> { + let validated_height = self + .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) + .await?; + let height_bytes = validated_height.to_bytes()?; + self.read_row(table, label, &height_bytes) + } + + /// Cursor-scans and decodes every `StoredEntryVar` row in the validated + /// inclusive `start..=end` height range. + pub(super) async fn scan_rows( + &self, + table: lmdb::Database, + label: &str, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + if end.0 < start.0 { + return Err(FinalisedStateError::Custom( + "invalid block range: end < start".to_string(), + )); + } + + self.validate_block_range(start, end).await?; + let start_bytes = start.to_bytes()?; + let end_bytes = end.to_bytes()?; + + let raw_entries = tokio::task::block_in_place(|| { + let txn = self.env.begin_ro_txn()?; + let mut raw_entries = Vec::new(); + let mut cursor = match txn.open_ro_cursor(table) { + Ok(cursor) => cursor, + Err(lmdb::Error::NotFound) => { + return Err(FinalisedStateError::DataUnavailable(format!( + "{label} data missing from db" + ))); + } + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + for (k, v) in cursor.iter_from(&start_bytes[..]) { + if k > &end_bytes[..] { + break; + } + raw_entries.push(v.to_vec()); + } + Ok::>, FinalisedStateError>(raw_entries) + })?; + + raw_entries + .into_iter() + .map(|bytes| { + StoredEntryVar::::from_bytes(&bytes) + .map(|e| e.item) + .map_err(|e| FinalisedStateError::Custom(format!("{label} decode error: {e}"))) + }) + .collect() + } +} diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs index 6bdb3cd5a..901d40060 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs @@ -114,10 +114,10 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN { + if key.len() != AddrScript::latest_versioned_len()? { continue; } - if val.len() != StoredEntryFixed::::VERSIONED_LEN { + if val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } raw_records.push(val.to_vec()); @@ -215,8 +215,8 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN - || val.len() != StoredEntryFixed::::VERSIONED_LEN + if key.len() != AddrScript::latest_versioned_len()? + || val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } @@ -285,8 +285,8 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN - || val.len() != StoredEntryFixed::::VERSIONED_LEN + if key.len() != AddrScript::latest_versioned_len()? + || val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } @@ -373,8 +373,8 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN - || val.len() != StoredEntryFixed::::VERSIONED_LEN + if key.len() != AddrScript::latest_versioned_len()? + || val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } @@ -525,8 +525,8 @@ impl DbV1 { }; // If the seek landed on a different key, there are no candidates for this addr. - if cur_key.len() != AddrScript::VERSIONED_LEN - || &cur_key[..AddrScript::VERSIONED_LEN] != addr_script_bytes + if cur_key.len() != AddrScript::latest_versioned_len()? + || &cur_key[..AddrScript::latest_versioned_len()?] != addr_script_bytes { return Ok(results); } @@ -536,12 +536,13 @@ impl DbV1 { loop { // Validate lengths, same as original function. - if cur_key.len() != AddrScript::VERSIONED_LEN { + if cur_key.len() != AddrScript::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history key length mismatch".into(), )); } - if cur_val.len() != StoredEntryFixed::::VERSIONED_LEN { + if cur_val.len() != StoredEntryFixed::::latest_versioned_len()? + { return Err(FinalisedStateError::Custom( "address history value length mismatch".into(), )); @@ -584,8 +585,8 @@ impl DbV1 { Some(k) => k, None => break, }; - if k.len() != AddrScript::VERSIONED_LEN - || &k[..AddrScript::VERSIONED_LEN] != addr_script_bytes + if k.len() != AddrScript::latest_versioned_len()? + || &k[..AddrScript::latest_versioned_len()?] != addr_script_bytes { break; } @@ -714,7 +715,7 @@ impl DbV1 { // - [10] flags // - [11..=18] value // - [19..=50] checksum - if val.len() == StoredEntryFixed::::VERSIONED_LEN + if val.len() == StoredEntryFixed::::latest_versioned_len()? && val[2..6] == height_be { let flags = val[10]; @@ -728,7 +729,7 @@ impl DbV1 { break; } } - } else if val.len() != StoredEntryFixed::::VERSIONED_LEN { + } else if val.len() != StoredEntryFixed::::latest_versioned_len()? { tracing::warn!("bad addrhist dup (len={})", val.len()); } @@ -778,12 +779,12 @@ impl DbV1 { let mut cur = txn.open_rw_cursor(self.address_history)?; for (key, val) in cur.iter_dup_of(&addr_bytes)? { - if key.len() != AddrScript::VERSIONED_LEN { + if key.len() != AddrScript::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history key length mismatch".into(), )); } - if val.len() != StoredEntryFixed::::VERSIONED_LEN { + if val.len() != StoredEntryFixed::::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history value length mismatch".into(), )); @@ -793,7 +794,13 @@ impl DbV1 { continue; } - let mut hist_record = [0u8; StoredEntryFixed::::VERSIONED_LEN]; + let stored_entry_len = StoredEntryFixed::::latest_versioned_len()?; + if stored_entry_len != val.len() || stored_entry_len != 51 { + return Err(FinalisedStateError::Custom( + "address history value length mismatch".into(), + )); + } + let mut hist_record = [0u8; 51]; hist_record.copy_from_slice(val); let flags = hist_record[10]; @@ -846,12 +853,12 @@ impl DbV1 { let mut cur = txn.open_rw_cursor(self.address_history)?; for (key, val) in cur.iter_dup_of(&addr_bytes)? { - if key.len() != AddrScript::VERSIONED_LEN { + if key.len() != AddrScript::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history key length mismatch".into(), )); } - if val.len() != StoredEntryFixed::::VERSIONED_LEN { + if val.len() != StoredEntryFixed::::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history value length mismatch".into(), )); @@ -861,8 +868,13 @@ impl DbV1 { continue; } - // we've located the exact duplicate bytes we built earlier. - let mut hist_record = [0u8; StoredEntryFixed::::VERSIONED_LEN]; + let stored_entry_len = StoredEntryFixed::::latest_versioned_len()?; + if stored_entry_len != val.len() || stored_entry_len != 51 { + return Err(FinalisedStateError::Custom( + "address history value length mismatch".into(), + )); + } + let mut hist_record = [0u8; 51]; hist_record.copy_from_slice(val); // parse flags (located at byte index 10 in the StoredEntry layout) @@ -933,7 +945,7 @@ impl DbV1 { let height_key = Height(block_height).to_bytes()?; let stored_bytes = ro.get(self.transparent, &height_key)?; - Self::find_txout_in_stored_transparent_tx_list(stored_bytes, tx_index, out_index) + Self::find_txout_in_stored_transparent_tx_list(stored_bytes, tx_index, out_index)? .ok_or_else(|| { FinalisedStateError::Custom("Previous output not found at given index".into()) }) @@ -954,64 +966,70 @@ impl DbV1 { stored: &[u8], target_tx_idx: usize, target_output_idx: usize, - ) -> Option { + ) -> Result, FinalisedStateError> { const CHECKSUM_LEN: usize = 32; if stored.len() < TransactionHash::VERSION_TAG_LEN + 8 + CHECKSUM_LEN { - return None; + return Ok(None); } let mut cursor = &stored[TransactionHash::VERSION_TAG_LEN..]; - let item_len = CompactSize::read(&mut cursor).ok()? as usize; + let item_len = CompactSize::read(&mut cursor)? as usize; if cursor.len() < item_len + CHECKSUM_LEN { - return None; + return Ok(None); } - let (_record_version, mut remaining) = cursor.split_first()?; - let vec_len = CompactSize::read(&mut remaining).ok()? as usize; + let Some((_record_version, mut remaining)) = cursor.split_first() else { + return Ok(None); + }; + let vec_len = CompactSize::read(&mut remaining)? as usize; for i in 0..vec_len { - let (option_tag, rest) = remaining.split_first()?; + let Some((option_tag, rest)) = remaining.split_first() else { + return Ok(None); + }; remaining = rest; if *option_tag == 0 { // None: nothing to skip, go to next if i == target_tx_idx { - return None; + return Ok(None); } } else if *option_tag == 1 { - let (_tx_version, rest) = remaining.split_first()?; + let Some((_tx_version, rest)) = remaining.split_first() else { + return Ok(None); + }; remaining = rest; - let vin_len = CompactSize::read(&mut remaining).ok()? as usize; + let vin_len = CompactSize::read(&mut remaining)? as usize; for _ in 0..vin_len { - if remaining.len() < TxInCompact::VERSIONED_LEN { - return None; + if remaining.len() < TxInCompact::latest_versioned_len()? { + return Ok(None); } - remaining = &remaining[TxInCompact::VERSIONED_LEN..]; + remaining = &remaining[TxInCompact::latest_versioned_len()?..]; } - let vout_len = CompactSize::read(&mut remaining).ok()? as usize; + let vout_len = CompactSize::read(&mut remaining)? as usize; for out_idx in 0..vout_len { - if remaining.len() < TxOutCompact::VERSIONED_LEN { - return None; + if remaining.len() < TxOutCompact::latest_versioned_len()? { + return Ok(None); } - let out_bytes = &remaining[..TxOutCompact::VERSIONED_LEN]; + let out_bytes = &remaining[..TxOutCompact::latest_versioned_len()?]; if i == target_tx_idx && out_idx == target_output_idx { - return TxOutCompact::from_bytes(out_bytes).ok(); + return Ok(TxOutCompact::from_bytes(out_bytes).ok()); } - remaining = &remaining[TxOutCompact::VERSIONED_LEN..]; + remaining = &remaining[TxOutCompact::latest_versioned_len()?..]; } } else { // Non-canonical Option tag - return None; + return Ok(None); } } - None + Ok(None) } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs index e826d2631..df3aad9ca 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs @@ -1490,11 +1490,11 @@ impl DbV1 { Err(lmdb::Error::NotFound) => return Ok(None), Err(error) => return Err(FinalisedStateError::LmdbError(error)), }; - Ok(Self::find_txout_in_stored_transparent_tx_list( + Self::find_txout_in_stored_transparent_tx_list( stored, location.tx_index() as usize, outpoint.prev_index() as usize, - )) + ) } /// Fetches the full [`TransparentCompactTx`] for `txid`, read through `txn` (no new txn). @@ -1710,7 +1710,7 @@ mod tests { use std::sync::Arc; use tempfile::TempDir; use zaino_common::network::ActivationHeights; - use zaino_common::{DatabaseConfig, Network, StorageConfig, SyncWriteBatchSize}; + use zaino_common::{DatabaseConfig, StorageConfig, SyncWriteBatchSize}; fn p2pkh_out(value: u64) -> TxOutCompact { TxOutCompact::new(value, [0x11; 20], 0).expect("P2PKH script_type should be valid") @@ -2111,7 +2111,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source.clone()).await.unwrap(); @@ -2161,7 +2161,7 @@ mod tests { /// by a small range (`write_blocks_to_height`'s steady-state branch) — must produce exactly the /// accumulator a full from-genesis rebuild produces at the same tip, for all five fields. This is /// the correctness gate for `update_tx_out_set_accumulator_for_range`: with regtest coinbase - /// maturity of 100, splitting the sync at height 100 guarantees the second segment spends outputs + /// maturity `COINBASE_MATURITY`, splitting the sync at that height guarantees the second segment spends outputs /// created in the first (exercising the `transactions` "Set B" decrement) as well as outputs both /// created and spent within the range (the XOR-cancel case). #[tokio::test(flavor = "multi_thread")] @@ -2171,6 +2171,7 @@ mod tests { use crate::chain_index::finalised_state::capability::{ CapabilityRequest, DbRead, TransparentHistExt, }; + use zaino_common::consensus::COINBASE_MATURITY; let blocks = load_test_vectors().unwrap().blocks; let source = build_mockchain_source(blocks); @@ -2185,14 +2186,17 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source.clone()).await.unwrap(); - // First segment builds the accumulator to height 100 (no watermark yet => full rebuild), - // the second advances it by a 100-block range => the incremental update path under test. - zaino_db.sync_to_height(Height(100), &source).await.unwrap(); + // First segment builds the accumulator to `COINBASE_MATURITY` (no watermark yet => full + // rebuild), the second advances it to the fixture tip => the incremental update path under test. + zaino_db + .sync_to_height(Height(COINBASE_MATURITY), &source) + .await + .unwrap(); // Background catch-up (>LONG_RUNNING_SYNC_THRESHOLD); wait for the persistent build + watermark. zaino_db.wait_until_synced().await; @@ -2200,15 +2204,15 @@ mod tests { .backend_for_cap(CapabilityRequest::WriteCore) .unwrap(); - // The watermark must sit at 100 here: that (together with gap 100 <= the incremental cap) - // pins the next sync to the incremental branch rather than a silent rebuild fallback that - // would make the comparison below trivial. + // The watermark must sit at `COINBASE_MATURITY` here: that (together with the gap to the tip + // <= the incremental cap) pins the next sync to the incremental branch rather than a silent + // rebuild fallback that would make the comparison below trivial. assert_eq!( backend .read_tx_out_set_accumulator_built_height() .await .unwrap(), - Some(Height(100)), + Some(Height(COINBASE_MATURITY)), "first segment must leave the accumulator watermark at the synced tip" ); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs index 45d55b154..16cd7eabd 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs @@ -219,12 +219,32 @@ impl DbV1 { } } - // *** commitment_tree_data (fixed) *** + // *** ironwood *** + // + // Ironwood (NU6.3) was introduced in schema v1.3.0. Blocks written before that upgrade — and + // any block below NU6.3 activation — have no ironwood entry, so a missing row is valid and + // simply means "no ironwood data at this height". When an entry is present its checksum is + // verified like the other pools. + { + match ro.get(self.ironwood, &height_key) { + Ok(raw) => { + let entry = StoredEntryVar::::from_bytes(raw) + .map_err(|e| fail(&format!("ironwood corrupt data: {e}")))?; + if !entry.verify(&height_key) { + return Err(fail("ironwood checksum mismatch")); + } + } + Err(lmdb::Error::NotFound) => {} + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + } + } + + // *** commitment_tree_data (var) *** { let raw = ro .get(self.commitment_tree_data, &height_key) .map_err(FinalisedStateError::LmdbError)?; - let entry = StoredEntryFixed::::from_bytes(raw) + let entry = StoredEntryVar::::from_bytes(raw) .map_err(|e| fail(&format!("commitment_tree corrupt bytes: {e}")))?; if !entry.verify(&height_key) { return Err(fail("commitment_tree checksum mismatch")); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 016e00d64..c833d7abc 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -19,7 +19,8 @@ fn approx_indexed_block_bytes(block: &IndexedBlock) -> u64 { + transparent.outputs().len() + tx.sapling().spends().len() + tx.sapling().outputs().len() - + tx.orchard().actions().len(); + + tx.orchard().actions().len() + + tx.ironwood().actions().len(); 256 + items as u64 * 128 }) .sum() @@ -43,6 +44,177 @@ use crate::version; /// /// This trait represents the mutating surface (append / delete tip / update metadata). Writes are /// performed via LMDB write transactions and validated before becoming visible as “known-good”. +/// Per-transaction pool lists for one block's row entries, with the duplicate-txid +/// guard applied. +struct BlockPoolLists { + /// `(txid, transparent)` pairs — both halves come from one binding per + /// transaction, so misalignment is structurally impossible. + transactions: Vec<(TransactionHash, Option)>, + sapling: Vec>, + orchard: Vec>, + ironwood: Vec>, +} + +/// Builds the per-transaction pool lists for one block: each pool records +/// `Some(compact data)` for a transaction with data in that pool, `None` otherwise, +/// keeping every list index-aligned with the block's txids. +fn extract_block_pool_lists(block: &IndexedBlock) -> Result { + let block_height = block.context.index.height; + let block_hash = block.context.index.hash; + + let tx_len = block.transactions().len(); + let mut transactions: Vec<(TransactionHash, Option)> = + Vec::with_capacity(tx_len); + let mut txid_set: HashSet = HashSet::with_capacity(tx_len); + let mut sapling = Vec::with_capacity(tx_len); + let mut orchard = Vec::with_capacity(tx_len); + let mut ironwood = Vec::with_capacity(tx_len); + + for tx in block.transactions() { + let hash = tx.txid(); + if !txid_set.insert(*hash) { + return Err(FinalisedStateError::InvalidBlock { + height: block_height.0, + hash: block_hash, + reason: format!("duplicate transaction hash in block: {hash:?}"), + }); + } + + // Transparent transactions — paired with the txid at the source binding. + let transparent_data = + if tx.transparent().inputs().is_empty() && tx.transparent().outputs().is_empty() { + None + } else { + Some(tx.transparent().clone()) + }; + transactions.push((*hash, transparent_data)); + + // Sapling transactions + let sapling_data = if tx.sapling().spends().is_empty() && tx.sapling().outputs().is_empty() + { + None + } else { + Some(tx.sapling().clone()) + }; + sapling.push(sapling_data); + + // Orchard transactions + let orchard_data = if tx.orchard().actions().is_empty() { + None + } else { + Some(tx.orchard().clone()) + }; + orchard.push(orchard_data); + + // Ironwood transactions (NU6.3; modelled with the Orchard compact types). + let ironwood_data = if tx.ironwood().actions().is_empty() { + None + } else { + Some(tx.ironwood().clone()) + }; + ironwood.push(ironwood_data); + } + + Ok(BlockPoolLists { + transactions, + sapling, + orchard, + ironwood, + }) +} + +/// Cheap in-memory correctness check: the block's txids must reproduce the header's +/// merkle root. +fn verify_header_merkle_root( + txids: &[TransactionHash], + block: &IndexedBlock, +) -> Result<(), FinalisedStateError> { + let txid_bytes: Vec<[u8; 32]> = txids.iter().map(|txid| txid.0).collect(); + let computed_merkle_root = DbV1::calculate_block_merkle_root(&txid_bytes); + if &computed_merkle_root != block.data().merkle_root() { + return Err(FinalisedStateError::InvalidBlock { + height: block.context.index.height.0, + hash: block.context.index.hash, + reason: "header merkle root does not match block txids".to_string(), + }); + } + Ok(()) +} + +/// One block's row entries, ready to put. Everything is keyed by the block height +/// except `height_entry` (the hash-keyed height index). `ironwood_entry` is `None` +/// when the block has no ironwood data — the ironwood table is sparse; readers treat +/// an absent row as "no ironwood data". +struct BlockRowEntries { + height_entry: StoredEntryFixed, + header_entry: StoredEntryVar, + commitment_tree_entry: StoredEntryVar, + txid_entry: StoredEntryVar, + transparent_entry: StoredEntryVar, + sapling_entry: StoredEntryVar, + orchard_entry: StoredEntryVar, + ironwood_entry: Option>, +} + +#[allow(clippy::too_many_arguments)] +fn build_block_row_entries( + block: &IndexedBlock, + block_hash_bytes: &[u8], + block_height_bytes: &[u8], + txids: Vec, + transparent: Vec>, + sapling: Vec>, + orchard: Vec>, + ironwood: Vec>, +) -> BlockRowEntries { + BlockRowEntries { + height_entry: StoredEntryFixed::new(block_hash_bytes, block.context.index.height), + header_entry: StoredEntryVar::new( + block_height_bytes, + BlockHeaderData::new(block.context, *block.data()), + ), + // Stored as a `StoredEntryVar` because `CommitmentTreeData` V2 is + // variable-length (optional Ironwood root). + commitment_tree_entry: StoredEntryVar::new( + block_height_bytes, + *block.commitment_tree_data(), + ), + txid_entry: StoredEntryVar::new(block_height_bytes, TxidList::new(txids)), + transparent_entry: StoredEntryVar::new( + block_height_bytes, + TransparentTxList::new(transparent), + ), + sapling_entry: StoredEntryVar::new(block_height_bytes, SaplingTxList::new(sapling)), + orchard_entry: StoredEntryVar::new(block_height_bytes, OrchardTxList::new(orchard)), + ironwood_entry: ironwood_entry(ironwood, block_height_bytes), + } +} + +/// Builds the sparse ironwood row entry for a block's per-transaction ironwood list: `Some` only +/// when at least one transaction carries ironwood data, so a block with none stores no ironwood row +/// (readers treat an absent row as "no ironwood data"). +fn ironwood_entry( + ironwood: Vec>, + block_height_bytes: &[u8], +) -> Option> { + ironwood + .iter() + .any(Option::is_some) + .then(|| StoredEntryVar::new(block_height_bytes, OrchardTxList::new(ironwood))) +} + +/// Builds the sparse ironwood row entry for `block` (the same value the write path stores). Used by +/// the v1.2.1 → v1.3.0 migration to backfill the ironwood table from validator-fetched blocks. +pub(crate) fn build_block_ironwood_entry( + block: &IndexedBlock, + block_height_bytes: &[u8], +) -> Result>, FinalisedStateError> { + Ok(ironwood_entry( + extract_block_pool_lists(block)?.ironwood, + block_height_bytes, + )) +} + impl DbWrite for DbV1 { async fn write_block(&self, block: IndexedBlock) -> Result<(), FinalisedStateError> { self.write_block(block).await @@ -57,15 +229,17 @@ impl DbWrite for DbV1 { height: Height, source: &S, ) -> Result<(), FinalisedStateError> { - use crate::chain_index::finalised_state::build_indexed_block_from_source; - use zebra_chain::parameters::NetworkUpgrade; + use crate::chain_index::finalised_state::{ + build_indexed_block_from_source, PoolActivationHeights, + }; - let network = self.config.network; - let zebra_network = network.to_zebra_network(); - let sapling_activation_height = NetworkUpgrade::Sapling - .activation_height(&zebra_network) + let zebra_network = self.config.network.clone(); + let pool_activations = PoolActivationHeights::resolve(&zebra_network); + let sapling_activation_height = pool_activations + .sapling .expect("Sapling activation height must be set"); - let nu5_activation_height = NetworkUpgrade::Nu5.activation_height(&zebra_network); + let nu5_activation_height = pool_activations.nu5; + let nu6_3_activation_height = pool_activations.nu6_3; // Seed `parent_chainwork` from the current tip header (the block before the first one we // write). On an empty database this is genesis with zero chainwork. Read raw rather than via @@ -107,7 +281,7 @@ impl DbWrite for DbV1 { info!( start_height, target = height.0, - ?network, + ?zebra_network, "write_blocks_to_height: syncing finalised blocks" ); @@ -152,9 +326,10 @@ impl DbWrite for DbV1 { let build_start = std::time::Instant::now(); let block = build_indexed_block_from_source( source, - network, + zebra_network.clone(), sapling_activation_height, nu5_activation_height, + nu6_3_activation_height, next, parent_chainwork, ) @@ -178,7 +353,7 @@ impl DbWrite for DbV1 { info!( current = next - 1, target = height.0, - ?network, + ?zebra_network, "write_blocks_to_height: syncing" ); last_progress_log = std::time::Instant::now(); @@ -229,6 +404,7 @@ impl DbWrite for DbV1 { network, sapling_activation_height, nu5_activation_height, + nu6_3_activation_height, height_int, parent_chainwork, ) @@ -406,32 +582,17 @@ impl DbV1 { return Ok(()); } - // Build DBHeight - let height_entry = StoredEntryFixed::new(&block_hash_bytes, block.context.index.height); - - // Build header - let header_entry = StoredEntryVar::new( - &block_height_bytes, - BlockHeaderData::new(block.context, *block.data()), - ); - - // Build commitment tree data - let commitment_tree_entry = - StoredEntryFixed::new(&block_height_bytes, *block.commitment_tree_data()); + // Build the per-transaction pool lists (with the duplicate-txid guard applied). + // The accumulator consumes the paired `(txid, transparent)` slice; for storage + // the pairs are `unzip`ped into the `TxidList` / `TransparentTxList` shapes. + let pool_lists = extract_block_pool_lists(&block)?; - // Build transaction indexes. - // - // `transactions` pairs each transaction hash with its transparent data. Both halves - // are sourced from the same `tx` in the loop below, so misalignment is structurally - // impossible — the pair shares one binding. Downstream the accumulator consumes the - // paired slice; for storage we `unzip` into the existing `TxidList` / `TransparentTxList` - // shapes. - let tx_len = block.transactions().len(); - let mut transactions: Vec<(TransactionHash, Option)> = - Vec::with_capacity(tx_len); - let mut txid_set: HashSet = HashSet::with_capacity(tx_len); - let mut sapling = Vec::with_capacity(tx_len); - let mut orchard = Vec::with_capacity(tx_len); + #[cfg(feature = "transparent_address_history_experimental")] + let txid_set: HashSet = pool_lists + .transactions + .iter() + .map(|(hash, _)| *hash) + .collect(); let mut spent_map: HashMap = HashMap::new(); @@ -447,42 +608,6 @@ impl DbV1 { #[allow(clippy::unused_enumerate_index)] for (_tx_index, tx) in block.transactions().iter().enumerate() { - let hash = tx.txid(); - - if !txid_set.insert(*hash) { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: format!("duplicate transaction hash in block: {hash:?}"), - }); - } - - // Transparent transactions — paired with the txid at the source binding. - let transparent_data = - if tx.transparent().inputs().is_empty() && tx.transparent().outputs().is_empty() { - None - } else { - Some(tx.transparent().clone()) - }; - transactions.push((*hash, transparent_data)); - - // Sapling transactions - let sapling_data = - if tx.sapling().spends().is_empty() && tx.sapling().outputs().is_empty() { - None - } else { - Some(tx.sapling().clone()) - }; - sapling.push(sapling_data); - - // Orchard transactions - let orchard_data = if tx.orchard().actions().is_empty() { - None - } else { - Some(tx.orchard().clone()) - }; - orchard.push(orchard_data); - // Transaction location let tx_index = u16::try_from(_tx_index).map_err(|_| FinalisedStateError::InvalidBlock { @@ -526,7 +651,8 @@ impl DbV1 { let prev_tx_hash = TransactionHash(*prev_outpoint.prev_txid()); if txid_set.contains(&prev_tx_hash) { // Locate the paired (txid, transparent_data) within this block. - if let Some((tx_index, (_, Some(prev_transparent)))) = transactions + if let Some((tx_index, (_, Some(prev_transparent)))) = pool_lists + .transactions .iter() .enumerate() .find(|(_, (h, _))| h == &prev_tx_hash) @@ -589,12 +715,18 @@ impl DbV1 { .maybe_calculate_tx_out_set_info_accumulator_after_block( update_tx_out_set, block_height, - &transactions, + &pool_lists.transactions, &spent_map, ) .await?; // Split the paired vector into the per-table shapes used for storage. + let BlockPoolLists { + transactions, + sapling, + orchard, + ironwood, + } = pool_lists; let (txids, transparent): (Vec, Vec>) = transactions.into_iter().unzip(); @@ -603,17 +735,7 @@ impl DbV1 { // lets us mark the block validated after a successful write without the expensive // post-commit re-read + spent-index cross-check (which only re-verifies on-disk integrity of // bytes we just wrote from memory, and is redundant for our own writes). - { - let txid_bytes: Vec<[u8; 32]> = txids.iter().map(|txid| txid.0).collect(); - let computed_merkle_root = Self::calculate_block_merkle_root(&txid_bytes); - if &computed_merkle_root != block.data().merkle_root() { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: "header merkle root does not match block txids".to_string(), - }); - } - } + verify_header_merkle_root(&txids, &block)?; // Reverse txid index entries (`txid -> TxLocation`). Built before `txids` is moved into // the `TxidList` below, and sorted by txid so the random-keyed `txid_location` B-tree @@ -631,11 +753,16 @@ impl DbV1 { } txid_location_entries.sort_by_key(|entry| entry.0); - let txid_entry = StoredEntryVar::new(&block_height_bytes, TxidList::new(txids)); - let transparent_entry = - StoredEntryVar::new(&block_height_bytes, TransparentTxList::new(transparent)); - let sapling_entry = StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); - let orchard_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); + let entries = build_block_row_entries( + &block, + &block_hash_bytes, + &block_height_bytes, + txids, + transparent, + sapling, + orchard, + ironwood, + ); // if any database writes fail, or block validation fails, remove block from database and return err. let zaino_db = self.detached_handle(); @@ -646,21 +773,21 @@ impl DbV1 { txn.put( zaino_db.headers, &block_height_bytes, - &header_entry.to_bytes()?, + &entries.header_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.heights, &block_hash_bytes, - &height_entry.to_bytes()?, + &entries.height_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.txids, &block_height_bytes, - &txid_entry.to_bytes()?, + &entries.txid_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; @@ -678,28 +805,37 @@ impl DbV1 { txn.put( zaino_db.transparent, &block_height_bytes, - &transparent_entry.to_bytes()?, + &entries.transparent_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.sapling, &block_height_bytes, - &sapling_entry.to_bytes()?, + &entries.sapling_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.orchard, &block_height_bytes, - &orchard_entry.to_bytes()?, + &entries.orchard_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; + if let Some(ironwood_entry) = &entries.ironwood_entry { + txn.put( + zaino_db.ironwood, + &block_height_bytes, + &ironwood_entry.to_bytes()?, + WriteFlags::NO_OVERWRITE, + )?; + } + txn.put( zaino_db.commitment_tree_data, &block_height_bytes, - &commitment_tree_entry.to_bytes()?, + &entries.commitment_tree_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; @@ -1113,56 +1249,10 @@ impl DbV1 { } } - let height_entry = StoredEntryFixed::new(&block_hash_bytes, block_height); - let header_entry = StoredEntryVar::new( - &block_height_bytes, - BlockHeaderData::new(block.context, *block.data()), - ); - let commitment_tree_entry = - StoredEntryFixed::new(&block_height_bytes, *block.commitment_tree_data()); - - let tx_len = block.transactions().len(); - let mut txids: Vec = Vec::with_capacity(tx_len); - let mut txid_set: HashSet = HashSet::with_capacity(tx_len); - let mut transparent: Vec> = Vec::with_capacity(tx_len); - let mut sapling = Vec::with_capacity(tx_len); - let mut orchard = Vec::with_capacity(tx_len); + // Build the per-transaction pool lists (with the duplicate-txid guard applied). + let pool_lists = extract_block_pool_lists(block)?; for (tx_index, tx) in block.transactions().iter().enumerate() { - let hash = tx.txid(); - if !txid_set.insert(*hash) { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: format!("duplicate transaction hash in block: {hash:?}"), - }); - } - txids.push(*hash); - - let transparent_data = if tx.transparent().inputs().is_empty() - && tx.transparent().outputs().is_empty() - { - None - } else { - Some(tx.transparent().clone()) - }; - transparent.push(transparent_data); - - let sapling_data = - if tx.sapling().spends().is_empty() && tx.sapling().outputs().is_empty() { - None - } else { - Some(tx.sapling().clone()) - }; - sapling.push(sapling_data); - - let orchard_data = if tx.orchard().actions().is_empty() { - None - } else { - Some(tx.orchard().clone()) - }; - orchard.push(orchard_data); - let tx_index = u16::try_from(tx_index).map_err(|_| FinalisedStateError::InvalidBlock { height: block_height.0, @@ -1175,70 +1265,82 @@ impl DbV1 { spent_batch.push((prev_outpoint.to_bytes()?, tx_location)); } - txid_location_batch.push(((*hash).into(), tx_location)); + txid_location_batch.push(((*tx.txid()).into(), tx_location)); } + let BlockPoolLists { + transactions, + sapling, + orchard, + ironwood, + } = pool_lists; + let (txids, transparent): (Vec, Vec>) = + transactions.into_iter().unzip(); + // Cheap in-memory correctness check: txids must reproduce the header merkle root. - let txid_bytes: Vec<[u8; 32]> = txids.iter().map(|t| t.0).collect(); - let computed_merkle_root = Self::calculate_block_merkle_root(&txid_bytes); - if &computed_merkle_root != block.data().merkle_root() { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: "header merkle root does not match block txids".to_string(), - }); - } + verify_header_merkle_root(&txids, block)?; - let txid_entry = StoredEntryVar::new(&block_height_bytes, TxidList::new(txids)); - let transparent_entry = - StoredEntryVar::new(&block_height_bytes, TransparentTxList::new(transparent)); - let sapling_entry = - StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); - let orchard_entry = - StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); + let entries = build_block_row_entries( + block, + &block_hash_bytes, + &block_height_bytes, + txids, + transparent, + sapling, + orchard, + ironwood, + ); // Height-keyed tables (+ the hash-keyed `heights`, one entry/block) written per block. put_idempotent( &mut txn, self.headers, &block_height_bytes, - &header_entry.to_bytes()?, + &entries.header_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.heights, &block_hash_bytes, - &height_entry.to_bytes()?, + &entries.height_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.txids, &block_height_bytes, - &txid_entry.to_bytes()?, + &entries.txid_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.transparent, &block_height_bytes, - &transparent_entry.to_bytes()?, + &entries.transparent_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.sapling, &block_height_bytes, - &sapling_entry.to_bytes()?, + &entries.sapling_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.orchard, &block_height_bytes, - &orchard_entry.to_bytes()?, + &entries.orchard_entry.to_bytes()?, )?; + if let Some(ironwood_entry) = &entries.ironwood_entry { + put_idempotent( + &mut txn, + self.ironwood, + &block_height_bytes, + &ironwood_entry.to_bytes()?, + )?; + } put_idempotent( &mut txn, self.commitment_tree_data, &block_height_bytes, - &commitment_tree_entry.to_bytes()?, + &entries.commitment_tree_entry.to_bytes()?, )?; prev_height = Some(block_height.0); @@ -1436,7 +1538,8 @@ impl DbV1 { let prev_tx_hash = TransactionHash(*prev_outpoint.prev_txid()); if txid_set.contains(&prev_tx_hash) { // Locate the paired (txid, transparent_data) within this block. - if let Some((tx_index, (_, Some(prev_transparent)))) = transactions + if let Some((tx_index, (_, Some(prev_transparent)))) = pool_lists + .transactions .iter() .enumerate() .find(|(_, (h, _))| h == &prev_tx_hash) @@ -1645,13 +1748,15 @@ impl DbV1 { } } - // Delete block data + // Delete block data. `ironwood` is `NotFound`-tolerant below, so deleting a pre-v1.3.0 + // block (which has no ironwood row) is a no-op. for &db in &[ zaino_db.headers, zaino_db.txids, zaino_db.transparent, zaino_db.sapling, zaino_db.orchard, + zaino_db.ironwood, zaino_db.commitment_tree_data, ] { match txn.del(db, &block_height_bytes, None) { diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index cb7d4bd58..3e5f5bd51 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -160,7 +160,7 @@ use crate::{ }, config::ChainIndexConfig, error::FinalisedStateError, - Height, TransparentTxList, TxLocation, TxidList, ZainoVersionedSerde as _, + CommitmentTreeData, Height, TransparentTxList, TxLocation, TxidList, ZainoVersionedSerde as _, }; use lmdb::{Transaction, WriteFlags}; @@ -343,7 +343,7 @@ impl MigrationManager { .router .init_or_take_ephemeral( self.source.clone(), - self.cfg.network.to_zebra_network(), + self.cfg.network.clone(), EphemeralMode::Full, db_height, ) @@ -378,6 +378,7 @@ impl MigrationManager { (1, 0, 0) => Ok(MigrationStep::Migration1_0_0To1_1_0(Migration1_0_0To1_1_0)), (1, 1, 0) => Ok(MigrationStep::Migration1_1_0To1_2_0(Migration1_1_0To1_2_0)), (1, 2, 0) => Ok(MigrationStep::Migration1_2_0To1_2_1(Migration1_2_0To1_2_1)), + (1, 2, 1) => Ok(MigrationStep::Migration1_2_1To1_3_0(Migration1_2_1To1_3_0)), (_, _, _) => Err(FinalisedStateError::Custom(format!( "Missing migration from version {}", self.current_version @@ -395,6 +396,7 @@ enum MigrationStep { Migration1_0_0To1_1_0(Migration1_0_0To1_1_0), Migration1_1_0To1_2_0(Migration1_1_0To1_2_0), Migration1_2_0To1_2_1(Migration1_2_0To1_2_1), + Migration1_2_1To1_3_0(Migration1_2_1To1_3_0), } impl MigrationStep { @@ -409,6 +411,9 @@ impl MigrationStep { MigrationStep::Migration1_2_0To1_2_1(_step) => { >::TO_VERSION } + MigrationStep::Migration1_2_1To1_3_0(_step) => { + >::TO_VERSION + } } } @@ -423,6 +428,9 @@ impl MigrationStep { MigrationStep::Migration1_2_0To1_2_1(step) => { >::migration_type(step) } + MigrationStep::Migration1_2_1To1_3_0(step) => { + >::migration_type(step) + } } } @@ -436,6 +444,7 @@ impl MigrationStep { MigrationStep::Migration1_0_0To1_1_0(step) => step.migrate(router, cfg, source).await, MigrationStep::Migration1_1_0To1_2_0(step) => step.migrate(router, cfg, source).await, MigrationStep::Migration1_2_0To1_2_1(step) => step.migrate(router, cfg, source).await, + MigrationStep::Migration1_2_1To1_3_0(step) => step.migrate(router, cfg, source).await, } } } @@ -482,6 +491,34 @@ impl Migration for Migration1_0_0To1_1_0 { /// - No shadow database. struct Migration1_1_0To1_2_0; +/// Writes `value` under `key` with `NO_OVERWRITE`, tolerating an existing byte-identical row. +/// +/// Migrations use this to stay idempotent on crash-resume: a resumed pass may revisit rows it has +/// already committed, and each such row must match the rebuilt bytes exactly. Any difference — +/// a conflicting rebuild or a corrupt existing row — aborts the migration with the message +/// `describe` produces. `describe` runs only on that error path, so the per-row success path +/// allocates nothing. +fn put_idempotent( + txn: &mut lmdb::RwTransaction<'_>, + db: lmdb::Database, + key: &[u8], + value: &[u8], + describe: impl FnOnce() -> String, +) -> Result<(), FinalisedStateError> { + match txn.put(db, &key, &value, WriteFlags::NO_OVERWRITE) { + Ok(()) => Ok(()), + Err(lmdb::Error::KeyExist) => { + let existing = txn.get(db, &key).map_err(FinalisedStateError::LmdbError)?; + if existing == value { + Ok(()) + } else { + Err(FinalisedStateError::Custom(describe())) + } + } + Err(error) => Err(FinalisedStateError::LmdbError(error)), + } +} + /// Flushes a buffered batch of `spent` index entries in sorted key order, then commits them /// together with the Stage B progress watermark and fsyncs. /// @@ -502,26 +539,12 @@ fn flush_migration_spent_batch( let mut txn = env.begin_rw_txn()?; for (outpoint_bytes, tx_location) in buffer.iter() { let entry_bytes = StoredEntryFixed::new(outpoint_bytes, *tx_location).to_bytes()?; - match txn.put( - spent_db, - outpoint_bytes, - &entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - Err(lmdb::Error::KeyExist) => { - let existing = txn - .get(spent_db, outpoint_bytes) - .map_err(FinalisedStateError::LmdbError)?; - if existing != entry_bytes { - return Err(FinalisedStateError::Custom(format!( - "conflicting existing spent entry during batched migration for outpoint {}", - hex::encode(outpoint_bytes) - ))); - } - } - Err(error) => return Err(FinalisedStateError::LmdbError(error)), - } + put_idempotent(&mut txn, spent_db, outpoint_bytes, &entry_bytes, || { + format!( + "conflicting existing spent entry during batched migration for outpoint {}", + hex::encode(outpoint_bytes) + ) + })?; } let progress = StoredEntryFixed::new(progress_key, up_to_height + 1); @@ -680,42 +703,20 @@ impl Migration for Migration1_1_0To1_2_0 { let entry_bytes = StoredEntryFixed::new(txid_bytes, *tx_location).to_bytes()?; - match txn.put( + // Idempotent on resume: an existing entry must match byte-for-byte, which + // also rejects a corrupt existing row (its checksum bytes would differ). + put_idempotent( + &mut txn, txid_location_db, txid_bytes, &entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - - // Idempotent on resume: an existing entry must match exactly. - Err(lmdb::Error::KeyExist) => { - let existing_bytes = txn - .get(txid_location_db, txid_bytes) - .map_err(FinalisedStateError::LmdbError)?; - let existing_entry = - StoredEntryFixed::::from_bytes(existing_bytes) - .map_err(|error| { - FinalisedStateError::Custom(format!( - "corrupt existing txid_location entry: {error}" - )) - })?; - if !existing_entry.verify(txid_bytes) { - return Err(FinalisedStateError::Custom( - "existing txid_location entry checksum mismatch" - .to_string(), - )); - } - if existing_entry.inner() != tx_location { - return Err(FinalisedStateError::Custom(format!( - "conflicting txid_location entry at height {}", - height.0 - ))); - } - } - - Err(error) => return Err(FinalisedStateError::LmdbError(error)), - } + || { + format!( + "conflicting or corrupt existing txid_location entry at height {}", + height.0 + ) + }, + )?; } let progress = @@ -1004,3 +1005,308 @@ impl Migration for Migration1_2_0To1_2_1 { patch: 1, }; } + +/// Minor migration: v1.2.1 → v1.3.0. +/// +/// Introduces the Ironwood (NU6.3) shielded pool to the finalised state: +/// - a new per-height `ironwood` table (`ironwood_1_3_0`), created empty by `DbV1::spawn`. New +/// blocks populate it via the write path; here it is backfilled for any stored block at or above +/// NU6.3 activation (see below). +/// - the `commitment_tree_data` table is rebuilt from the legacy fixed-length +/// `StoredEntryFixed` (V1) rows into the variable-length +/// `StoredEntryVar` (V2) layout, which carries the optional Ironwood root and +/// size. The new rows are written to `commitment_tree_data_1_3_0` (the primary's +/// `commitment_tree_data` handle), then the legacy table is cleared. +/// +/// Per-height rebuild strategy, branching on NU6.3 activation: +/// - **Below activation:** rebuilt in place from the legacy on-disk commitment row (Ironwood +/// root/size default to `None`/`0` via `CommitmentTreeData` V1 decode). No validator access. +/// - **At or above activation:** the legacy data predates Ironwood, so both the commitment row +/// (now carrying the Ironwood root/size) and the sparse ironwood row are rebuilt from +/// validator-fetched block data via [`build_indexed_block_from_source`]. This lets the migration +/// run on a database already synced past NU6.3, rather than forcing a full re-index. +/// +/// Safety and resumability: +/// - Deterministic: a below-activation row is derived only from its legacy row; an at/above-activation +/// row is refetched from immutable finalised history, so a resumed rebuild reproduces the same bytes. +/// - Resumable: the next height to rebuild is stored in the metadata DB under a temporary key. +/// - Crash-safe: each height's rebuilt rows and the progress update commit in the same transaction; +/// idempotent on resume (`NO_OVERWRITE` + verify-match). +struct Migration1_2_1To1_3_0; + +impl Migration for Migration1_2_1To1_3_0 { + const CURRENT_VERSION: DbVersion = DbVersion { + major: 1, + minor: 2, + patch: 1, + }; + + const TO_VERSION: DbVersion = DbVersion { + major: 1, + minor: 3, + patch: 0, + }; + + fn migration_type(&self) -> MigrationType { + MigrationType::Minor + } + + async fn migrate( + &self, + router: Arc>, + cfg: ChainIndexConfig, + source: T, + ) -> Result<(), FinalisedStateError> { + use lmdb::DatabaseFlags; + + use crate::chain_index::finalised_state::{ + build_indexed_block_from_source, + finalised_source::v1::write_core::build_block_ironwood_entry, PoolActivationHeights, + }; + + // Temporary metadata entry recording the next height to rebuild, removed on completion. + const MIGRATION_CTD_PROGRESS_KEY: &[u8] = + b"_migration_commitment_tree_data_progress_1_3_0_next_height"; + + info!("Starting v1.2.1 → v1.3.0 migration (Ironwood)."); + + // Use the persistent primary directly (an ephemeral passthrough serves reads during the + // migration; `backend(WriteCore)` would route there and has no LMDB env). + let backend = router.primary_backend(); + let env = backend.env()?; + let metadata_db = backend.metadata_db()?; + // The primary's `commitment_tree_data` handle is the new `commitment_tree_data_1_3_0` table; + // `ironwood_db` is the new (v1.3.0) sparse ironwood table backfilled below. + let new_ctd_db = backend.commitment_tree_data_db()?; + let ironwood_db = backend.ironwood_db()?; + + // Open the legacy fixed-length commitment table by name. On a pre-v1.3.0 database it already + // exists; `open_or_create_db` creating it empty on an unexpected fresh DB is harmless (the + // rebuild loop below only runs when a tip exists, and an empty legacy table yields no rows). + let legacy_ctd_db = + crate::chain_index::finalised_state::finalised_source::open_or_create_db( + &env, + "commitment_tree_data_1_0_0", + DatabaseFlags::empty(), + ) + .await?; + + // Network-upgrade activation heights (shared with `write_blocks_to_height`). Blocks at or + // above NU6.3 have their ironwood root/size and ironwood tx list rebuilt from the validator + // (the legacy on-disk data predates ironwood); blocks below it are rebuilt in place from + // the legacy commitment row, with no ironwood. + let network = cfg.network.clone(); + let pool_activations = PoolActivationHeights::resolve(&network); + let sapling_activation_height = pool_activations.sapling; + let nu5_activation_height = pool_activations.nu5; + let nu6_3_activation_height = pool_activations.nu6_3; + + // Mark migration in progress (observability only; resumption uses the progress key). + { + let mut metadata: DbMetadata = backend.get_metadata().await?; + if metadata.migration_status == MigrationStatus::Empty { + metadata.migration_status = MigrationStatus::PartialBuidInProgress; + backend.update_metadata(metadata).await?; + } + } + + // Reads the temporary progress height, returning `None` if the key is absent. + let read_progress = |key: &[u8]| -> Result, FinalisedStateError> { + let txn = env.begin_ro_txn()?; + match txn.get(metadata_db, &key) { + Ok(bytes) => { + let entry = StoredEntryFixed::::from_bytes(bytes).map_err(|error| { + FinalisedStateError::Custom(format!( + "corrupt v1.3.0 migration progress entry: {error}" + )) + })?; + if !entry.verify(key) { + return Err(FinalisedStateError::Custom( + "v1.3.0 migration progress checksum mismatch".to_string(), + )); + } + Ok(Some(entry.inner().0)) + } + Err(lmdb::Error::NotFound) => Ok(None), + Err(error) => Err(FinalisedStateError::LmdbError(error)), + } + }; + + // Nothing to rebuild on an empty database; fall through to finalisation. + if let Some(db_tip) = backend.db_height().await? { + let db_tip = db_tip.0; + + let mut next_height = + read_progress(MIGRATION_CTD_PROGRESS_KEY)?.unwrap_or(GENESIS_HEIGHT.0); + + info!( + resume_height = next_height, + db_tip, + "v1.3.0 migration: rebuilding commitment_tree_data into StoredEntryVar (V2)" + ); + let started = std::time::Instant::now(); + + while next_height <= db_tip { + let height = Height::try_from(next_height) + .map_err(|error| FinalisedStateError::Custom(error.to_string()))?; + let height_bytes = height.to_bytes()?; + + let ironwood_active = + nu6_3_activation_height.is_some_and(|activation| next_height >= activation.0); + + // Prepare the rows to write. Reads / validator fetches happen before the write txn + // so the commit stays short. + let commitment_bytes: Vec; + let ironwood_bytes: Option>; + + if ironwood_active { + // Post-NU6.3: the legacy row carries no ironwood root/size and the ironwood + // table has no row, so rebuild both from validator-fetched block data. + let sapling_activation_height = sapling_activation_height.ok_or_else(|| { + FinalisedStateError::Custom( + "Sapling activation height must be set to backfill ironwood" + .to_string(), + ) + })?; + let block = build_indexed_block_from_source( + &source, + network.clone(), + sapling_activation_height, + nu5_activation_height, + nu6_3_activation_height, + next_height, + // Chainwork is irrelevant here: only the commitment-tree and ironwood rows + // are extracted, and neither depends on it. + None, + ) + .await + .map_err(|error| { + FinalisedStateError::Custom(format!( + "v1.3.0 ironwood backfill failed at height {next_height}: {error}. \ + This backfill refetches every stored block from NU6.3 activation \ + through the database tip; ensure the backing validator serves that \ + range, or wipe the finalised-state directory and re-index from the \ + validator." + )) + })?; + + commitment_bytes = + StoredEntryVar::new(&height_bytes, *block.commitment_tree_data()) + .to_bytes()?; + ironwood_bytes = match build_block_ironwood_entry(&block, &height_bytes)? { + Some(entry) => Some(entry.to_bytes()?), + None => None, + }; + } else { + // Pre-NU6.3: rebuild the commitment row in place from the legacy fixed-length + // row (ironwood defaults to none). + let commitment_tree_data: CommitmentTreeData = { + let txn = env.begin_ro_txn()?; + let raw = txn + .get(legacy_ctd_db, &height_bytes) + .map_err(FinalisedStateError::LmdbError)?; + let entry = StoredEntryFixed::::from_bytes(raw) + .map_err(|error| { + FinalisedStateError::Custom(format!( + "legacy commitment_tree_data corrupt data: {error}" + )) + })?; + if !entry.verify(&height_bytes) { + return Err(FinalisedStateError::Custom( + "legacy commitment_tree_data checksum mismatch".to_string(), + )); + } + *entry.inner() + }; + commitment_bytes = + StoredEntryVar::new(&height_bytes, commitment_tree_data).to_bytes()?; + ironwood_bytes = None; + } + + // Write commitment (+ ironwood) and advance progress atomically. + { + let mut txn = env.begin_rw_txn()?; + put_idempotent( + &mut txn, + new_ctd_db, + &height_bytes, + &commitment_bytes, + || { + format!( + "conflicting rebuilt commitment_tree_data at height {}", + height.0 + ) + }, + )?; + if let Some(bytes) = &ironwood_bytes { + put_idempotent(&mut txn, ironwood_db, &height_bytes, bytes, || { + format!("conflicting rebuilt ironwood at height {}", height.0) + })?; + } + + let progress = StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); + txn.put( + metadata_db, + &MIGRATION_CTD_PROGRESS_KEY, + &progress.to_bytes()?, + WriteFlags::empty(), + )?; + txn.commit()?; + } + + if next_height % SYNC_CHECKPOINT_INTERVAL == 0 { + env.sync(true)?; + } + if next_height % 50_000 == 0 { + info!( + height = next_height, + db_tip, + elapsed = ?started.elapsed(), + "v1.3.0 migration progress" + ); + } + + next_height = height.0 + 1; + } + + env.sync(true)?; + info!( + db_tip, + elapsed = ?started.elapsed(), + "v1.3.0 migration: commitment_tree_data rebuild complete" + ); + + // Drop the legacy commitment data: clearing frees its pages back to the env freelist. + { + let mut txn = env.begin_rw_txn()?; + txn.clear_db(legacy_ctd_db)?; + txn.commit()?; + } + env.sync(true)?; + } + + // Finalise: advance the version durably (the completion gate) before removing the progress + // key, so a crash re-selects and cheaply resumes this migration rather than skipping it. + env.sync(true)?; + let mut metadata: DbMetadata = backend.get_metadata().await?; + metadata.version = >::TO_VERSION; + metadata.schema_hash = + crate::chain_index::finalised_state::finalised_source::v1::DB_SCHEMA_V1_HASH; + metadata.migration_status = MigrationStatus::Empty; + backend.update_metadata(metadata).await?; + env.sync(true)?; + + { + let mut txn = env.begin_rw_txn()?; + match txn.del(metadata_db, &MIGRATION_CTD_PROGRESS_KEY, None) { + Ok(()) | Err(lmdb::Error::NotFound) => {} + Err(error) => return Err(FinalisedStateError::LmdbError(error)), + } + txn.commit()?; + } + env.sync(true)?; + + info!("v1.2.1 to v1.3.0 migration complete."); + Ok(()) + } +} diff --git a/packages/zaino-state/src/chain_index/finalised_state/reader.rs b/packages/zaino-state/src/chain_index/finalised_state/reader.rs index 1952d2a1a..c2885b4b5 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/reader.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/reader.rs @@ -313,6 +313,37 @@ impl DbReader { .await } + /// Fetch the serialized ironwood (NU6.3) compact tx for the given TxLocation, if present. + pub(crate) async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + self.db(CapabilityRequest::BlockShieldedExt)? + .get_ironwood(tx_location) + .await + } + + /// Fetch block ironwood transaction data by height. + pub(crate) async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + self.db(CapabilityRequest::BlockShieldedExt)? + .get_block_ironwood(height) + .await + } + + /// Fetches block ironwood tx data for the given height range. + pub(crate) async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + self.db(CapabilityRequest::BlockShieldedExt)? + .get_block_range_ironwood(start, end) + .await + } + /// Fetch block commitment tree data by height. pub(crate) async fn get_block_commitment_tree_data( &self, diff --git a/packages/zaino-state/src/chain_index/non_finalised_state.rs b/packages/zaino-state/src/chain_index/non_finalised_state.rs index 5e473e977..aee18b01a 100644 --- a/packages/zaino-state/src/chain_index/non_finalised_state.rs +++ b/packages/zaino-state/src/chain_index/non_finalised_state.rs @@ -1,7 +1,7 @@ use super::{ finalised_state::{reader::DbReader, FinalisedState}, source::BlockchainSource, - NON_FINALIZED_DEPTH, + OPERATIONAL_NFS_DEPTH, }; #[cfg(feature = "prometheus")] use crate::metric_names::*; @@ -26,14 +26,14 @@ use zebra_state::HashOrHeight; /// but that height can lag far behind the tip while the finalised DB syncs in the background, and /// is pinned at `0` in ephemeral mode. Without an independent floor the snapshot would grow by one /// block per new block indefinitely. This caps retention to a fixed window regardless, a small -/// margin above [`NON_FINALIZED_DEPTH`] so it never trims inside the reorg-possible range. +/// margin above [`OPERATIONAL_NFS_DEPTH`] so it never trims inside the reorg-possible range. /// /// It also bounds the non-finalised ancestry walkers ([`NonFinalizedState::handle_reorg`] and /// [`NonFinalizedState::add_nonbest_block`]): neither should recurse further back than the window /// they maintain. The bound is load-bearing for `add_nonbest_block` on the state backend, where /// `source.get_block` serves *any* block by hash (including finalised blocks below the window), so /// without it a side chain rooted below the anchor would recurse to genesis and overflow the stack. -const MAX_NFS_DEPTH: u32 = NON_FINALIZED_DEPTH + 10; +const MAX_NFS_DEPTH: u32 = OPERATIONAL_NFS_DEPTH + 10; /// Holds the block cache #[derive(Debug)] @@ -99,7 +99,7 @@ impl ChainIndexSnapshot { #[derive(Debug, Clone)] /// A snapshot of the nonfinalized state as it existed when this was created. pub(crate) struct NonfinalizedBlockCacheSnapshot { - /// the set of all known blocks < 100 blocks old + /// the set of all known blocks less than `OPERATIONAL_NFS_DEPTH` blocks old /// this includes all blocks on-chain, as well as /// all blocks known to have been on-chain before being /// removed by a reorg. Blocks reorged away have no height. @@ -320,7 +320,7 @@ impl NonFinalizedState { .map_err(|e| InitError::InvalidNodeData(Box::new(e)))? .ok_or_else(|| InitError::InvalidNodeData(Box::new(MissingGenesisBlock)))?; - let (sapling_root_and_len, orchard_root_and_len) = source + let (sapling_root_and_len, orchard_root_and_len, ironwood_root_and_len) = source .get_commitment_tree_roots(genesis_block.hash().into()) .await .map_err(|e| InitError::InvalidNodeData(Box::new(e)))?; @@ -328,6 +328,7 @@ impl NonFinalizedState { let tree_roots = TreeRootData { sapling: sapling_root_and_len, orchard: orchard_root_and_len, + ironwood: ironwood_root_and_len, }; // Genesis has no parent — pass None so create_block_context computes @@ -385,10 +386,14 @@ impl NonFinalizedState { )) })?; - let (sapling, orchard) = source + let (sapling, orchard, ironwood) = source .get_commitment_tree_roots(block.hash().into()) .await?; - let tree_roots = TreeRootData { sapling, orchard }; + let tree_roots = TreeRootData { + sapling, + orchard, + ironwood, + }; Self::create_indexed_block_with_optional_roots( block.as_ref(), @@ -433,7 +438,7 @@ impl NonFinalizedState { ) -> Result<(), SyncError> { let mut initial_state = self.get_snapshot(); let local_finalized_tip = finalized_db.to_reader().db_height().await?; - // Anchor floor: the non-finalised state must never start more than `NON_FINALIZED_DEPTH` + // Anchor floor: the non-finalised state must never start more than `OPERATIONAL_NFS_DEPTH` // blocks below the chain tip, even when the finalised DB tip lags far behind during // background catch-up. Without this floor a freshly-initialised (or genesis-fallback) // snapshot would try to bridge the entire gap from the finalised tip up to the chain tip one @@ -444,7 +449,7 @@ impl NonFinalizedState { local_finalized_tip .map(|height| height.0) .unwrap_or(0) - .max(u32::from(chain_height).saturating_sub(NON_FINALIZED_DEPTH)), + .max(u32::from(chain_height).saturating_sub(OPERATIONAL_NFS_DEPTH)), ); if initial_state.best_tip.height.0 < anchor_height.0 { let anchor_block = Self::resolve_anchor_block( @@ -520,7 +525,7 @@ impl NonFinalizedState { // we need to work backwards from it and update heights_to_hashes // with it and all its parents. } - if initial_state.best_tip.height + NON_FINALIZED_DEPTH + if initial_state.best_tip.height + OPERATIONAL_NFS_DEPTH < working_snapshot.best_tip.height { self.update(finalized_db.clone(), initial_state, working_snapshot) @@ -606,7 +611,7 @@ impl NonFinalizedState { height_to_recurse_to: Option, ) -> Result<(), SyncError> { if height_to_recurse_to - .is_some_and(|height| height + 100 < working_snapshot.best_tip.height) + .is_some_and(|height| height + MAX_NFS_DEPTH < working_snapshot.best_tip.height) { return Err(SyncError::ReorgFailure( "reorg detection recursed beyond reason".to_string(), @@ -809,12 +814,13 @@ impl NonFinalizedState { &self, block_hash: BlockHash, ) -> Result { - let (sapling_root_and_len, orchard_root_and_len) = + let (sapling_root_and_len, orchard_root_and_len, ironwood_root_and_len) = self.source.get_commitment_tree_roots(block_hash).await?; Ok(TreeRootData { sapling: sapling_root_and_len, orchard: orchard_root_and_len, + ironwood: ironwood_root_and_len, }) } @@ -829,7 +835,7 @@ impl NonFinalizedState { parent_chainwork: Option, network: Network, ) -> Result { - let (sapling_root, sapling_size, orchard_root, orchard_size) = + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood) = tree_roots.clone().extract_with_defaults(); let metadata = BlockMetadata::new( @@ -837,6 +843,7 @@ impl NonFinalizedState { sapling_size as u32, orchard_root, orchard_size as u32, + ironwood.map(|(root, size)| (root, size as u32)), parent_chainwork, network, ); diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 73e3e13de..b1fa80e76 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -1,21 +1,25 @@ //! Traits and types for the blockchain source thats serves zaino, commonly a validator connection. -use std::{error::Error, str::FromStr as _, sync::Arc}; +use std::{error::Error, sync::Arc}; use crate::chain_index::{ types::{BlockHash, TransactionHash}, ShieldedPool, }; use crate::SendFut; -use futures::{future::join, TryFutureExt as _}; +use futures::TryFutureExt as _; use incrementalmerkletree::frontier::CommitmentTree; use tower::{Service, ServiceExt as _}; -use zaino_common::Network; use zaino_fetch::jsonrpsee::{ connector::{JsonRpSeeConnector, RpcRequestError}, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, - GetBlockError, GetBlockResponse, GetTransactionResponse, GetTreestateResponse, + block_header::GetBlockHeader, + block_subsidy::GetBlockSubsidy, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + GetBlockError, GetBlockResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, + GetSpentInfoResponse, GetTransactionResponse, GetTreestateResponse, GetTxOutResponse, }, }; use zcash_primitives::merkle_tree::{read_commitment_tree, write_commitment_tree}; @@ -24,7 +28,9 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos}, + methods::{ + AddressBalance, GetAddressUtxos, GetBlockchainInfoResponse, GetInfo, SentTransactionHash, + }, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; @@ -34,15 +40,29 @@ pub(crate) mod mockchain_source; pub mod validator_connector; pub use validator_connector::*; -/// Serialized sapling and orchard treestates `(sapling, orchard)`, each `None` -/// when the pool has no treestate at the queried block. -pub(crate) type TreestateBytes = (Option>, Option>); +/// One pool's treestate for a block, as reported by the backing validator. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PoolTreestate { + /// The pool's note commitment tree root (32 bytes), when the validator reports one. + pub final_root: Option>, + /// The pool's serialized note commitment tree. + pub final_state: Vec, +} + +/// Per-pool treestates `(sapling, orchard, ironwood)`, each `None` when the pool has no +/// treestate at the queried block. +pub(crate) type TreestateBytes = ( + Option, + Option, + Option, +); -/// Sapling and orchard note-commitment tree roots `(sapling, orchard)`, each +/// Sapling and orchard note-commitment tree roots `(sapling, orchard, ironwood)`, each /// paired with its tree size; `None` when the pool has no root at the block. pub(crate) type ShieldedTreeRoots = ( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, ); /// Receiver for newly observed nonfinalized blocks, delivered as `(hash, block)`. @@ -61,6 +81,33 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { id: HashOrHeight, ) -> impl SendFut>>>; + /// Returns the `getblock`-shaped verbose block for the given hash or height. + /// + /// `verbosity` follows the zcashd `getblock` convention (0 = raw, 1 = object with + /// txids, 2 = object with full transaction data). + fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> impl SendFut>; + + /// Returns the `getblockheader`-shaped block header for the given block hash. + /// + /// When `verbose` is false the header is returned in raw hex form; when true it is + /// returned as a structured object. + fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> impl SendFut>; + + /// Returns the `getblockdeltas`-shaped transparent input/output deltas for the block + /// with the given hash. + fn get_block_deltas( + &self, + hash: String, + ) -> impl SendFut>; + // ********** Transaction methods ********** /// Returns the transaction by txid @@ -93,6 +140,88 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { &self, ) -> impl SendFut>>; + /// Returns the proof-of-work difficulty of the best chain as a multiple of the + /// minimum difficulty (the `getdifficulty` RPC value). + fn get_difficulty(&self) -> impl SendFut>; + + /// A watch stream of the source's chain-tip changes, when the source owns + /// one locally. Only the `Direct` validator connection (which drives its + /// own Zebra syncer) does; every other source observes tips by polling, + /// and inherits this `None` default. + fn chain_tip_change(&self) -> Option { + None + } + + /// Returns the `getblockchaininfo` response. + fn get_blockchain_info( + &self, + ) -> impl SendFut>; + + // ********** Node-passthrough methods ********** + // + // These have no local-index equivalent and always proxy to the backing validator's + // JSON-RPC interface. + + /// Returns the `getinfo` response. + fn get_info(&self) -> impl SendFut>; + + /// Returns the `getpeerinfo` response. + fn get_peer_info(&self) -> impl SendFut>; + + /// Returns the validator's `getchaintips` response. Serves as the + /// `getchaintips` fallback while the local index is still building its + /// finalised state and has no non-finalised snapshot to answer from. + fn get_chain_tips( + &self, + ) -> impl SendFut< + BlockchainSourceResult, + >; + + /// Returns the `getblocksubsidy` response at the given height. + fn get_block_subsidy( + &self, + height: u32, + ) -> impl SendFut>; + + /// Returns the `getmininginfo` response. + fn get_mining_info(&self) -> impl SendFut>; + + /// Returns the `gettxout` response for the given outpoint. + fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> impl SendFut>; + + /// Returns the `getspentinfo` response for the given request. + fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> impl SendFut>; + + /// Returns the `getnetworksolps` response. + fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> impl SendFut>; + + /// Submits a raw transaction to the network via the validator's mempool + /// (`sendrawtransaction`). + fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> impl SendFut>; + + /// Returns the full `z_gettreestate` response for the given hash-or-height string. + /// + /// Node-passthrough fallback for treestates not locally serviceable by the index. + fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> impl SendFut>; + /// Returns the sapling and orchard treestate by hash fn get_treestate(&self, id: BlockHash) -> impl SendFut>; @@ -229,6 +358,15 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { fn subscribe_to_blocks_received(&self) -> Option> { None } + + /// Release any long-lived resources the source owns (e.g. a background + /// syncer task feeding a `ReadStateService`). + /// + /// Default is a no-op — poll-only sources (`JsonRpSeeConnector`) and test + /// mockchains own nothing to tear down. Sources that spawn their own + /// validator plumbing (the `State` arm of `ValidatorConnector`, which owns + /// the Zebra syncer task) override this to abort that task on shutdown. + fn shutdown(&self) {} } /// Sleep up to `duration`, but return early if `change_rx` resolves first. @@ -257,11 +395,54 @@ pub(super) async fn wait_or_source_change( /// An error originating from a blockchain source. #[derive(Debug, thiserror::Error)] pub enum BlockchainSourceError { - /// Unrecoverable error. + /// Unrecoverable error described only by a message (no underlying + /// typed error exists, e.g. an unexpected response shape). // TODO: Add logic for handling recoverable errors if any are identified // one candidate may be ephemerable network hiccoughs #[error("critical error in backing block source: {0}")] Unrecoverable(String), + /// Unrecoverable error whose typed cause is preserved as + /// [`std::error::Error::source`]. zaino-serve recovers zcashd-compatible + /// RPC error codes by downcast-walking `source()` chains, so errors that + /// wrap a typed transport or RPC error must use this variant rather than + /// [`Self::Unrecoverable`] with a stringified cause. + #[error("critical error in backing block source: {message}")] + UnrecoverableWithSource { + /// Rendered description, including the cause's `Display` output so + /// top-level log lines match the previous stringified form. + message: String, + /// The typed cause, available to `source()`-chain walks. + #[source] + source: Box, + }, +} + +impl BlockchainSourceError { + /// Wraps a typed error, preserving it for `source()`-chain recovery. + /// Accepts both concrete error types and already-boxed errors (e.g. + /// zebra's `BoxError`). + pub(crate) fn unrecoverable( + error: impl Into>, + ) -> Self { + let source = error.into(); + Self::UnrecoverableWithSource { + message: source.to_string(), + source, + } + } + + /// Wraps a typed error with a context prefix, preserving the error for + /// `source()`-chain recovery. + pub(crate) fn unrecoverable_context( + context: impl std::fmt::Display, + error: impl Into>, + ) -> Self { + let source = error.into(); + Self::UnrecoverableWithSource { + message: format!("{context}: {source}"), + source, + } + } } /// Error type returned when invalid data is returned by the validator. diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 5f262527b..a6bc290e8 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -1,21 +1,32 @@ //! Mock BlockchainSourceResult implementation. +use super::validator_connector::{ + assemble_block_deltas, build_block_header_object, build_verbose_block, + confirmations_from_depth, final_orchard_root, final_sapling_root, median_of_block_times, + zebra_block_header_to_wire, +}; use super::*; use std::collections::{HashMap, HashSet}; +use std::str::FromStr as _; use std::sync::{ atomic::{AtomicU32, Ordering}, Arc, Mutex, }; use zaino_common::network::ActivationHeights; -use zaino_fetch::jsonrpsee::response::address_deltas::BlockInfo; +use zaino_fetch::jsonrpsee::response::{ + address_deltas::BlockInfo, block_deltas::BlockDeltas, block_header::GetBlockHeader, +}; use zebra_chain::{block::Block, orchard::tree as orchard, sapling::tree as sapling}; use zebra_chain::{ - block::Height, + block::{Height, SerializedBlock}, parameters::NetworkKind, - serialization::BytesInDisplayOrder as _, + serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, transparent::{Address, OutPoint, Output, OutputIndex}, }; -use zebra_rpc::{client::TransactionObject, methods::ValidateAddresses as _}; +use zebra_rpc::{ + client::{BlockObject, HexData, Input, TransactionObject}, + methods::{GetBlock, GetBlockHeaderResponse, GetBlockTransaction, ValidateAddresses as _}, +}; use zebra_state::HashOrHeight; /// Build the txid → (height, tx) lookup map used by @@ -125,7 +136,7 @@ fn normalize_requested_addresses_for_network( /// transparent address prefixes, so output-derived transparent addresses use /// `NetworkKind::Testnet`. fn mockchain_network() -> zebra_chain::parameters::Network { - zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network() + ActivationHeights::default().to_regtest_network() } /// A test-only mock implementation of BlockchainReader using ordered lists by height. @@ -169,6 +180,9 @@ pub(crate) struct MockchainSource { /// neither know nor care how many `mine_blocks` events occurred /// between wakes. blocks_received_broadcaster: tokio::sync::watch::Sender<()>, + /// Records whether [`BlockchainSource::shutdown`] ran, so teardown tests + /// can assert the index releases its source. Shared across clones. + shutdown_called: Arc, } impl MockchainSource { @@ -235,9 +249,16 @@ impl MockchainSource { )), get_block_hook: Arc::new(Mutex::new(None)), blocks_received_broadcaster: tokio::sync::watch::channel(()).0, + shutdown_called: Arc::new(std::sync::atomic::AtomicBool::new(false)), } } + /// Whether [`BlockchainSource::shutdown`] has run on this source (or any + /// clone of it). + pub(crate) fn shutdown_called(&self) -> bool { + self.shutdown_called.load(Ordering::SeqCst) + } + /// When set to true, `get_best_block_height` and `get_best_block_hash` /// return `BlockchainSourceError::Unrecoverable`. pub(crate) fn set_failing(&self, fail: bool) { @@ -332,6 +353,98 @@ impl MockchainSource { self.active_height() as usize } + /// Resolves a hash-or-height request to a block index within the active chain. + fn resolve_index(&self, id: &HashOrHeight) -> Option { + match id { + HashOrHeight::Height(height) => self.valid_height(height.0), + HashOrHeight::Hash(hash) => self.valid_hash(hash), + } + } + + /// The zebra hash of the block after `height_index`, if it is within the active chain. + fn next_block_hash(&self, height_index: usize) -> Option { + let next = height_index + 1; + (next <= self.active_chain_height_as_usize()).then(|| self.blocks[next].hash()) + } + + /// Builds the zebra `getblockheader` response for the block at `height_index` from the + /// test-vector block and tree-root data, using the same builders as the validator path. + fn block_header_response_at( + &self, + height_index: usize, + verbose: bool, + ) -> BlockchainSourceResult { + let block = &self.blocks[height_index]; + let header = &block.header; + if !verbose { + return Ok(GetBlockHeaderResponse::Raw(HexData( + header + .zcash_serialize_to_vec() + .map_err(BlockchainSourceError::unrecoverable)?, + ))); + } + + let network = mockchain_network(); + let hash = block.hash(); + let height = block.coinbase_height().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block missing coinbase height".to_string()) + })?; + let depth = self.active_height().checked_sub(height.0); + let confirmations = confirmations_from_depth(depth); + + let (sapling_root_bytes, sapling_tree_size) = match &self.roots[height_index].0 { + Some((root, size)) => (<[u8; 32]>::from(*root), *size), + None => ([0u8; 32], 0), + }; + let final_sapling_root = final_sapling_root(sapling_root_bytes, height, &network); + let next_block_hash = self.next_block_hash(height_index); + + let header_obj = build_block_header_object( + header, + hash, + height, + confirmations, + final_sapling_root, + sapling_tree_size, + next_block_hash, + &network, + )?; + Ok(GetBlockHeaderResponse::Object(Box::new(header_obj))) + } + + /// Median time past over the 11-block window ending at `start`, walking backwards via + /// verbosity-1 `getblock` lookups against the mock vectors. + async fn median_time_past(&self, start: &BlockObject) -> BlockchainSourceResult { + const MEDIAN_TIME_PAST_WINDOW: usize = 11; + let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); + let start_time = start.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: start block missing time".into()) + })?; + times.push(start_time); + + let mut prev = start.previous_block_hash(); + for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { + let Some(hash) = prev else { + break; // genesis + }; + match self + .get_block_verbose(HashOrHeight::Hash(hash), Some(1)) + .await + { + Ok(GetBlock::Object(object)) => { + if let Some(time) = object.time() { + times.push(time); + } + prev = object.previous_block_hash(); + } + Ok(GetBlock::Raw(_)) => break, + Err(_) => break, + } + } + + median_of_block_times(times) + } + fn block_height_at_index(&self, block_index: usize) -> Height { self.blocks[block_index] .coinbase_height() @@ -450,6 +563,263 @@ impl BlockchainSource for MockchainSource { } } + async fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + let verbosity = verbosity.unwrap_or(1); + let height_index = self + .resolve_index(&hash_or_height) + .ok_or_else(|| BlockchainSourceError::Unrecoverable("block not found".to_string()))?; + + match verbosity { + 0 => Ok(GetBlock::Raw(SerializedBlock::from(Arc::clone( + &self.blocks[height_index], + )))), + 1 | 2 => { + let block = &self.blocks[height_index]; + let network = mockchain_network(); + + let GetBlockHeaderResponse::Object(header_obj) = + self.block_header_response_at(height_index, true)? + else { + unreachable!("`true` yields an object") + }; + + let height = block.coinbase_height().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "block missing coinbase height".to_string(), + ) + })?; + let (orchard_root_bytes, orchard_tree_size) = match &self.roots[height_index].1 { + Some((root, size)) => (<[u8; 32]>::from(*root), *size), + None => ([0u8; 32], 0), + }; + let final_orchard_root = final_orchard_root(orchard_root_bytes, height, &network); + + // The verbose block reports the block's serialized byte size; the mock has the + // full block, so serialize it to measure. + let size = block + .zcash_serialize_to_vec() + .map_err(BlockchainSourceError::unrecoverable)? + .len() as i64; + + // `chain_supply` / `value_pools` are cumulative pool balances that the test + // vectors do not carry, so they are `None` for the mock. + Ok(build_verbose_block( + &header_obj, + block, + verbosity, + size, + final_orchard_root, + orchard_tree_size, + // The mock's test vectors do not carry an ironwood tree size. + 0, + None, + None, + &network, + )) + } + more_than_two => Err(BlockchainSourceError::Unrecoverable(format!( + "invalid verbosity of {more_than_two}" + ))), + } + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> BlockchainSourceResult { + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; + let height_index = self.resolve_index(&hash_or_height).ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) + })?; + let header = self.block_header_response_at(height_index, verbose)?; + zebra_block_header_to_wire(header) + } + + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; + let GetBlock::Object(object) = self.get_block_verbose(hash_or_height, Some(2)).await? + else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected raw block".to_string(), + )); + }; + + // The mock holds every transaction, so prevout resolution is a direct index lookup. + let mut prevtx_cache: HashMap< + zebra_chain::transaction::Hash, + Arc, + > = HashMap::new(); + for tx in object.tx() { + let GetBlockTransaction::Object(txo) = tx else { + continue; + }; + for input in txo.inputs() { + let Input::NonCoinbase { txid: prevtxid, .. } = input else { + continue; + }; + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(BlockchainSourceError::unrecoverable)?; + if prevtx_cache.contains_key(&prev_hash) { + continue; + } + let (_height, prev_tx) = self.txid_index.get(&prev_hash).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not found in mock chain" + )) + })?; + prevtx_cache.insert(prev_hash, Arc::clone(prev_tx)); + } + } + + let median_time = self.median_time_past(&object).await?; + assemble_block_deltas(&object, &prevtx_cache, median_time, &mockchain_network()) + } + + // ********** Chain methods ********** + + async fn get_difficulty(&self) -> BlockchainSourceResult { + let tip_index = self.active_chain_height_as_usize(); + let tip_block = self.blocks.get(tip_index).ok_or_else(|| { + BlockchainSourceError::Unrecoverable("mock chain has no tip block".to_string()) + })?; + Ok(tip_block + .header + .difficulty_threshold + .relative_to_network(&mockchain_network())) + } + + async fn get_blockchain_info( + &self, + ) -> BlockchainSourceResult { + // Needs cumulative pool value balances (TipPoolValues) and on-disk size, which the + // vectors don't carry. Test vectors must be extended to serve this method; tracked + // by the update-test-vectors follow-up (see "Future work"). + unimplemented!( + "MockchainSource cannot serve get_blockchain_info until test vectors are extended" + ) + } + + // ********** Node-passthrough methods ********** + // + // These are node-only RPCs with no chain data in the vectors. Test vectors must be + // extended to let MockchainSource serve them; tracked by the update-test-vectors + // follow-up (see "Future work"). + + async fn get_info(&self) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_info until test vectors are extended") + } + + async fn get_peer_info( + &self, + ) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_peer_info until test vectors are extended") + } + + /// Records the release so teardown tests can assert the index shuts its + /// source down (the mock owns no real background work). + fn shutdown(&self) { + self.shutdown_called.store(true, Ordering::SeqCst); + } + + /// A single active tip at the mockchain's current active height, matching + /// what a validator with no side branches reports. + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + if self + .force_requests_against_source_to_fail + .load(Ordering::SeqCst) + { + return Err(BlockchainSourceError::Unrecoverable( + "forced source failure".into(), + )); + } + let height = self.active_height(); + let Some(index) = self.valid_height(height) else { + return Ok(vec![]); + }; + Ok(vec![ + zaino_fetch::jsonrpsee::response::chain_tips::ChainTip::new( + height, + self.blocks[index].hash().to_string(), + 0, + zaino_fetch::jsonrpsee::response::chain_tips::ChainTipStatus::Active, + ), + ]) + } + + async fn get_block_subsidy( + &self, + _height: u32, + ) -> BlockchainSourceResult + { + unimplemented!( + "MockchainSource cannot serve get_block_subsidy until test vectors are extended" + ) + } + + async fn get_mining_info( + &self, + ) -> BlockchainSourceResult + { + unimplemented!( + "MockchainSource cannot serve get_mining_info until test vectors are extended" + ) + } + + async fn get_tx_out( + &self, + _txid: String, + _n: u32, + _include_mempool: Option, + ) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_tx_out until test vectors are extended") + } + + async fn get_spent_info( + &self, + _request: zaino_fetch::jsonrpsee::response::GetSpentInfoRequest, + ) -> BlockchainSourceResult { + unimplemented!( + "MockchainSource cannot serve get_spent_info until test vectors are extended" + ) + } + + async fn get_network_sol_ps( + &self, + _blocks: Option, + _height: Option, + ) -> BlockchainSourceResult { + unimplemented!( + "MockchainSource cannot serve get_network_sol_ps until test vectors are extended" + ) + } + + async fn send_raw_transaction( + &self, + _raw_transaction_hex: String, + ) -> BlockchainSourceResult { + // The mock chain has no mempool to accept submissions. + unimplemented!("MockchainSource cannot serve send_raw_transaction") + } + + async fn get_treestate_by_id( + &self, + _hash_or_height: String, + ) -> BlockchainSourceResult { + // The `z_get_treestate` local path serves the mock; the node-passthrough fallback + // is never reached, so this is left unimplemented. + unimplemented!("MockchainSource cannot serve the get_treestate_by_id passthrough") + } + // ********** Transaction methods ********** async fn get_transaction( @@ -551,24 +921,34 @@ impl BlockchainSource for MockchainSource { } /// Returns the sapling and orchard treestate by hash - async fn get_treestate( - &self, - id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>)> { + /// + /// TODO: Update test vectors to support ironwood. + async fn get_treestate(&self, id: BlockHash) -> BlockchainSourceResult { let active_chain_height = self.active_height() as usize; // serve up to active tip if let Some(height) = self.hashes.iter().position(|h| h == &id) { if height <= active_chain_height { let (sapling_state, orchard_state) = &self.treestates[height]; - Ok((Some(sapling_state.clone()), Some(orchard_state.clone()))) + Ok(( + Some(super::PoolTreestate { + final_root: None, + final_state: sapling_state.clone(), + }), + Some(super::PoolTreestate { + final_root: None, + final_state: orchard_state.clone(), + }), + None, + )) } else { - Ok((None, None)) + Ok((None, None, None)) } } else { - Ok((None, None)) + Ok((None, None, None)) } } + /// TODO: Update test vectors to support ironwood. async fn get_subtree_roots( &self, pool: ShieldedPool, @@ -652,28 +1032,32 @@ impl BlockchainSource for MockchainSource { } } } + ShieldedPool::Ironwood => {} } Ok(subtree_roots) } + /// TODO: Update test vectors to support ironwood. async fn get_commitment_tree_roots( &self, id: BlockHash, ) -> BlockchainSourceResult<( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, )> { let active_chain_height = self.active_height() as usize; // serve up to active tip if let Some(height) = self.hashes.iter().position(|h| h == &id) { if height <= active_chain_height { - Ok(self.roots[height]) + let (sapling, orchard) = self.roots[height]; + Ok((sapling, orchard, None)) } else { - Ok((None, None)) + Ok((None, None, None)) } } else { - Ok((None, None)) + Ok((None, None, None)) } } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index df4f28e23..e2130c218 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1,9 +1,42 @@ //! validator connected blockchain source. -use hex::FromHex as _; -use zaino_fetch::jsonrpsee::response::address_deltas::BlockInfo; -use zebra_chain::serialization::BytesInDisplayOrder as _; -use zebra_rpc::methods::ValidateAddresses as _; +use std::collections::HashMap; +use std::str::FromStr as _; + +use chrono::{DateTime, Utc}; +use futures::future::join3; +use hex::{FromHex as _, ToHex as _}; +use indexmap::IndexMap; +use tracing::info; +use zaino_fetch::jsonrpsee::response::{ + address_deltas::BlockInfo, + block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, + block_header::GetBlockHeader, + block_subsidy::GetBlockSubsidy, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + GetInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, + GetTxOutResponse, +}; +use zebra_rpc::sync::init_read_state_with_syncer; + +use crate::config::{CommonBackendConfig, DirectConnectionConfig}; +use zebra_chain::{ + amount::{Amount, NonNegative}, + block::{Header, SerializedBlock}, + chain_tip::NetworkChainTipHeightEstimator, + parameters::{ConsensusBranchId, NetworkUpgrade}, + serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, +}; +use zebra_rpc::{ + client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, + methods::{ + chain_tip_difficulty, ConsensusBranchIdHex, GetBlock, GetBlockHeaderObject, + GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, + GetInfo, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, TipConsensusBranch, + ValidateAddresses as _, + }, +}; use crate::Height; @@ -33,8 +66,14 @@ pub struct State { pub read_state_service: ReadStateService, /// Temporarily used to fetch mempool data. pub mempool_fetcher: JsonRpSeeConnector, - /// Current network type being run. - pub network: Network, + /// The runtime network (activation schedule adopted from the validator). + pub network: zebra_chain::parameters::Network, + /// Watches the Zebra syncer's chain-tip changes; served to consumers via + /// [`ValidatorConnector::chain_tip_change`]. + pub chain_tip_change: zebra_state::ChainTipChange, + /// Handle to the Zebra `ReadStateService` sync task, kept alive for the lifetime of + /// the connector and aborted on [`ValidatorConnector::shutdown`]. Shared across clones. + pub sync_task_handle: Option>>, } /// A connection to a validator. @@ -50,7 +89,351 @@ pub enum ValidatorConnector { Fetch(JsonRpSeeConnector), } +impl ValidatorConnector { + /// The JSON-RPC connector for this validator, used by the node-passthrough RPCs that + /// have no local-index equivalent. The `State` variant proxies these through its + /// `mempool_fetcher` until the `ReadStateService` can serve them. + fn json_rpc_connector(&self) -> &JsonRpSeeConnector { + match self { + ValidatorConnector::State(state) => &state.mempool_fetcher, + ValidatorConnector::Fetch(fetch) => fetch, + } + } + + /// Spawns a JSON-RPC-backed [`ValidatorConnector::Fetch`] from the common backend + /// config, returning the connector, the validator's `getinfo` response (used by + /// the backend to build its `ServiceMetadata`), and the runtime network whose + /// activation schedule was adopted from the validator (zaino#1076). + /// + /// Owns the `JsonRpSeeConnector` setup that previously lived in `FetchService::spawn`. + pub(crate) async fn spawn_fetch( + common: &CommonBackendConfig, + ) -> Result<(Self, GetInfoResponse, zebra_chain::parameters::Network), BlockchainSourceError> + { + let fetcher = JsonRpSeeConnector::new_from_config_parts( + &common.validator_rpc_address, + common.validator_rpc_user.clone(), + common.validator_rpc_password.clone(), + common.validator_cookie_path.clone(), + ) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + + let info = fetcher + .get_info() + .await + .map_err(BlockchainSourceError::unrecoverable)?; + + // A freshly started validator answers RPC before it has fetched and + // committed its first block: zebra serves getblockchaininfo and an + // empty getrawmempool while getbestblockhash returns "No blocks in + // state" until genesis arrives, which can take minutes of peer + // discovery. Everything downstream assumes a servable tip + // (Mempool::spawn returns Critical on get_best_block_hash and the + // ChainIndex sync loop escalates to CriticalError), so wait here + // instead of failing spawn and exit-looping the whole process. + while let Err(e) = fetcher.get_best_blockhash().await { + info!(%e, "Waiting for validator to serve its first block"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + + let network = adopt_network(common, &fetcher).await?; + + Ok((ValidatorConnector::Fetch(fetcher), info, network)) + } + + /// Spawns a `ReadStateService`-backed [`ValidatorConnector::State`] from the common + /// backend config plus the [`DirectConnectionConfig`], returning the connector plus + /// the validator's `getinfo` response. + /// + /// Owns the JSON-RPC + Zebra chain-syncer setup for the `Direct` connection: builds + /// the mempool JSON-RPC fetcher, launches the syncer and `ReadStateService`, then + /// blocks until the syncer has caught up to the validator's best chain tip (comparing + /// tip *hash* as well as height so a reorg mid-sync cannot report a false match). The + /// syncer task handle is retained on the `State` so [`ValidatorConnector::shutdown`] + /// can abort it. + pub(crate) async fn spawn_state( + common: &CommonBackendConfig, + direct: &DirectConnectionConfig, + ) -> Result<(Self, GetInfoResponse, zebra_chain::parameters::Network), BlockchainSourceError> + { + let map_err = + |error: &dyn std::fmt::Display| BlockchainSourceError::Unrecoverable(error.to_string()); + + let rpc_client = JsonRpSeeConnector::new_from_config_parts( + &common.validator_rpc_address, + common.validator_rpc_user.clone(), + common.validator_rpc_password.clone(), + common.validator_cookie_path.clone(), + ) + .await + .map_err(|error| map_err(&error))?; + + let info = rpc_client + .get_info() + .await + .map_err(|error| map_err(&error))?; + + let network = adopt_network(common, &rpc_client).await?; + + info!( + grpc_address = %direct.validator_grpc_address, + "Launching Chain Syncer" + ); + let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = + init_read_state_with_syncer( + direct.validator_state_config.clone(), + &network, + direct.validator_grpc_address, + ) + .await + .map_err(|error| map_err(&error))? + .map_err(|error| map_err(&error))?; + + info!("Chain syncer launched"); + + // Wait for ReadStateService to catch up to the validator's best chain tip. + // Height alone is insufficient during reorgs: the same height can refer to + // different blocks until JSON-RPC and ReadStateService agree on tip hash. + loop { + let blockchain_info = rpc_client + .get_blockchain_info() + .await + .map_err(|error| map_err(&error))?; + let server_height = blockchain_info.blocks; + let server_tip_hash = blockchain_info.best_block_hash; + + let syncer_response = read_state_service + .ready() + .and_then(|service| service.call(ReadRequest::Tip)) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + // A freshly started validator answers RPC before it has committed + // its first block (getblockchaininfo reports a genesis-hash + // placeholder on an empty state), and the syncer has no tip until + // genesis arrives, which can take minutes of peer discovery. + // Everything below assumes a servable tip, so wait here instead + // of failing spawn and exit-looping the whole process. + let Some((syncer_height, syncer_tip_hash)) = + expected_read_response!(syncer_response, Tip) + else { + info!("Waiting for validator to serve its first block"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + continue; + }; + + if server_height == syncer_height && server_tip_hash == syncer_tip_hash { + info!( + height = syncer_height.0, + tip_hash = %syncer_tip_hash, + "ReadStateService synced with Zebra" + ); + break; + } else { + info!( + syncer_height = syncer_height.0, + validator_height = server_height.0, + syncer_tip_hash = %syncer_tip_hash, + validator_tip_hash = %server_tip_hash, + "ReadStateService syncing with Zebra" + ); + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + continue; + } + } + + let source = ValidatorConnector::State(State { + read_state_service, + mempool_fetcher: rpc_client, + network: network.clone(), + chain_tip_change, + sync_task_handle: Some(Arc::new(sync_task_handle)), + }); + + Ok((source, info, network)) + } + + /// The backing [`ReadStateService`], when this connector is `State`-backed. + /// + /// Test-only escape hatch: live tests recompute expected chain data directly off + /// the `ReadStateService`. Production code goes through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub(crate) fn read_state_service(&self) -> Option<&ReadStateService> { + match self { + ValidatorConnector::State(state) => Some(&state.read_state_service), + ValidatorConnector::Fetch(_) => None, + } + } +} + +/// Constructs the runtime network at first contact with the validator. +/// +/// The validator is the single source of truth for activation heights +/// (zaino#1076): the config carries only a network kind, and the runtime +/// network is constructed here before anything consumes one. On regtest the +/// schedule is adopted wholesale from the validator's report. On Mainnet and +/// The Public Testnet the compiled zebra parameters are used, but the +/// validator's reported schedule is verified against them first, so a +/// validator reporting custom activation heights under the `PubTestnet` +/// kind (a regtest net in configured-testnet clothing) fails loud here +/// instead of silently drifting from the index. There is no fallback: a +/// silently wrong schedule is the failure mode this removes. +/// +/// Adoption happens exactly once, at spawn. A validator restarted +/// mid-session with a different schedule is out of scope (the test harness +/// restarts validator and indexer together); it would invalidate the index +/// without being re-detected here. +async fn adopt_network( + common: &CommonBackendConfig, + rpc_client: &JsonRpSeeConnector, +) -> Result { + let blockchain_info = rpc_client.get_blockchain_info().await.map_err(|error| { + BlockchainSourceError::Unrecoverable(format!( + "cannot fetch activation heights from the validator at {}: {error}", + common.validator_rpc_address + )) + })?; + + // Shared by the Mainnet / The Public Testnet arms: the compiled network is only + // trusted after the validator's report agrees with it. + let verified = |network: zebra_chain::parameters::Network| { + verify_reported_upgrades(&network, &blockchain_info.upgrades).map_err(|reason| { + BlockchainSourceError::Unrecoverable(format!( + "the validator at {} disagrees with the compiled {network} parameters: {reason}", + common.validator_rpc_address + )) + })?; + Ok(network) + }; + + match common.network { + zaino_common::Network::Mainnet => verified(zebra_chain::parameters::Network::Mainnet), + zaino_common::Network::PubTestnet => { + verified(zebra_chain::parameters::Network::new_default_testnet()) + } + zaino_common::Network::Regtest => { + let heights = + activation_heights_from_upgrades(&blockchain_info.upgrades).map_err(|reason| { + BlockchainSourceError::Unrecoverable(format!( + "cannot adopt activation heights from the validator at {}: {reason}", + common.validator_rpc_address + )) + })?; + info!(?heights, "Adopted activation heights from the validator"); + Ok(heights.to_regtest_network()) + } + } +} + +/// Checks every `(upgrade, height)` the validator reports against `network`'s +/// compiled activation schedule, rejecting the first disagreement. +/// +/// Only reported entries are checked — the report's *completeness* is not: +/// the map is keyed by consensus branch ID, so upgrades zebra considers +/// unconfigured are simply absent, and an older validator may not know the +/// newest upgrades yet. +fn verify_reported_upgrades( + network: &zebra_chain::parameters::Network, + upgrades: &IndexMap, +) -> Result<(), String> { + for upgrade_info in upgrades.values() { + let (upgrade, height, _status) = upgrade_info.into_parts(); + let compiled = upgrade.activation_height(network); + if compiled != Some(height) { + return Err(format!( + "the validator reports {upgrade:?} activating at height {}, \ + but the compiled parameters say {:?}", + height.0, + compiled.map(|compiled_height| compiled_height.0) + )); + } + } + Ok(()) +} + +/// Builds the regtest activation heights from the validator's reported +/// upgrade schedule (`getblockchaininfo.upgrades`). +/// +/// The validator's configured activation heights are authoritative: the +/// config type is a payload-free kind, so both connector arms construct the +/// runtime network here at first contact, before anything consumes a +/// `Network` (zaino#1076). An upgrade absent from the validator's map is +/// never-activated — nothing is backfilled from defaults. Mainnet and +/// The Public Testnet use zebra's compiled parameters and never take this +/// path. +/// +/// `before_overwinter` is always `None` here: the map is keyed by consensus +/// branch ID, which `BeforeOverwinter` does not have, so it can never appear +/// in the report. Correctness relies on zebra's `new_regtest` supplying its +/// own `BeforeOverwinter` handling when `to_regtest_network` builds the +/// runtime network from these heights. +fn activation_heights_from_upgrades( + upgrades: &IndexMap, +) -> Result { + let mut heights = zaino_common::config::network::ActivationHeights::NEVER_ACTIVATED; + for upgrade_info in upgrades.values() { + let (upgrade, height, _status) = upgrade_info.into_parts(); + // Genesis is height 0 by definition; it has no configuration slot. + let Some(slot) = heights.slot_mut(upgrade) else { + continue; + }; + if slot.replace(height.0).is_some() { + return Err(format!("validator reported {upgrade:?} twice")); + } + } + Ok(heights) +} + +/// Serialized empty Orchard-shaped commitment tree in the RPC encoding — what a pool +/// reports when it has no treestate to serve, and the encoding both backend arms must +/// agree on. +fn empty_orchard_tree_rpc_bytes() -> Vec { + let mut tree = vec![]; + write_commitment_tree( + &CommitmentTree::::empty(), + &mut tree, + ) + .expect("can write to Vec"); + tree +} + +/// The ironwood slot of the source treestate tuple, given the ironwood treestate the +/// validator reported for the block (if any). +/// +/// The z_gettreestate ironwood field is documented as "Only present from NU6.3, so that +/// pre-NU6.3 responses are unchanged": when the validator reported no ironwood treestate +/// (below NU6.3 activation, or on a network with no NU6.3 activation height) the slot +/// stays `None`, so the response omits the field exactly as zebrad does. This function +/// names that contract — do not back-fill an empty tree here. +fn ironwood_treestate_slot(validator_ironwood: Option) -> Option { + validator_ironwood +} + +/// Maps a validator-reported treestate (from the JSON-RPC `z_gettreestate` response) +/// into the connector's pool slot. +fn fetch_pool_treestate_slot(treestate: zebra_rpc::client::Treestate) -> Option { + let final_root = treestate.commitments().final_root().clone(); + treestate + .commitments() + .final_state() + .clone() + .map(|final_state| PoolTreestate { + final_root, + final_state, + }) +} + impl BlockchainSource for ValidatorConnector { + /// `Some` for the `State` variant, which drives its own Zebra syncer; the + /// JSON-RPC `Fetch` variant has no local tip-change stream and keeps the + /// trait's `None` default semantics. + fn chain_tip_change(&self) -> Option { + match self { + ValidatorConnector::State(state) => Some(state.chain_tip_change.clone()), + ValidatorConnector::Fetch(_) => None, + } + } + // ********** Block methods ********** async fn get_block( @@ -72,7 +455,7 @@ impl BlockchainSource for ValidatorConnector { { Ok(GetBlockResponse::Raw(raw_block)) => Ok(Some(Arc::new( zebra_chain::block::Block::zcash_deserialize(raw_block.as_ref()) - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))), Ok(_) => unreachable!(), Err(e) => match e { @@ -80,7 +463,7 @@ impl BlockchainSource for ValidatorConnector { // TODO/FIX: zcashd returns this transport error when a block is requested higher than current chain. is this correct? RpcRequestError::Transport(zaino_fetch::jsonrpsee::error::TransportError::ErrorStatusCode(500)) => Ok(None), RpcRequestError::ServerWorkQueueFull => Err(BlockchainSourceError::Unrecoverable("Work queue full. not yet implemented: handling of ephemeral network errors.".to_string())), - _ => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + _ => Err(BlockchainSourceError::unrecoverable(e)), }, } } @@ -88,7 +471,7 @@ impl BlockchainSource for ValidatorConnector { "Read Request of Block returned Read Response of {otherwise:#?} \n\ This should be deterministically unreachable" ), - Err(e) => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + Err(e) => Err(BlockchainSourceError::unrecoverable(e)), }, ValidatorConnector::Fetch(fetch) => { match fetch @@ -97,7 +480,7 @@ impl BlockchainSource for ValidatorConnector { { Ok(GetBlockResponse::Raw(raw_block)) => Ok(Some(Arc::new( zebra_chain::block::Block::zcash_deserialize(raw_block.as_ref()) - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))), Ok(_) => unreachable!(), Err(e) => match e { @@ -105,13 +488,201 @@ impl BlockchainSource for ValidatorConnector { // TODO/FIX: zcashd returns this transport error when a block is requested higher than current chain. is this correct? RpcRequestError::Transport(zaino_fetch::jsonrpsee::error::TransportError::ErrorStatusCode(500)) => Ok(None), RpcRequestError::ServerWorkQueueFull => Err(BlockchainSourceError::Unrecoverable("Work queue full. not yet implemented: handling of ephemeral network errors.".to_string())), - _ => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + _ => Err(BlockchainSourceError::unrecoverable(e)), }, } } } } + async fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => { + state.get_block_inner(hash_or_height, verbosity).await + } + ValidatorConnector::Fetch(fetch) => fetch + .get_block(hash_or_height.to_string(), verbosity) + .await + .map_err(BlockchainSourceError::unrecoverable)? + .try_into() + .map_err(|error: zebra_chain::serialization::SerializationError| { + BlockchainSourceError::unrecoverable(error) + }), + } + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => { + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; + let header = state + .get_block_header_inner(hash_or_height, Some(verbose)) + .await?; + zebra_block_header_to_wire(header) + } + ValidatorConnector::Fetch(fetch) => fetch + .get_block_header(hash, verbose) + .await + .map_err(BlockchainSourceError::unrecoverable), + } + } + + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => state.get_block_deltas(hash).await, + ValidatorConnector::Fetch(fetch) => fetch + .get_block_deltas(hash) + .await + .map_err(BlockchainSourceError::unrecoverable), + } + } + + // ********** Chain methods ********** + + async fn get_difficulty(&self) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => chain_tip_difficulty( + state.network.clone(), + state.read_state_service.clone(), + false, + ) + .await + .map_err(BlockchainSourceError::unrecoverable), + ValidatorConnector::Fetch(fetch) => Ok(fetch + .get_difficulty() + .await + .map_err(BlockchainSourceError::unrecoverable)? + .0), + } + } + + async fn get_blockchain_info(&self) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => state.get_blockchain_info().await, + ValidatorConnector::Fetch(fetch) => fetch + .get_blockchain_info() + .await + .map_err(BlockchainSourceError::unrecoverable)? + .try_into() + .map_err(|_error| { + BlockchainSourceError::Unrecoverable( + "getblockchaininfo: chainwork not hex-encoded integer".to_string(), + ) + }), + } + } + + // ********** Node-passthrough methods ********** + + async fn get_info(&self) -> BlockchainSourceResult { + Ok(self + .json_rpc_connector() + .get_info() + .await + .map_err(BlockchainSourceError::unrecoverable)? + .into()) + } + + async fn get_peer_info(&self) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_peer_info() + .await + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + self.json_rpc_connector() + .get_chain_tips() + .await + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn get_block_subsidy(&self, height: u32) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_block_subsidy(height) + .await + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn get_mining_info(&self) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_mining_info() + .await + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_tx_out(txid, n, include_mempool) + .await + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_spent_info(request) + .await + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_network_sol_ps(blocks, height) + .await + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> BlockchainSourceResult { + // ReadStateService does not yet interface with the mempool, so both variants + // submit via the JSON-RPC connector. + self.json_rpc_connector() + .send_raw_transaction(raw_transaction_hex) + .await + .map(SentTransactionHash::from) + .map_err(BlockchainSourceError::unrecoverable) + } + + async fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_treestate(hash_or_height) + .await + .map_err(BlockchainSourceError::unrecoverable)? + .try_into() + .map_err(|_error| { + BlockchainSourceError::Unrecoverable("failed to parse treestate".to_string()) + }) + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain @@ -128,7 +699,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { // Check state for transaction let mut read_state_service = read_state_service.clone(); @@ -144,7 +715,7 @@ impl BlockchainSource for ValidatorConnector { }) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("state read failed: {e}")) + BlockchainSourceError::unrecoverable_context("state read failed", e) })?; if let zebra_state::ReadResponse::AnyChainTransaction(opt) = response { @@ -181,9 +752,10 @@ impl BlockchainSource for ValidatorConnector { .get_raw_transaction(zebra_txid.to_string(), Some(0)) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch transaction data", + e, + ) })? { serialized_transaction } else { @@ -196,9 +768,10 @@ impl BlockchainSource for ValidatorConnector { std::io::Cursor::new(serialized_transaction.as_ref()), ) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not deserialize transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not deserialize transaction data", + e, + ) })?; Ok(Some((transaction.into(), GetTransactionLocation::Mempool))) } else { @@ -211,9 +784,10 @@ impl BlockchainSource for ValidatorConnector { .get_raw_transaction(txid.to_rpc_hex(), Some(1)) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch transaction data", + e, + ) })? { transaction_object } else { @@ -226,9 +800,10 @@ impl BlockchainSource for ValidatorConnector { transaction_object.hex().as_ref(), )) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not deserialize transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not deserialize transaction data", + e, + ) })?; let location = match transaction_object.height() { Some(-1) => GetTransactionLocation::NonbestChain, @@ -258,7 +833,7 @@ impl BlockchainSource for ValidatorConnector { .get_raw_mempool() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not fetch mempool data: {e}")) + BlockchainSourceError::unrecoverable_context("could not fetch mempool data", e) })? .transactions; @@ -266,9 +841,10 @@ impl BlockchainSource for ValidatorConnector { .into_iter() .map(|txid_str| { zebra_chain::transaction::Hash::from_str(&txid_str).map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "invalid transaction id '{txid_str}': {e}" - )) + BlockchainSourceError::unrecoverable_context( + format!("invalid transaction id '{txid_str}'"), + e, + ) }) }) .collect::>()?; @@ -285,7 +861,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { match read_state_service.best_tip() { Some((_height, hash)) => Ok(Some(hash)), @@ -296,9 +872,10 @@ impl BlockchainSource for ValidatorConnector { .get_best_blockhash() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .0, )) @@ -310,9 +887,10 @@ impl BlockchainSource for ValidatorConnector { .get_best_blockhash() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .0, )), @@ -327,7 +905,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { match read_state_service.best_tip() { Some((height, _hash)) => Ok(Some(height)), @@ -338,9 +916,10 @@ impl BlockchainSource for ValidatorConnector { .get_block_count() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .into(), )) @@ -352,21 +931,22 @@ impl BlockchainSource for ValidatorConnector { .get_block_count() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .into(), )), } } - /// Returns the Sapling and Orchard treestate by blockhash. + /// Returns the Sapling, Orchard and Ironwood treestate by blockhash. async fn get_treestate( &self, // Sould this be HashOrHeight? id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>)> { + ) -> BlockchainSourceResult { let hash_or_height: HashOrHeight = HashOrHeight::Hash(zebra_chain::block::Hash(id.into())); match self { ValidatorConnector::State(state) => { @@ -394,8 +974,9 @@ impl BlockchainSource for ValidatorConnector { } }; - let sapling = match zebra_chain::parameters::NetworkUpgrade::Sapling - .activation_height(&state.network.to_zebra_network()) + let sapling = match ShieldedPool::Sapling + .activation_upgrade() + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -417,12 +998,16 @@ impl BlockchainSource for ValidatorConnector { _ => None, } .and_then(|sap_response| { - expected_read_response!(sap_response, SaplingTree) - .map(|tree| tree.to_rpc_bytes()) + expected_read_response!(sap_response, SaplingTree).map(|tree| PoolTreestate { + // finalRoot exactly as zebrad serves it: display-order root bytes. + final_root: Some(tree.root().bytes_in_display_order().to_vec()), + final_state: tree.to_rpc_bytes(), + }) }); - let orchard = match zebra_chain::parameters::NetworkUpgrade::Nu5 - .activation_height(&state.network.to_zebra_network()) + let orchard = match ShieldedPool::Orchard + .activation_upgrade() + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -444,11 +1029,46 @@ impl BlockchainSource for ValidatorConnector { _ => None, } .and_then(|orch_response| { - expected_read_response!(orch_response, OrchardTree) - .map(|tree| tree.to_rpc_bytes()) + expected_read_response!(orch_response, OrchardTree).map(|tree| PoolTreestate { + // finalRoot exactly as zebrad serves it: display-order root bytes. + final_root: Some(tree.root().bytes_in_display_order().to_vec()), + final_state: tree.to_rpc_bytes(), + }) + }); + + let ironwood = match ShieldedPool::Ironwood + .activation_upgrade() + .activation_height(&state.network) + { + Some(activation_height) if height >= activation_height => Some( + state + .read_state_service + .ready() + .and_then(|service| { + service.call(ReadRequest::IronwoodTree(hash_or_height)) + }) + .await + .map_err(|_e| { + BlockchainSourceError::Unrecoverable( + InvalidData(format!( + "could not fetch ironwood treestate of block {id}" + )) + .to_string(), + ) + })?, + ), + _ => None, + } + .and_then(|irw_response| { + expected_read_response!(irw_response, IronwoodTree).map(|tree| PoolTreestate { + // finalRoot exactly as zebrad serves it: display-order root bytes. + final_root: Some(tree.root().bytes_in_display_order().to_vec()), + final_state: tree.to_rpc_bytes(), + }) }); + let ironwood = ironwood_treestate_slot(ironwood); - Ok((sapling, orchard)) + Ok((sapling, orchard, ironwood)) } ValidatorConnector::Fetch(fetch) => { let treestate = fetch @@ -466,25 +1086,28 @@ impl BlockchainSource for ValidatorConnector { let mut tree = vec![]; write_commitment_tree(&sapling_crypto::CommitmentTree::empty(), &mut tree) .expect("can write to Vec"); - Some(tree) + Some(PoolTreestate { + final_root: None, + final_state: tree, + }) }, - |t| t.commitments().final_state().clone(), + fetch_pool_treestate_slot, ); let orchard = treestate.orchard.map_or_else( || { - let mut tree = vec![]; - write_commitment_tree( - &CommitmentTree::::empty(), - &mut tree, - ) - .expect("can write to Vec"); - Some(tree) + Some(PoolTreestate { + final_root: None, + final_state: empty_orchard_tree_rpc_bytes(), + }) }, - |t| t.commitments().final_state().clone(), + fetch_pool_treestate_slot, ); - Ok((sapling, orchard)) + let ironwood = + ironwood_treestate_slot(treestate.ironwood.and_then(fetch_pool_treestate_slot)); + + Ok((sapling, orchard, ironwood)) } } } @@ -502,6 +1125,7 @@ impl BlockchainSource for ValidatorConnector { let request = match pool { ShieldedPool::Sapling => ReadRequest::SaplingSubtrees { start_index, limit }, ShieldedPool::Orchard => ReadRequest::OrchardSubtrees { start_index, limit }, + ShieldedPool::Ironwood => ReadRequest::IronwoodSubtrees { start_index, limit }, }; state .read_state_service @@ -519,11 +1143,20 @@ impl BlockchainSource for ValidatorConnector { .iter() .map(|(_index, subtree)| (subtree.root.to_repr(), subtree.end_height.0)) .collect(), + ShieldedPool::Ironwood => { + expected_read_response!(response, IronwoodSubtrees) + .iter() + .map(|(_index, subtree)| { + (subtree.root.to_repr(), subtree.end_height.0) + }) + .collect() + } }) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) }) } @@ -532,9 +1165,10 @@ impl BlockchainSource for ValidatorConnector { .get_subtrees_by_index(pool.pool_string(), start_index, max_entries) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) })?; Ok(subtrees @@ -555,9 +1189,10 @@ impl BlockchainSource for ValidatorConnector { }) .collect::, _>>() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) })?) } } @@ -570,30 +1205,35 @@ impl BlockchainSource for ValidatorConnector { ) -> BlockchainSourceResult<( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, )> { match self { ValidatorConnector::State(state) => { - let (sapling_tree_response, orchard_tree_response) = - join( + let (sapling_tree_response, orchard_tree_response, ironwood_tree_response) = + join3( state.read_state_service.clone().call( zebra_state::ReadRequest::SaplingTree(HashOrHeight::Hash(id.into())), ), state.read_state_service.clone().call( zebra_state::ReadRequest::OrchardTree(HashOrHeight::Hash(id.into())), ), + state.read_state_service.clone().call( + zebra_state::ReadRequest::IronwoodTree(HashOrHeight::Hash(id.into())), + ), ) .await; - let (sapling_tree, orchard_tree) = match ( + let (sapling_tree, orchard_tree, ironwood_tree) = match ( //TODO: Better readstateservice error handling - sapling_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, - orchard_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + sapling_tree_response.map_err(BlockchainSourceError::unrecoverable)?, + orchard_tree_response.map_err(BlockchainSourceError::unrecoverable)?, + ironwood_tree_response.map_err(BlockchainSourceError::unrecoverable)?, ) { - (ReadResponse::SaplingTree(saptree), ReadResponse::OrchardTree(orctree)) => { - (saptree, orctree) - } - (_, _) => panic!("Bad response"), + ( + ReadResponse::SaplingTree(saptree), + ReadResponse::OrchardTree(orctree), + ReadResponse::IronwoodTree(irwtree), + ) => (saptree, orctree, irwtree), + (_, _, _) => panic!("Bad response"), }; Ok(( @@ -603,6 +1243,9 @@ impl BlockchainSource for ValidatorConnector { orchard_tree .as_deref() .map(|tree| (tree.root(), tree.count())), + ironwood_tree + .as_deref() + .map(|tree| (tree.root(), tree.count())), )) } ValidatorConnector::Fetch(fetch) => { @@ -619,10 +1262,13 @@ impl BlockchainSource for ValidatorConnector { .to_string(), ) } - _ => BlockchainSourceError::Unrecoverable(e.to_string()), + _ => BlockchainSourceError::unrecoverable(e), })?; let GetTreestateResponse { - sapling, orchard, .. + sapling, + orchard, + ironwood, + .. } = tree_responses; let sapling_frontier = sapling .map_or_else( @@ -636,7 +1282,7 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let orchard_frontier = orchard .map_or_else( || Some(Ok(CommitmentTree::empty())), @@ -649,7 +1295,20 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; + let ironwood_frontier = ironwood + .map_or_else( + || Some(Ok(CommitmentTree::empty())), + |t| { + t.commitments().final_state().as_ref().map(|final_state| { + read_commitment_tree::( + final_state.as_slice(), + ) + }) + }, + ) + .transpose() + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let sapling_root = sapling_frontier .map(|tree| { zebra_chain::sapling::tree::Root::try_from(tree.root().to_bytes()) @@ -657,7 +1316,7 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; let orchard_root = orchard_frontier .map(|tree| { @@ -666,9 +1325,18 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) + })?; + let ironwood_root = ironwood_frontier + .map(|tree| { + zebra_chain::orchard::tree::Root::try_from(tree.root().to_repr()) + .map(|root| (root, tree.size() as u64)) + }) + .transpose() + .map_err(|e| { + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; - Ok((sapling_root, orchard_root)) + Ok((sapling_root, orchard_root, ironwood_root)) } } } @@ -693,15 +1361,11 @@ impl BlockchainSource for ValidatorConnector { GetAddressDeltasParams::Address(a) => (vec![a.clone()], 0, 0, false), }; - let tip = self.get_best_block_height().await?.unwrap().into(); - let mut start = Height(start_raw); - let mut end = Height(end_raw); - if end == Height(0) || end > tip { - end = tip; - } - if start > tip { - start = tip; - } + let (start, end) = clamp_deltas_range_to_tip( + self.get_best_block_height().await?, + start_raw, + end_raw, + )?; let transactions: Vec> = { let tx_ids_request = @@ -732,7 +1396,7 @@ impl BlockchainSource for ValidatorConnector { transaction.clone(), height, None, - &state.network.to_zebra_network(), + &state.network, None, None, Some(matches!( @@ -765,14 +1429,10 @@ impl BlockchainSource for ValidatorConnector { let response = read_state .ready() .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })? + .map_err(BlockchainSourceError::unrecoverable)? .call(ReadRequest::BlockHeader(hash_or_height)) .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })?; + .map_err(BlockchainSourceError::unrecoverable)?; match response { ReadResponse::BlockHeader { hash, .. } => Ok(BlockInfo::new( @@ -793,14 +1453,10 @@ impl BlockchainSource for ValidatorConnector { let response = read_state .ready() .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })? + .map_err(BlockchainSourceError::unrecoverable)? .call(ReadRequest::BlockHeader(hash_or_height)) .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })?; + .map_err(BlockchainSourceError::unrecoverable)?; match response { ReadResponse::BlockHeader { hash, .. } => Ok(BlockInfo::new( @@ -827,7 +1483,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_address_deltas(params) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), } } @@ -840,14 +1496,14 @@ impl BlockchainSource for ValidatorConnector { let mut state = state.read_state_service.clone(); let strings_set = address_strings.valid_addresses().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid address: {error}")) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?; let response = state .ready() .and_then(|service| service.call(ReadRequest::AddressBalance(strings_set))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (balance, received) = match response { ReadResponse::AddressBalance { balance, received } => (balance, received), @@ -872,7 +1528,7 @@ impl BlockchainSource for ValidatorConnector { .collect(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into()), } } @@ -890,7 +1546,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(ReadRequest::Tip)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (chain_height, _chain_hash) = expected_read_response!(response, Tip) .ok_or_else(|| { @@ -917,9 +1573,7 @@ impl BlockchainSource for ValidatorConnector { addresses: GetAddressBalanceRequest::new(addresses) .valid_addresses() .map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "invalid address: {error}" - )) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?, height_range: zebra_chain::block::Height(start) @@ -929,7 +1583,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let hashes = expected_read_response!(response, AddressesTransactionIds); @@ -958,14 +1612,15 @@ impl BlockchainSource for ValidatorConnector { fetch .get_address_txids(addresses, start, end) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .transactions .iter() .map(|txid_string| { TransactionHash::from_hex(txid_string.as_bytes()).map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "invalid txid from getaddresstxids `{txid_string}`: {error}" - )) + BlockchainSourceError::unrecoverable_context( + format!("invalid txid from getaddresstxids `{txid_string}`"), + error, + ) }) }) .collect::, BlockchainSourceError>>() @@ -982,7 +1637,7 @@ impl BlockchainSource for ValidatorConnector { let mut state = state.read_state_service.clone(); let valid_addresses = addresses.valid_addresses().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid address: {error}")) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?; let request = ReadRequest::UtxosByAddresses(valid_addresses); @@ -990,7 +1645,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let utxos = expected_read_response!(response, AddressUtxos); let mut last_output_location = @@ -1033,7 +1688,7 @@ impl BlockchainSource for ValidatorConnector { .collect(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into_iter() .map(|utxos| utxos.into()) .collect()), @@ -1052,13 +1707,15 @@ impl BlockchainSource for ValidatorConnector { > { match self { ValidatorConnector::State(State { - read_state_service, - mempool_fetcher: _, - network: _, + read_state_service, .. }) => { match read_state_service .clone() - .call(zebra_state::ReadRequest::NonFinalizedBlocksListener) + // Empty `known_chain_tips` requests every block currently in the + // non-finalized state (the prior unit-variant behaviour). + .call(zebra_state::ReadRequest::NonFinalizedBlocksListener { + known_chain_tips: Default::default(), + }) .await { Ok(ReadResponse::NonFinalizedBlocksListener(listener)) => { @@ -1072,4 +1729,1259 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(_fetch) => Ok(None), } } + + fn shutdown(&self) { + // Only the `State` arm owns a long-lived resource: the Zebra chain-syncer + // task feeding the `ReadStateService`. Abort it so the backend drops cleanly. + // The `Fetch` arm holds only a stateless JSON-RPC client — nothing to tear down. + if let ValidatorConnector::State(State { + sync_task_handle: Some(handle), + .. + }) = self + { + handle.abort(); + } + } +} + +impl State { + /// Builds the `getblockheader`-shaped block header from the `ReadStateService`. + /// + /// Returns the zebra `GetBlockHeaderResponse` shape. This is the shared State-path + /// builder used by both [`ValidatorConnector::get_block_header`] (which converts it + /// to the JSON-RPC wire form) and [`State::get_block_inner`] (which reuses the + /// object fields to assemble a verbose block). + /// + /// Moved from the former `StateServiceSubscriber::get_block_header_inner`; the error + /// type is now [`BlockchainSourceError`] rather than the backend error, so the + /// zcashd-compatible `LegacyCode` on the not-in-best-chain case survives only in the + /// message string. + pub(super) async fn get_block_header_inner( + &self, + hash_or_height: HashOrHeight, + verbose: Option, + ) -> BlockchainSourceResult { + let mut state = self.read_state_service.clone(); + let verbose = verbose.unwrap_or(true); + let network = self.network.clone(); + + let ReadResponse::BlockHeader { + header, + hash, + height, + next_block_hash, + } = state + .ready() + .and_then(|service| service.call(ReadRequest::BlockHeader(hash_or_height))) + .await + .map_err(|_| { + BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) + })? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to BlockHeader request".to_string(), + )); + }; + + let response = if !verbose { + GetBlockHeaderResponse::Raw(HexData( + header + .zcash_serialize_to_vec() + .map_err(BlockchainSourceError::unrecoverable)?, + )) + } else { + let ReadResponse::SaplingTree(sapling_tree) = state + .ready() + .and_then(|service| service.call(ReadRequest::SaplingTree(hash_or_height))) + .await + .map_err(BlockchainSourceError::unrecoverable)? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to SaplingTree request".to_string(), + )); + }; + // This could be `None` if there's a chain reorg between state queries. + let sapling_tree = sapling_tree.ok_or_else(|| { + BlockchainSourceError::Unrecoverable("missing sapling tree for block".to_string()) + })?; + + let ReadResponse::Depth(depth) = state + .ready() + .and_then(|service| service.call(ReadRequest::Depth(hash))) + .await + .map_err(BlockchainSourceError::unrecoverable)? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to Depth request".to_string(), + )); + }; + + // Confirmations are one more than the depth. Depth is limited by height, so + // it will never overflow an i64. + let confirmations = confirmations_from_depth(depth); + + let sapling_tree_size = sapling_tree.count(); + let final_sapling_root = + final_sapling_root(sapling_tree.root().into(), height, &network); + + let block_header = build_block_header_object( + &header, + hash, + height, + confirmations, + final_sapling_root, + sapling_tree_size, + next_block_hash, + &network, + )?; + + GetBlockHeaderResponse::Object(Box::new(block_header)) + }; + + Ok(response) + } + + /// Builds the `getblock`-shaped verbose block from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_block_inner`; the error type is + /// now [`BlockchainSourceError`]. + pub(super) async fn get_block_inner( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + let mut state_1 = self.read_state_service.clone(); + let verbosity = verbosity.unwrap_or(1); + match verbosity { + 0 => { + let request = ReadRequest::Block(hash_or_height); + let response = state_1 + .ready() + .and_then(|service| service.call(request)) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + let block = expected_read_response!(response, Block); + block + .map(SerializedBlock::from) + .map(GetBlock::Raw) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block not found".to_string()) + }) + } + 1 | 2 => { + let network = self.network.clone(); + let state_2 = self.read_state_service.clone(); + let state_4 = self.read_state_service.clone(); + let state_5 = self.read_state_service.clone(); + + let blockandsize_future = { + let req = ReadRequest::BlockAndSize(hash_or_height); + async move { state_1.ready().and_then(|service| service.call(req)).await } + }; + let orchard_future = { + let req = ReadRequest::OrchardTree(hash_or_height); + async move { + state_2 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; + let ironwood_future = { + let req = ReadRequest::IronwoodTree(hash_or_height); + async move { + state_5 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; + let block_info_future = { + let req = ReadRequest::BlockInfo(hash_or_height); + async move { + state_4 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; + let (fullblock, orchard_tree_response, ironwood_tree_response, header, block_info) = futures::join!( + blockandsize_future, + orchard_future, + ironwood_future, + self.get_block_header_inner(hash_or_height, Some(true)), + block_info_future + ); + + let header_obj = match header? { + GetBlockHeaderResponse::Raw(_hex_data) => unreachable!( + "`true` was passed to get_block_header, an object should be returned" + ), + GetBlockHeaderResponse::Object(get_block_header_object) => { + get_block_header_object + } + }; + + let (block, size, block_info) = match (fullblock, block_info) { + ( + Ok(ReadResponse::BlockAndSize(Some((block, size)))), + Ok(ReadResponse::BlockInfo(Some(block_info))), + ) => (block, size, block_info), + (Ok(ReadResponse::Block(None)), Ok(ReadResponse::BlockInfo(None))) => { + return Err(BlockchainSourceError::Unrecoverable( + "block not found".to_string(), + )); + } + (Ok(unexpected), Ok(unexpected2)) => { + unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") + } + (Err(e), _) | (_, Err(e)) => { + return Err(BlockchainSourceError::unrecoverable(e)); + } + }; + + let orchard_tree_response = + orchard_tree_response.map_err(BlockchainSourceError::unrecoverable)?; + let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("missing orchard tree".to_string()) + })?; + + let ironwood_tree_response = + ironwood_tree_response.map_err(BlockchainSourceError::unrecoverable)?; + let ironwood_tree = expected_read_response!(ironwood_tree_response, IronwoodTree) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("missing ironwood tree".to_string()) + })?; + + let final_orchard_root = + final_orchard_root(orchard_tree.root().into(), header_obj.height(), &network); + + let (chain_supply, value_pools) = ( + Some(GetBlockchainInfoBalance::chain_supply( + *block_info.value_pools(), + )), + Some(GetBlockchainInfoBalance::value_pools( + *block_info.value_pools(), + None, + )), + ); + + Ok(build_verbose_block( + &header_obj, + &block, + verbosity, + size as i64, + final_orchard_root, + orchard_tree.count(), + ironwood_tree.count(), + chain_supply, + value_pools, + &network, + )) + } + more_than_two => Err(BlockchainSourceError::Unrecoverable(format!( + "invalid verbosity of {more_than_two}" + ))), + } + } + + /// Builds the `getblockdeltas` response from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_block_deltas`; resolves each + /// spent prevout via a best-chain `ReadRequest::Transaction` (finalised + + /// non-finalised), then hands off to the shared [`assemble_block_deltas`]. + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; + let GetBlock::Object(object) = self.get_block_inner(hash_or_height, Some(2)).await? else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected raw block".to_string(), + )); + }; + + // Per-call cache: many inputs may reference the same prevtxid, so each previous + // transaction is fetched at most once. + let mut prevtx_cache: HashMap< + zebra_chain::transaction::Hash, + Arc, + > = HashMap::new(); + for tx in object.tx() { + let GetBlockTransaction::Object(txo) = tx else { + continue; + }; + for input in txo.inputs() { + let Input::NonCoinbase { txid: prevtxid, .. } = input else { + continue; + }; + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(BlockchainSourceError::unrecoverable)?; + if prevtx_cache.contains_key(&prev_hash) { + continue; + } + let mut state = self.read_state_service.clone(); + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::Transaction(prev_hash))) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + let mined_tx = expected_read_response!(response, Transaction).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not in best chain" + )) + })?; + prevtx_cache.insert(prev_hash, mined_tx.tx); + } + } + + let median_time = self.median_time_past(&object).await?; + let network = self.network.clone(); + assemble_block_deltas(&object, &prevtx_cache, median_time, &network) + } + + /// Median time past over the 11-block window ending at `start`, walking backwards via + /// verbosity-1 `getblock` lookups against the `ReadStateService`. + // TODO(DRY): MockchainSource duplicates this walk; the only difference is the + // per-block fetch. A shared helper generic over an async block fetcher would unify them. + async fn median_time_past(&self, start: &BlockObject) -> BlockchainSourceResult { + const MEDIAN_TIME_PAST_WINDOW: usize = 11; + let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); + let start_time = start.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: start block missing time".into()) + })?; + times.push(start_time); + + let mut prev = start.previous_block_hash(); + for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { + let Some(hash) = prev else { + break; // genesis + }; + match self + .get_block_inner(HashOrHeight::Hash(hash), Some(1)) + .await + { + Ok(GetBlock::Object(object)) => { + if let Some(time) = object.time() { + times.push(time); + } + prev = object.previous_block_hash(); + } + Ok(GetBlock::Raw(_)) => break, + Err(_) => break, // use values collected so far + } + } + + median_of_block_times(times) + } + + /// Builds the `getblockchaininfo` response from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_blockchain_info`; error type is + /// now [`BlockchainSourceError`], and the difficulty is propagated rather than + /// unwrapped. + async fn get_blockchain_info(&self) -> BlockchainSourceResult { + let mut state = self.read_state_service.clone(); + let network = self.network.clone(); + + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::TipPoolValues)) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + let (height, hash, balance) = match response { + ReadResponse::TipPoolValues { + tip_height, + tip_hash, + value_balance, + } => (tip_height, tip_hash, value_balance), + unexpected => { + unreachable!("Unexpected response from state service: {unexpected:?}") + } + }; + + let usage_response = state + .ready() + .and_then(|service| service.call(ReadRequest::UsageInfo)) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + let size_on_disk = expected_read_response!(usage_response, UsageInfo); + + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::BlockHeader(hash.into()))) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + let header = match response { + ReadResponse::BlockHeader { header, .. } => header, + unexpected => { + unreachable!("Unexpected response from state service: {unexpected:?}") + } + }; + + let now = Utc::now(); + let zebra_estimated_height = + NetworkChainTipHeightEstimator::new(header.time, height, &network) + .estimate_height_at(now); + let estimated_height = if header.time > now || zebra_estimated_height < height { + height + } else { + zebra_estimated_height + }; + + let upgrades = IndexMap::from_iter(network.full_activation_list().into_iter().filter_map( + |(activation_height, network_upgrade)| { + // Zebra defines network upgrades by consensus rule changes, zcashd by ZIPs; + // upgrades with a consensus branch ID are the same in both. + network_upgrade.branch_id().map(|branch_id| { + // zcashd's RPC ignores Disabled network upgrades, so Zebra does too. + let status = if height >= activation_height { + NetworkUpgradeStatus::Active + } else { + NetworkUpgradeStatus::Pending + }; + ( + ConsensusBranchIdHex::new(branch_id.into()), + NetworkUpgradeInfo::from_parts(network_upgrade, activation_height, status), + ) + }) + }, + )); + + let next_block_height = + (height + 1).expect("valid chain tips are a lot less than Height::MAX"); + let consensus = TipConsensusBranch::from_parts( + ConsensusBranchIdHex::new( + NetworkUpgrade::current(&network, height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) + .into(), + ) + .inner(), + ConsensusBranchIdHex::new( + NetworkUpgrade::current(&network, next_block_height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) + .into(), + ) + .inner(), + ); + + let difficulty = + chain_tip_difficulty(network.clone(), self.read_state_service.clone(), false) + .await + .map_err(BlockchainSourceError::unrecoverable)?; + + let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); + + Ok(GetBlockchainInfoResponse::new( + network.bip70_network_name(), + height, + hash, + estimated_height, + GetBlockchainInfoBalance::chain_supply(balance), + // TODO: account for new delta_pools arg? + GetBlockchainInfoBalance::value_pools(balance, None), + upgrades, + consensus, + height, + difficulty, + verification_progress, + // TODO: store work in the finalized state for each height + // (see https://github.com/ZcashFoundation/zebra/issues/7109) + 0, + false, + size_on_disk, + // TODO (copied from zebra): investigate whether this needs implementing + // (it's sprout-only in zcashd) + 0, + )) + } +} + +/// Confirmations are one more than the depth, or -1 when the block is not on the best +/// chain. Depth is limited by height, so it never overflows an `i64`. +pub(crate) fn confirmations_from_depth(depth: Option) -> i64 { + const NOT_IN_BEST_CHAIN_CONFIRMATIONS: i64 = -1; + depth + .map(|depth| i64::from(depth) + 1) + .unwrap_or(NOT_IN_BEST_CHAIN_CONFIRMATIONS) +} + +/// The `finalsaplingroot` field: the sapling note-commitment tree root in display +/// (big-endian) byte order once Sapling has activated, else all-zero. +pub(crate) fn final_sapling_root( + root: [u8; 32], + height: zebra_chain::block::Height, + network: &zebra_chain::parameters::Network, +) -> [u8; 32] { + match NetworkUpgrade::Sapling.activation_height(network) { + Some(activation) if height >= activation => { + let mut root = root; + root.reverse(); + root + } + _ => [0; 32], + } +} + +/// The `finalorchardroot` field: `Some(root)` once NU5 has activated, else `None`. +pub(crate) fn final_orchard_root( + root: [u8; 32], + height: zebra_chain::block::Height, + network: &zebra_chain::parameters::Network, +) -> Option<[u8; 32]> { + match NetworkUpgrade::Nu5.activation_height(network) { + Some(activation) if height >= activation => Some(root), + _ => None, + } +} + +/// Assembles a `getblockheader` object from its already-resolved primitive pieces. +/// +/// Shared by the [`ValidatorConnector`] State path (which reads the pieces from the +/// `ReadStateService`) and by `MockchainSource` (which reads them from test vectors), +/// so the two never drift. +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_block_header_object( + header: &Header, + hash: zebra_chain::block::Hash, + height: zebra_chain::block::Height, + confirmations: i64, + final_sapling_root: [u8; 32], + sapling_tree_size: u64, + next_block_hash: Option, + network: &zebra_chain::parameters::Network, +) -> BlockchainSourceResult { + let mut nonce = *header.nonce; + nonce.reverse(); + let difficulty = header.difficulty_threshold.relative_to_network(network); + let block_commitments = + header_to_block_commitments(header, network, height, final_sapling_root)?; + + Ok(GetBlockHeaderObject::new( + hash, + confirmations, + height, + header.version, + header.merkle_root, + block_commitments, + final_sapling_root, + sapling_tree_size, + header.time.timestamp(), + nonce, + header.solution, + header.difficulty_threshold, + difficulty, + header.previous_block_hash, + next_block_hash, + )) +} + +/// Assembles a verbose (`verbosity` 1 or 2) `getblock` object from a resolved header +/// object plus the block's transactions and pool aggregates. +/// +/// Shared by the [`ValidatorConnector`] State path and `MockchainSource`. `chain_supply` +/// / `value_pools` are the cumulative pool balances (present for the validator, `None` +/// for the mock, whose vectors don't carry them). +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_verbose_block( + header_obj: &GetBlockHeaderObject, + block: &zebra_chain::block::Block, + verbosity: u8, + size: i64, + final_orchard_root: Option<[u8; 32]>, + orchard_tree_size: u64, + ironwood_tree_size: u64, + chain_supply: Option, + value_pools: Option<[GetBlockchainInfoBalance; 6]>, + network: &zebra_chain::parameters::Network, +) -> GetBlock { + let transactions_response: Vec = block + .transactions + .iter() + .map(|transaction| match verbosity { + 1 => GetBlockTransaction::Hash(transaction.hash()), + 2 => GetBlockTransaction::Object(Box::new(TransactionObject::from_transaction( + transaction.clone(), + Some(header_obj.height()), + Some(header_obj.confirmations()), + network, + DateTime::::from_timestamp(header_obj.time(), 0), + Some(header_obj.hash()), + // A non-optional header height indicates a mainchain block; sidechain + // data is out of scope for now. TODO: return Some(true/false) once resolved. + None, + transaction.hash(), + ))), + _ => unreachable!("verbosity known to be 1 or 2"), + }) + .collect(); + + let trees = GetBlockTrees::new( + header_obj.sapling_tree_size(), + orchard_tree_size, + ironwood_tree_size, + ); + let transaction_count = transactions_response.len(); + + GetBlock::Object(Box::new(BlockObject::new( + header_obj.hash(), + header_obj.confirmations(), + Some(size), + Some(header_obj.height()), + Some(header_obj.version()), + Some(header_obj.merkle_root()), + Some(header_obj.block_commitments()), + Some(header_obj.final_sapling_root()), + final_orchard_root, + transaction_count, + transactions_response, + Some(header_obj.time()), + Some(header_obj.nonce()), + Some(header_obj.solution()), + Some(header_obj.bits()), + Some(header_obj.difficulty()), + chain_supply, + value_pools, + trees, + Some(header_obj.previous_block_hash()), + header_obj.next_block_hash(), + ))) +} + +/// Computes the `blockcommitments` field for a block header. +/// +/// Moved verbatim (bar the error type) from the former `state.rs` +/// `header_to_block_commitments`. +pub(crate) fn header_to_block_commitments( + header: &Header, + network: &zebra_chain::parameters::Network, + height: zebra_chain::block::Height, + final_sapling_root: [u8; 32], +) -> BlockchainSourceResult<[u8; 32]> { + let hash = match header.commitment(network, height).map_err(|error| { + BlockchainSourceError::unrecoverable_context("invalid block commitment", error) + })? { + zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, + zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, + zebra_chain::block::Commitment::ChainHistoryActivationReserved => [0; 32], + zebra_chain::block::Commitment::ChainHistoryRoot(root) => root.bytes_in_display_order(), + zebra_chain::block::Commitment::ChainHistoryBlockTxAuthCommitment(hash) => { + hash.bytes_in_display_order() + } + }; + Ok(hash) +} + +/// Converts the zebra `GetBlockHeaderResponse` (Raw/Object) into the JSON-RPC wire +/// `getblockheader` response (Compact/Verbose) expected by the backends. +/// +/// The two shapes serialize to the same JSON, so a serde round-trip is the least-error- +/// prone conversion. +pub(crate) fn zebra_block_header_to_wire( + header: GetBlockHeaderResponse, +) -> BlockchainSourceResult { + let value = serde_json::to_value(&header).map_err(BlockchainSourceError::unrecoverable)?; + serde_json::from_value(value).map_err(BlockchainSourceError::unrecoverable) +} + +/// Returns the median of a non-empty set of block times. +pub(crate) fn median_of_block_times(mut times: Vec) -> BlockchainSourceResult { + if times.is_empty() { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: no block times collected for median".to_string(), + )); + } + times.sort_unstable(); + Ok(times[times.len() / 2]) +} + +/// Maps a verbose-transaction output to its `getblockdeltas` output delta, +/// or `None` when the output does not participate in address deltas: only +/// outputs with exactly one derivable address are attributed. Zero means no +/// address to credit (nonstandard script); more than one (e.g. bare multisig) +/// has no single owning address, and zcashd's getblockdeltas omits such +/// outputs rather than crediting the first address. +fn output_delta_from_verbose(vout: &zebra_rpc::client::Output) -> Option { + let address = + vout.script_pub_key() + .addresses() + .as_ref() + .and_then(|addresses| match addresses.as_slice() { + [address] => Some(address.clone()), + _ => None, + })?; + let satoshis: Amount = vout.value_zat().try_into().ok()?; + Some(OutputDelta { + address, + satoshis, + index: vout.n(), + }) +} + +/// Assembles a `getblockdeltas` response from a verbosity-2 `getblock` object plus a +/// cache of previous transactions (keyed by txid) needed to resolve each spend's address +/// and value, its median time, and the running network. +/// +/// Shared by the [`ValidatorConnector`] State path and `MockchainSource`: both resolve +/// the prevout transactions their own way, then hand the assembled inputs here so the +/// delta-shaping logic lives in one place. +pub(crate) fn assemble_block_deltas( + object: &BlockObject, + prevtx_cache: &HashMap< + zebra_chain::transaction::Hash, + Arc, + >, + median_time: i64, + network: &zebra_chain::parameters::Network, +) -> BlockchainSourceResult { + let mut deltas = Vec::with_capacity(object.tx().len()); + for (tx_index, tx) in object.tx().iter().enumerate() { + let GetBlockTransaction::Object(txo) = tx else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected hash when expecting object".to_string(), + )); + }; + let txid = txo.txid().to_string(); + + let mut inputs: Vec = Vec::new(); + for (i, vin) in txo.inputs().iter().enumerate() { + let (prevtxid, prevout) = match vin { + Input::Coinbase { .. } => continue, + Input::NonCoinbase { + txid: prevtxid, + vout: prevout, + .. + } => (prevtxid, *prevout), + }; + + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(BlockchainSourceError::unrecoverable)?; + let prev_tx = prevtx_cache.get(&prev_hash).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not resolved" + )) + })?; + + let output = prev_tx.outputs().get(prevout as usize).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout index {prevout} out of range for {prevtxid}" + )) + })?; + + // Nonstandard script ⇒ no derivable address ⇒ skip (matches the outputs branch). + let address = match output.address(network) { + Some(address) => address.to_string(), + None => continue, + }; + + // Inputs are debits, so the amount leaves the address. + let satoshis: Amount = (-output.value().zatoshis()).try_into().map_err(|error| { + BlockchainSourceError::unrecoverable_context( + format!("getblockdeltas: input amount out of range for {prevtxid}:{prevout}"), + error, + ) + })?; + + inputs.push(InputDelta { + address, + satoshis, + index: i as u32, + prevtxid: prevtxid.clone(), + prevout, + }); + } + + let outputs: Vec = txo + .outputs() + .iter() + .filter_map(output_delta_from_verbose) + .collect(); + + deltas.push(BlockDelta { + txid, + index: tx_index as u32, + inputs, + outputs, + }); + } + + Ok(BlockDeltas { + hash: object.hash().to_string(), + confirmations: object.confirmations(), + size: object.size().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block size missing".to_string()) + })?, + height: object + .height() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block height missing".to_string(), + ) + })? + .0, + version: object.version().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block version missing".to_string(), + ) + })?, + merkle_root: object + .merkle_root() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block merkle root missing".to_string(), + ) + })? + .encode_hex::(), + deltas, + time: object.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block time missing".to_string()) + })?, + median_time, + nonce: hex::encode(object.nonce().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block nonce missing".to_string()) + })?), + bits: object + .bits() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block bits missing".to_string(), + ) + })? + .to_string(), + difficulty: object.difficulty().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block difficulty missing".to_string(), + ) + })?, + previous_block_hash: object.previous_block_hash().map(|hash| hash.to_string()), + next_block_hash: object.next_block_hash().map(|hash| hash.to_string()), + }) +} + +#[cfg(test)] +mod zebra_block_header_to_wire { + use super::*; + + /// The Direct connection's non-verbose `getblockheader` builds a + /// `GetBlockHeaderResponse::Raw` and crosses to the wire type through the + /// serde round-trip under test: the raw hex must land in the untagged + /// `Compact` variant with the same bytes, or every non-verbose + /// `getblockheader` on a Direct connection errors at runtime. The + /// mockchain `get_block_header` test covers the same conversion end to + /// end; this pins it at its definition. + #[test] + fn raw_header_round_trips_to_compact_hex() { + let header_bytes = vec![0x01, 0x02, 0xab, 0xcd]; + let raw = GetBlockHeaderResponse::Raw(HexData(header_bytes.clone())); + + let wire = zebra_block_header_to_wire(raw) + .expect("the raw header shape must convert to the wire type"); + + assert_eq!(wire, GetBlockHeader::Compact(hex::encode(header_bytes))); + } +} + +#[cfg(test)] +mod fetch_pool_treestate_slot { + /// Regression test: the validator's finalRoot must pass through to the pool slot. + /// zebra populates `Commitments { finalRoot, finalState }` for every pool it + /// serves, but the slot mapping dropped the root, so zaino's z_gettreestate + /// responses carried no finalRoot for any pool. + /// + #[test] + fn final_root_passes_through() { + let final_root = vec![7u8; 32]; + let final_state = vec![1u8, 2, 3]; + let treestate = zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + Some(final_root.clone()), + Some(final_state.clone()), + )); + + let slot = super::fetch_pool_treestate_slot(treestate) + .expect("a treestate with a finalState maps to a populated slot"); + + assert_eq!(slot.final_state, final_state); + assert_eq!( + slot.final_root, + Some(final_root), + "the validator's finalRoot must pass through to the treestate slot" + ); + } + + /// An absent finalState maps to an absent slot. + #[test] + fn absent_final_state_maps_to_absent_slot() { + let treestate = + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new(None, None)); + assert_eq!(super::fetch_pool_treestate_slot(treestate), None); + } +} + +/// Clamps a `getaddressdeltas` height range to the current best tip: +/// `end == 0` means "to the tip", and both bounds clamp down to it. A source +/// that reports no best height (nothing indexed yet) is a typed error. +fn clamp_deltas_range_to_tip( + tip: Option, + start_raw: u32, + end_raw: u32, +) -> BlockchainSourceResult<(Height, Height)> { + let tip: Height = tip + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getaddressdeltas: the source reports no best block height".to_string(), + ) + })? + .into(); + let mut start = Height(start_raw); + let mut end = Height(end_raw); + if end == Height(0) || end > tip { + end = tip; + } + if start > tip { + start = tip; + } + Ok((start, end)) +} + +#[cfg(test)] +mod output_delta_from_verbose { + use super::*; + + fn output_with_addresses(addresses: Option>) -> zebra_rpc::client::Output { + zebra_rpc::client::Output::new( + 0.00000001, + 1, + 0, + zebra_rpc::client::ScriptPubKey::new( + "asm".to_string(), + zebra_chain::transparent::Script::new(&[0u8]), + addresses.as_ref().map(|a| a.len() as u32), + "pubkeyhash".to_string(), + addresses, + ), + ) + } + + /// An output with exactly one derivable address is attributed to it. + #[test] + fn single_address_output_is_attributed() { + let delta = + output_delta_from_verbose(&output_with_addresses(Some(vec!["t1address".to_string()]))) + .expect("a single-address output participates in deltas"); + assert_eq!(delta.address, "t1address"); + } + + /// An output with multiple derivable addresses (e.g. bare multisig) has no + /// single owning address: zcashd's getblockdeltas omits it, and crediting + /// the first address would fabricate balance for it. The pre-fix code took + /// `addresses.first()`. + #[test] + fn multi_address_output_is_omitted() { + assert_eq!( + output_delta_from_verbose(&output_with_addresses(Some(vec![ + "t1first".to_string(), + "t1second".to_string(), + ]))), + None, + "a multi-address output must be omitted, not attributed to the first address" + ); + } + + /// No derivable address (nonstandard script) means no delta. + #[test] + fn addressless_output_is_omitted() { + assert_eq!( + output_delta_from_verbose(&output_with_addresses(None)), + None + ); + } +} + +#[cfg(test)] +mod clamp_deltas_range_to_tip { + use super::*; + + /// A source that reports no best height (nothing indexed yet, or the + /// RPC fallback found no tip) must yield a typed error, not a panic: + /// this range clamp previously lived inline in `get_address_deltas` + /// behind `get_best_block_height().await?.unwrap()`. + #[test] + fn absent_tip_is_a_typed_error() { + assert!( + clamp_deltas_range_to_tip(None, 0, 0).is_err(), + "an absent tip must surface as a BlockchainSourceError" + ); + } + + /// `end == 0` means "to the tip", and both bounds clamp down to the tip. + #[test] + fn bounds_clamp_to_tip() { + let tip = Some(zebra_chain::block::Height(100)); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 5, 0).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(5), Height(100))); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 5, 400).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(5), Height(100))); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 300, 50).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(100), Height(50))); + } +} + +#[cfg(test)] +mod get_block_verbose { + use super::*; + + /// zaino-serve recovers zcashd-compatible RPC error codes by downcast-walking + /// [`std::error::Error::source`] chains (see + /// `getblock_error_object_from_indexer_error` and + /// `sendrawtransaction_error_object_from_indexer_error` in + /// `zaino-serve/src/rpc/jsonrpc/service.rs`). The connector boundary must + /// therefore preserve the typed cause instead of flattening it to a string, + /// or `getblock` failures surface as generic internal errors rather than the + /// legacy `-8` code lightwalletd-family clients key on. + #[tokio::test] + async fn fetch_error_keeps_transport_error_in_source_chain() { + // Port 1 refuses connections, so the request fails at the transport + // layer without contacting any validator. + let connector = zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector::new_with_basic_auth( + "http://127.0.0.1:1/" + .parse() + .expect("static url literal parses"), + "user".to_string(), + "password".to_string(), + ) + .expect("connector construction is network-free"); + let source = ValidatorConnector::Fetch(connector); + + let error = source + .get_block_verbose(HashOrHeight::Height(zebra_chain::block::Height(1)), Some(1)) + .await + .expect_err("a request to a closed port must fail"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut transport_error_reachable = false; + while let Some(source_error) = current { + if source_error + .downcast_ref::() + .is_some() + { + transport_error_reachable = true; + break; + } + current = source_error.source(); + } + assert!( + transport_error_reachable, + "the typed TransportError must stay reachable via the source() chain; \ + stringifying it strips the RPC error code the serve layer recovers" + ); + } +} + +#[cfg(test)] +mod ironwood_treestate_slot { + /// Regression test: when the validator reported no ironwood treestate (below NU6.3 + /// activation, or on a network with no NU6.3 activation height) the slot must stay + /// `None`, so z_gettreestate omits the ironwood field exactly as zebrad does + /// ("Only present from NU6.3, so that pre-NU6.3 responses are unchanged"). The slot + /// was previously back-filled with a serialized empty tree, emitting the field at + /// every height on every network. + #[test] + fn absent_validator_ironwood_stays_absent() { + assert_eq!( + super::ironwood_treestate_slot(None), + None, + "ironwood slot must stay absent when the validator reported no ironwood treestate" + ); + } + + /// A reported ironwood treestate passes through unchanged. + #[test] + fn reported_validator_ironwood_passes_through() { + let treestate = crate::chain_index::source::PoolTreestate { + final_root: None, + final_state: vec![1u8, 2, 3], + }; + assert_eq!( + super::ironwood_treestate_slot(Some(treestate.clone())), + Some(treestate) + ); + } +} + +#[cfg(test)] +mod activation_heights_from_upgrades { + use zaino_common::config::network::ActivationHeights; + + /// All-`None` heights: the starting point adoption fills from the + /// validator's map, and the expected value for every absent upgrade. + const NEVER_ACTIVATED: ActivationHeights = ActivationHeights::NEVER_ACTIVATED; + + fn upgrades_map( + json: &str, + ) -> indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + > { + serde_json::from_str(json).expect("upgrades fixture parses") + } + + fn adopted_heights( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, + ) -> ActivationHeights { + super::activation_heights_from_upgrades(upgrades).expect("valid schedule") + } + + /// An upgrade absent from the validator's map is never-activated — + /// nothing is backfilled from any default schedule. + #[test] + fn regtest_network_from_upgrades_leaves_absent_upgrades_never_activated() { + let upgrades = upgrades_map( + r#"{ + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU6", "activationheight": 2, "status": "active" } + }"#, + ); + + assert_eq!( + adopted_heights(&upgrades), + ActivationHeights { + nu5: Some(2), + nu6: Some(2), + ..NEVER_ACTIVATED + } + ); + } + + /// The ORCHARD_THEN_IRONWOOD transition shape: everything through NU6.2 + /// at 1–2, NU6.3 at 6 — the schedule the ironwood_activation fixtures + /// launch validators with. + #[test] + fn regtest_network_from_upgrades_reads_a_transition_schedule() { + let upgrades = upgrades_map( + r#"{ + "5ba81b19": { "name": "Overwinter", "activationheight": 1, "status": "active" }, + "76b809bb": { "name": "Sapling", "activationheight": 1, "status": "active" }, + "2bb40e60": { "name": "Blossom", "activationheight": 1, "status": "active" }, + "f5b9230b": { "name": "Heartwood", "activationheight": 1, "status": "active" }, + "e9ff75a6": { "name": "Canopy", "activationheight": 1, "status": "active" }, + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU6", "activationheight": 2, "status": "active" }, + "4dec4df0": { "name": "NU6.1", "activationheight": 2, "status": "active" }, + "5437f330": { "name": "NU6.2", "activationheight": 2, "status": "active" }, + "37a5165b": { "name": "NU6.3", "activationheight": 6, "status": "pending" } + }"#, + ); + + assert_eq!( + adopted_heights(&upgrades), + ActivationHeights { + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(6), + ..NEVER_ACTIVATED + } + ); + } + + /// A validator reporting the same upgrade twice is nonsense; adoption + /// must fail loudly rather than pick a height. + #[test] + fn regtest_network_from_upgrades_rejects_a_duplicate_upgrade() { + let upgrades = upgrades_map( + r#"{ + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU5", "activationheight": 3, "status": "pending" } + }"#, + ); + + let reason = + super::activation_heights_from_upgrades(&upgrades).expect_err("duplicate must fail"); + assert!( + reason.contains("twice"), + "error should name the duplication, got: {reason}" + ); + } +} + +#[cfg(test)] +mod verify_reported_upgrades { + fn upgrades_map( + json: &str, + ) -> indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + > { + serde_json::from_str(json).expect("upgrades fixture parses") + } + + /// A report agreeing with the compiled schedule passes. + #[test] + fn accepts_a_report_matching_the_compiled_schedule() { + let upgrades = upgrades_map( + r#"{ + "76b809bb": { "name": "Sapling", "activationheight": 419200, "status": "active" } + }"#, + ); + + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect("mainnet Sapling at 419200 matches the compiled parameters"); + } + + /// A reported height disagreeing with the compiled schedule fails loud — + /// the wrong-schedule / wrong-network drift class. + #[test] + fn rejects_a_mismatched_height() { + let upgrades = upgrades_map( + r#"{ + "76b809bb": { "name": "Sapling", "activationheight": 419201, "status": "active" } + }"#, + ); + + let reason = + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect_err("a wrong Sapling height must be rejected"); + assert!( + reason.contains("419201"), + "error should name the reported height, got: {reason}" + ); + } + + /// An upgrade the compiled schedule never activates must also be + /// rejected when the validator reports a height for it. + #[test] + fn rejects_an_upgrade_the_compiled_schedule_lacks() { + let upgrades = upgrades_map( + r#"{ + "77190ad8": { "name": "NU7", "activationheight": 123, "status": "pending" } + }"#, + ); + + let reason = + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect_err("an unscheduled upgrade with a reported height must be rejected"); + assert!( + reason.contains("None"), + "error should show the compiled schedule has no height, got: {reason}" + ); + } } diff --git a/packages/zaino-state/src/chain_index/tests.rs b/packages/zaino-state/src/chain_index/tests.rs index d2cd1d58e..47e954214 100644 --- a/packages/zaino-state/src/chain_index/tests.rs +++ b/packages/zaino-state/src/chain_index/tests.rs @@ -26,7 +26,7 @@ use std::path::{Path, PathBuf}; use tempfile::TempDir; use tokio::sync::OnceCell; use tokio::time::Duration; -use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; +use zaino_common::{network::ActivationHeights, DatabaseConfig, StorageConfig}; use crate::{ chain_index::{ @@ -134,7 +134,7 @@ async fn load_with_settings( }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let indexer = NodeBackedChainIndex::new_with_sync_timings(source.clone(), config, sync_timings) @@ -222,7 +222,7 @@ async fn v1_finalised_seed_dir(mode: MockchainMode) -> &'static Path { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source).await.unwrap(); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs index d2b455b22..34475c4ef 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use crate::chain_index::finalised_state::FinalisedState; @@ -45,7 +45,7 @@ pub(crate) async fn spawn_ephemeral_finalised_state( }, ephemeral: true, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let finalised_state = FinalisedState::spawn(config, source).await?; diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs index e0058faf6..11b399a6e 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs @@ -10,3 +10,5 @@ mod v1_0_to_v1_1; #[cfg(not(feature = "transparent_address_history_experimental"))] mod v1_1_to_v1_2; +#[cfg(not(feature = "transparent_address_history_experimental"))] +mod v1_2_to_v1_3; diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs index 735060d1e..3f70ca3aa 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use crate::chain_index::finalised_state::capability::{ DbCore as _, DbRead as _, DbVersion, MigrationStatus, @@ -34,7 +34,7 @@ async fn v1_0_to_v1_1_metadata_migration() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(150, blocks.clone()); @@ -51,7 +51,11 @@ async fn v1_0_to_v1_1_metadata_migration() { .await .unwrap(); - zaino_db.wait_until_ready().await; + // `wait_until_synced` (not `wait_until_ready`): a v1.1.0 database keeps its commitment rows in + // the legacy `commitment_tree_data_1_0_0` table (the v1.2.1 -> v1.3.0 migration has not run), so + // the background validator — which reads the v1.3.0 commitment table — can never reach the ready + // state. `wait_until_synced` waits for the migration to complete, which is what this test needs. + zaino_db.wait_until_synced().await; let metadata = zaino_db.get_metadata().await.unwrap(); @@ -96,7 +100,7 @@ async fn v1_0_to_v1_1_mixed_blockheaderdata_formats() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs index e0827f148..bfa0a32ef 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs @@ -4,13 +4,13 @@ use lmdb::{Cursor as _, Transaction as _, WriteFlags}; use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use crate::chain_index::finalised_state::capability::{ - BlockCoreExt as _, BlockTransparentExt as _, CapabilityRequest, DbRead as _, DbVersion, - MigrationStatus, + BlockCoreExt as _, CapabilityRequest, DbRead as _, DbVersion, MigrationStatus, + TransparentHistExt as _, }; -use crate::chain_index::finalised_state::entry::StoredEntryFixed; +use crate::chain_index::finalised_state::entry::{StoredEntryFixed, StoredEntryVar}; use crate::chain_index::finalised_state::finalised_source::v1::DB_SCHEMA_V1_HASH; use crate::chain_index::finalised_state::finalised_source::v1::TX_OUT_SET_INFO_ACCUMULATOR_KEY; use crate::chain_index::finalised_state::finalised_source::FinalisedSource; @@ -20,7 +20,134 @@ use crate::chain_index::tests::init_tracing; use crate::chain_index::tests::vectors::{ build_active_mockchain_source, load_test_vectors, TestVectorData, }; -use crate::{ChainIndexConfig, Height, TxLocation, ZainoVersionedSerde as _}; +use crate::{ChainIndexConfig, Height, TransparentTxList, TxLocation, ZainoVersionedSerde as _}; + +/// Reads a block's `TransparentTxList` **directly** from the `transparent` table, bypassing the +/// validated accessor (`get_block_transparent`), which routes through `validate_block_blocking` and +/// would read the v1.3.0 commitment table. On a database that has not yet run the v1.2.1 -> v1.3.0 +/// migration the commitment rows are still in the legacy table, so validation would fail; the +/// migration data these tests assert on (`txid_location`, `spent`, txout-set) is unaffected. +fn read_block_transparent_direct( + database_backend: &FinalisedSource, + height: Height, +) -> TransparentTxList { + use lmdb::Transaction as _; + let environment = database_backend.env().unwrap(); + let transparent_database = database_backend.transparent_db().unwrap(); + let transaction = environment.begin_ro_txn().unwrap(); + let raw = transaction + .get(transparent_database, &height.to_bytes().unwrap()) + .unwrap(); + StoredEntryVar::::from_bytes(raw) + .unwrap() + .inner() + .clone() +} + +/// Direct-read copy of the production oracle +/// (`tx_out_set_accumulator::expected_tx_out_set_info_accumulator`), differing only in that it reads +/// the transparent list directly (`read_block_transparent_direct`) instead of through the validated +/// `get_block_transparent`, so it works on a database whose commitment rows are still in the legacy +/// table (validation would fail there). The oracle logic is otherwise identical. +async fn expected_tx_out_set_info_accumulator_direct( + database_backend: &FinalisedSource, + max_height: Height, +) -> crate::chain_index::types::db::metadata::FinalisedTxOutSetInfoAccumulator { + use lmdb::Transaction as _; + + let environment = database_backend.env().unwrap(); + let spent_database = database_backend.spent_db().unwrap(); + + let mut expected_accumulator = + crate::chain_index::types::db::metadata::FinalisedTxOutSetInfoAccumulator::empty(); + + for height_raw in 0..=max_height.0 { + let height = Height(height_raw); + let transparent_transaction_list = read_block_transparent_direct(database_backend, height); + + for (transaction_index, transparent_transaction_opt) in + transparent_transaction_list.tx().iter().enumerate() + { + let Some(transparent_transaction) = transparent_transaction_opt else { + continue; + }; + if transparent_transaction.outputs().is_empty() { + continue; + } + + let transaction_index = u16::try_from(transaction_index).unwrap(); + let transaction_location = TxLocation::new(height.0, transaction_index); + // `get_txid` reads the `txids` table directly (no validation). + let transaction_hash = database_backend + .get_txid(transaction_location) + .await + .unwrap(); + + let mut unspent_outputs_for_transaction = 0u64; + let read_transaction = environment.begin_ro_txn().unwrap(); + + for (output_index, output) in transparent_transaction.outputs().iter().enumerate() { + if crate::chain_index::types::db::metadata::is_unspendable_tx_out(output) { + continue; + } + + let output_index = u32::try_from(output_index).unwrap(); + let outpoint = crate::Outpoint::new(transaction_hash.0, output_index); + let outpoint_bytes = outpoint.to_bytes().unwrap(); + + let still_unspent = match read_transaction.get(spent_database, &outpoint_bytes) { + Ok(spent_bytes) => { + let spent_entry = + StoredEntryFixed::::from_bytes(spent_bytes).unwrap(); + assert!( + spent_entry.verify(&outpoint_bytes), + "spent checksum mismatch for outpoint {outpoint:?}" + ); + spent_entry.inner().block_height() > max_height.0 + } + Err(lmdb::Error::NotFound) => true, + Err(error) => { + panic!("failed to read spent entry for outpoint {outpoint:?}: {error}") + } + }; + + if still_unspent { + unspent_outputs_for_transaction += 1; + expected_accumulator + .apply_added_output(&outpoint, output) + .unwrap(); + } + } + + if unspent_outputs_for_transaction > 0 { + expected_accumulator.transactions += 1; + } + } + } + + expected_accumulator +} + +/// Test assertion: the backend's maintained txout-set accumulator equals the independently recomputed +/// [`expected_tx_out_set_info_accumulator_direct`]. Direct-read equivalent of the production +/// `assert_tx_out_set_info_accumulator_matches_transparent_data`. +async fn assert_tx_out_set_info_accumulator_matches_transparent_data_direct( + database_backend: &FinalisedSource, +) { + let database_height = database_backend.db_height().await.unwrap().unwrap(); + let expected_accumulator = + expected_tx_out_set_info_accumulator_direct(database_backend, database_height).await; + + let actual_accumulator = database_backend + .get_tx_out_set_info_accumulator() + .await + .unwrap(); + + assert_eq!( + actual_accumulator, expected_accumulator, + "txout-set accumulator does not match transparent data and spent index" + ); +} const MIGRATION_SPENT_PROGRESS_KEY: &[u8] = b"_migration_spent_progress_1_2_0_next_height"; @@ -66,10 +193,7 @@ async fn assert_txid_location_index_matches_block_data( for height_raw in 0..=database_height.0 { let height = Height(height_raw); - let transparent_transaction_list = database_backend - .get_block_transparent(height) - .await - .unwrap(); + let transparent_transaction_list = read_block_transparent_direct(database_backend, height); for transaction_index in 0..transparent_transaction_list.tx().len() { let expected_location = TxLocation::new(height.0, transaction_index as u16); @@ -123,7 +247,7 @@ async fn simulate_interrupted_v1_1_to_v1_2_spent_index_migration( let spent_database = database_backend.spent_db().unwrap(); let (tx_out_set_info_accumulator_database, expected_resume_accumulator) = ( database_backend.tx_out_set_info_accumulator_db().unwrap(), - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::expected_tx_out_set_info_accumulator(database_backend, resume_height - 1).await, + expected_tx_out_set_info_accumulator_direct(database_backend, resume_height - 1).await, ); let spent_keys_to_delete: Vec> = { @@ -230,10 +354,7 @@ async fn assert_spent_index_matches_transparent_data( for height_raw in 0..=database_height.0 { let height = Height(height_raw); - let transparent_transaction_list = database_backend - .get_block_transparent(height) - .await - .unwrap(); + let transparent_transaction_list = read_block_transparent_direct(database_backend, height); let transaction = environment.begin_ro_txn().unwrap(); @@ -305,7 +426,7 @@ async fn v1_1_to_v1_2_spent_index_backfill_from_old_version() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); @@ -350,7 +471,7 @@ async fn v1_1_to_v1_2_spent_index_backfill_from_old_version() { assert_txid_location_index_matches_block_data(&migrated_backend).await; assert_spent_index_matches_transparent_data(&migrated_backend).await; - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::assert_tx_out_set_info_accumulator_matches_transparent_data(&migrated_backend).await; + assert_tx_out_set_info_accumulator_matches_transparent_data_direct(&migrated_backend).await; migrated_database.shutdown().await.unwrap(); } @@ -377,7 +498,7 @@ async fn v1_1_to_v1_2_spent_index_migration_resumes_after_crash() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); @@ -447,7 +568,7 @@ async fn v1_1_to_v1_2_spent_index_migration_resumes_after_crash() { assert_txid_location_index_matches_block_data(&resumed_backend).await; assert_spent_index_matches_transparent_data(&resumed_backend).await; - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::assert_tx_out_set_info_accumulator_matches_transparent_data(&resumed_backend).await; + assert_tx_out_set_info_accumulator_matches_transparent_data_direct(&resumed_backend).await; resumed_database.shutdown().await.unwrap(); } @@ -476,7 +597,7 @@ async fn v1_2_0_cache_missing_txid_location_index_is_rebuilt() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); @@ -533,7 +654,7 @@ async fn v1_2_0_cache_missing_txid_location_index_is_rebuilt() { assert_txid_location_index_matches_block_data(&healed_backend).await; assert_spent_index_matches_transparent_data(&healed_backend).await; - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::assert_tx_out_set_info_accumulator_matches_transparent_data(&healed_backend).await; + assert_tx_out_set_info_accumulator_matches_transparent_data_direct(&healed_backend).await; healed_database.shutdown().await.unwrap(); } diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs new file mode 100644 index 000000000..b7ac41d76 --- /dev/null +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs @@ -0,0 +1,102 @@ +//! V1.2.1 to V1.3.0 migration tests. +//! +//! Coverage note: these tests use `ActivationHeights::default()`, whose NU6.3 activation is `None`, +//! so the migration takes the below-activation branch (rebuild each commitment row in place from the +//! legacy fixed-length table, no ironwood). The at/above-activation *ironwood backfill* branch — +//! which refetches block data from the validator — cannot be exercised yet: `MockchainSource` +//! serves no ironwood commitment roots (see its `get_commitment_tree_roots` TODO) and the test +//! vectors carry no ironwood actions, so building a post-NU6.3 block would fail resolving the +//! (required) ironwood root. That path needs ironwood-capable test vectors before it can be tested. + +use std::path::PathBuf; +use tempfile::TempDir; +use zaino_common::network::ActivationHeights; +use zaino_common::{DatabaseConfig, StorageConfig}; + +use crate::chain_index::finalised_state::capability::{DbVersion, MigrationStatus}; +use crate::chain_index::finalised_state::finalised_source::v1::DB_VERSION_V1; +use crate::chain_index::finalised_state::FinalisedState; +use crate::chain_index::tests::init_tracing; +use crate::chain_index::tests::vectors::{ + build_active_mockchain_source, load_test_vectors, TestVectorData, +}; +use crate::{ChainIndexConfig, Height, StatusType}; + +fn v1_2_1() -> DbVersion { + DbVersion { + major: 1, + minor: 2, + patch: 1, + } +} + +/// Regression test for the startup ordering bug: opening a pre-1.3.0 cache used to start the +/// background validator concurrently with the v1.2.1 → v1.3.0 migration. The validator's +/// `initial_block_scan` read the freshly-created-empty `commitment_tree_data_1_3_0` table before the +/// migration rebuilt it, failing with `MDB_NOTFOUND` ("block scan") and latching `CriticalError`. +/// +/// The fix defers the validator until every migration finishes. This test builds an on-disk v1.2.1 +/// database, then reopens it through the production `FinalisedState::spawn` path (which migrates to +/// the current schema and only then starts the validator) and asserts it reaches `Ready` — i.e. the +/// migration completed *before* validation, and validation then passed. +// multi_thread required: the background validator runs blocking LMDB validation +// (`validate_block_blocking`) inline on its task, which would starve this test's status polling on +// a current-thread runtime. +#[tokio::test(flavor = "multi_thread")] +async fn v1_2_1_cache_migrates_to_current_then_validates() { + init_tracing(); + + let TestVectorData { blocks, .. } = load_test_vectors().unwrap(); + let active_height = Height(150); + + let temporary_directory: TempDir = tempfile::tempdir().unwrap(); + let database_path: PathBuf = temporary_directory.path().to_path_buf(); + + let database_config = ChainIndexConfig { + storage: StorageConfig { + database: DatabaseConfig { + path: database_path, + ..Default::default() + }, + ..Default::default() + }, + ephemeral: false, + db_version: 1, + network: ActivationHeights::default().to_regtest_network(), + }; + + let source = build_active_mockchain_source(active_height.0, blocks.clone()); + + // Build an on-disk v1.2.1 database (the pre-1.3.0 cache shape), then release it. + let old_database = + FinalisedState::build_db_to_version(database_config.clone(), source.clone(), v1_2_1()) + .await + .unwrap(); + old_database.wait_until_synced().await; + assert_eq!(old_database.get_metadata().await.unwrap().version, v1_2_1()); + old_database.shutdown().await.unwrap(); + drop(old_database); + + // Reopen through the production spawn path: it runs the v1.2.1 → v1.3.0 migration and only then + // starts the validator. Before the ordering fix this raced the migration and latched + // `CriticalError`; now it must migrate first and validate cleanly. + let migrated_database = FinalisedState::spawn(database_config.clone(), source.clone()) + .await + .unwrap(); + migrated_database.wait_until_synced().await; + + assert_eq!( + migrated_database.status(), + StatusType::Ready, + "the validator must run only after the migration completes, and then reach Ready" + ); + + let migrated_metadata = migrated_database.get_metadata().await.unwrap(); + assert_eq!(migrated_metadata.version, DB_VERSION_V1); + assert_eq!(migrated_metadata.migration_status, MigrationStatus::Empty); + + let migrated_height = migrated_database.db_height().await.unwrap().unwrap(); + assert_eq!(migrated_height, active_height); + + migrated_database.shutdown().await.unwrap(); +} diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index 496de74da..d3febab5b 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -4,10 +4,9 @@ use hex::ToHex; use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; -use crate::chain_index::finalised_state::capability::IndexedBlockExt; use crate::chain_index::finalised_state::finalised_source::FinalisedSource; use crate::chain_index::finalised_state::reader::DbReader; use crate::chain_index::finalised_state::FinalisedState; @@ -20,8 +19,12 @@ use crate::chain_index::tests::vectors::{ use crate::chain_index::types::TransactionHash; +use crate::chain_index::finalised_state::entry::StoredEntryVar; use crate::error::FinalisedStateError; -use crate::{BlockMetadata, BlockWithMetadata, ChainIndexConfig, Height, IndexedBlock}; +use crate::{ + BlockHeaderData, BlockMetadata, BlockWithMetadata, ChainIndexConfig, Height, IndexedBlock, + ZainoVersionedSerde as _, +}; use crate::{AddrScript, Outpoint}; @@ -41,7 +44,7 @@ pub(crate) async fn spawn_v1_zaino_db( }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source).await.unwrap(); @@ -88,6 +91,35 @@ pub(crate) async fn load_vectors_v1db_and_reader() -> ( // *** FinalisedState Tests *** +/// Regression test: blocks with no ironwood data must have NO ironwood row. +/// +/// The ironwood table is sparse — readers treat an absent row as "no ironwood data" +/// (an absent row reads back as an empty list; a stored all-`None` list reads back +/// with one `None` per transaction). The write path instead wrote an all-`None` +/// `OrchardTxList` for every block, paying one serialization + LMDB put per block +/// across the entire pre-NU6.3 chain for zero information. +/// +/// The test vectors are pre-NU6.3 regtest blocks, so no block carries ironwood data. +/// +/// multi_thread required: DbV1 reads run under `tokio::task::block_in_place`. +#[tokio::test(flavor = "multi_thread")] +async fn no_ironwood_row_for_blocks_without_ironwood_data() { + init_tracing(); + + let (_test_vector_data, _db_dir, _zaino_db, db_reader) = load_vectors_v1db_and_reader().await; + + let ironwood_list = db_reader + .get_block_ironwood(crate::Height(1)) + .await + .unwrap(); + assert!( + ironwood_list.tx().is_empty(), + "no ironwood row may be written for a block without ironwood data \ + (read back {} entries; an absent row reads back as an empty list)", + ironwood_list.tx().len(), + ); +} + #[tokio::test(flavor = "multi_thread")] async fn shutdown_returns_promptly() { super::assert_shutdown_returns_promptly("DbV1", spawn_v1_zaino_db).await; @@ -161,7 +193,7 @@ async fn save_db_to_file_and_reload() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_mockchain_source(blocks.clone()); @@ -240,29 +272,41 @@ async fn load_db_backend_from_file() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let finalized_state_backend: FinalisedSource = FinalisedSource::spawn_v1(&config).await.unwrap(); + // Read block headers directly from the `headers` table rather than via `get_chain_block`, which + // reconstructs the full block and validates (reading the v1.3.0 commitment table). This fixture + // is a legacy database whose commitment rows are in `commitment_tree_data_1_0_0`, so validation + // would fail; the header context asserted here is unaffected. + let read_header_direct = |height: Height| -> Option { + use lmdb::Transaction as _; + let environment = finalized_state_backend.env().unwrap(); + let headers_database = environment.open_db(Some("headers_1_0_0")).unwrap(); + let transaction = environment.begin_ro_txn().unwrap(); + match transaction.get(headers_database, &height.to_bytes().unwrap()) { + Ok(raw) => Some( + *StoredEntryVar::::from_bytes(raw) + .unwrap() + .inner(), + ), + Err(lmdb::Error::NotFound) => None, + Err(error) => panic!("failed to read header at height {}: {error}", height.0), + } + }; + let mut prev_hash = None; for height in 0..=100 { - let block = finalized_state_backend - .get_chain_block(Height(height)) - .await - .unwrap() - .unwrap(); + let header = read_header_direct(Height(height)).unwrap(); if let Some(prev_hash) = prev_hash { - assert_eq!(prev_hash, block.context.parent_hash); + assert_eq!(prev_hash, header.context.parent_hash); } - prev_hash = Some(block.context.index.hash); - assert_eq!(block.context.index.height, Height(height)); + prev_hash = Some(header.context.index.hash); + assert_eq!(header.context.index.height, Height(height)); } - assert!(finalized_state_backend - .get_chain_block(Height(101)) - .await - .unwrap() - .is_none()); + assert!(read_header_direct(Height(101)).is_none()); std::fs::remove_file(db_path.join("regtest").join("v1").join("lock.mdb")).unwrap() } @@ -292,8 +336,9 @@ async fn try_write_invalid_block() { sapling_tree_size as u32, orchard_root, orchard_tree_size as u32, + None, None, // no parent chainwork for this test - zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network(), + ActivationHeights::default().to_regtest_network(), ); let mut chain_block = diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 31518411e..2ee2de790 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -7,7 +7,7 @@ use crate::{ vectors::{indexed_block_chain, load_test_vectors, TestVectorBlockData}, }, types::{BestChainLocation, ChainScope, TransactionHash}, - ChainIndex, NodeBackedChainIndexSubscriber, + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndexSubscriber, }, BlockchainSource as _, Outpoint, }; @@ -16,8 +16,11 @@ use tokio_stream::StreamExt as _; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; -use zebra_chain::serialization::ZcashDeserializeInto; +use zaino_fetch::jsonrpsee::response::block_header::GetBlockHeader; +use zebra_chain::serialization::{ZcashDeserializeInto, ZcashSerialize as _}; use zebra_rpc::client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; +use zebra_rpc::methods::GetBlock; +use zebra_state::HashOrHeight; /// Polls the indexer's nonfinalized-state snapshot until its best-tip height /// equals `expected`, or panics after a 10 s budget. @@ -592,18 +595,18 @@ async fn get_treestate() { .. } in blocks.into_iter() { - let (sapling_bytes_opt, orchard_bytes_opt) = index_reader + let (sapling_bytes_opt, orchard_bytes_opt, _ironwood_bytes_opt) = index_reader .get_treestate(&crate::BlockHash(zebra_block.hash().0)) .await .unwrap(); assert_eq!( - sapling_bytes_opt.as_deref(), - Some(sapling_tree_state.as_slice()) + sapling_bytes_opt.map(|pool| pool.final_state), + Some(sapling_tree_state) ); assert_eq!( - orchard_bytes_opt.as_deref(), - Some(orchard_tree_state.as_slice()) + orchard_bytes_opt.map(|pool| pool.final_state), + Some(orchard_tree_state) ); } @@ -904,3 +907,397 @@ async fn get_outpoint_spenders_empty_and_single() { vec![None], ); } + +/// `z_get_block` served through the ChainIndex from the mock vectors: verbosity 0 +/// round-trips to the stored block, and verbosity 1 matches the source and reports the +/// block's hash / height / txids. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let expected_block = mockchain.get_block(id).await.unwrap().unwrap(); + + // Verbosity 0: the raw serialized block round-trips to the stored block. + let GetBlock::Raw(serialized) = index_reader + .z_get_block(height.to_string(), Some(0)) + .await + .unwrap() + else { + panic!("expected a raw block at verbosity 0"); + }; + let decoded: zebra_chain::block::Block = + serialized.as_ref().zcash_deserialize_into().unwrap(); + assert_eq!(decoded, *expected_block); + + // Verbosity 1: the ChainIndex delegates to the source and the object reports the + // block's hash, height, and every txid. + let via_index = index_reader + .z_get_block(height.to_string(), Some(1)) + .await + .unwrap(); + let via_source = mockchain.get_block_verbose(id, Some(1)).await.unwrap(); + let value = serde_json::to_value(&via_index).unwrap(); + assert_eq!(value, serde_json::to_value(&via_source).unwrap()); + assert_eq!( + value["hash"].as_str().unwrap(), + expected_block.hash().to_string() + ); + assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); + assert_eq!( + value["tx"].as_array().unwrap().len(), + expected_block.transactions.len() + ); + } +} + +/// `get_block_header` served through the ChainIndex from the mock vectors: the compact +/// (non-verbose) form round-trips to the stored header bytes, and the verbose form +/// matches the source and reports the right hash / height. +#[tokio::test(flavor = "multi_thread")] +async fn get_block_header() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let block = mockchain.get_block(id).await.unwrap().unwrap(); + let hash = block.hash().to_string(); + + // Non-verbose: the compact hex decodes to the block header's serialization. + let GetBlockHeader::Compact(compact) = index_reader + .get_block_header(hash.clone(), false) + .await + .unwrap() + else { + panic!("expected a compact header when verbose = false"); + }; + assert_eq!( + hex::decode(compact).unwrap(), + block.header.zcash_serialize_to_vec().unwrap() + ); + + // Verbose: the ChainIndex delegates to the source and reports hash / height. + let via_index = index_reader + .get_block_header(hash.clone(), true) + .await + .unwrap(); + let via_source = mockchain + .get_block_header(hash.clone(), true) + .await + .unwrap(); + let value = serde_json::to_value(&via_index).unwrap(); + assert_eq!(value, serde_json::to_value(&via_source).unwrap()); + assert_eq!(value["hash"].as_str().unwrap(), hash); + assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); + } +} + +/// `get_block_deltas` served through the ChainIndex from the mock vectors: it matches the +/// source, reports the right block hash / height, and surfaces transparent deltas. +#[tokio::test(flavor = "multi_thread")] +async fn get_block_deltas() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + let mut saw_delta_entries = false; + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let block = mockchain.get_block(id).await.unwrap().unwrap(); + let hash = block.hash().to_string(); + + let via_index = index_reader.get_block_deltas(hash.clone()).await.unwrap(); + let via_source = mockchain.get_block_deltas(hash.clone()).await.unwrap(); + assert_eq!( + serde_json::to_value(&via_index).unwrap(), + serde_json::to_value(&via_source).unwrap() + ); + assert_eq!(via_index.hash, hash); + assert_eq!(via_index.height, height); + if via_index + .deltas + .iter() + .any(|delta| !delta.inputs.is_empty() || !delta.outputs.is_empty()) + { + saw_delta_entries = true; + } + } + assert!( + saw_delta_entries, + "expected transparent deltas in at least one sampled block" + ); +} + +/// `get_difficulty` served through the ChainIndex from the mock vectors matches the +/// source and is a positive difficulty value. +#[tokio::test(flavor = "multi_thread")] +async fn get_difficulty() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let via_index = index_reader.get_difficulty().await.unwrap(); + let via_source = mockchain.get_difficulty().await.unwrap(); + assert_eq!(via_index, via_source); + assert!( + via_index > 0.0, + "difficulty should be positive, got {via_index}" + ); +} + +/// Drives the merged [`NodeBackedIndexerServiceSubscriber`] RPC layer over a +/// `MockchainSource`, confirming the service delegates to its chain index: the +/// service's `get_latest_block` reports the same tip the mockchain was synced to. +#[tokio::test(flavor = "multi_thread")] +async fn node_backed_indexer_service_serves_latest_block() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerServiceSubscriber; + use crate::LightWalletIndexer as _; + use zaino_common::network::ActivationHeights; + + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let expected_tip = (blocks.len() as u32) - 1; + wait_for_indexer_tip(&index_reader, expected_tip).await; + + let service = NodeBackedIndexerServiceSubscriber::new_for_test( + index_reader, + ActivationHeights::default().to_regtest_network(), + ); + + let latest = service.get_latest_block().await.unwrap(); + assert_eq!(latest.height, expected_tip as u64); +} + +/// Dropping the chain index without an explicit `shutdown()` call must still +/// release source-owned background work: `Drop` previously only cancelled the +/// sync worker, and the async `shutdown()`'s `?` on the DB teardown skipped +/// the source release when the DB shutdown errored — either way the Direct +/// connection's Zebra syncer task could outlive its index. +#[tokio::test(flavor = "multi_thread")] +async fn dropping_the_chain_index_releases_the_source() { + let (_blocks, indexer, _index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + assert!( + !mockchain.shutdown_called(), + "the source must not be shut down while the index is live" + ); + drop(indexer); + assert!( + mockchain.shutdown_called(), + "dropping the index must release source-owned background work" + ); +} + +/// zebra's `getblock` resolves negative heights against the tip (`-1` is the +/// tip block). The old Rpc backend forwarded the raw identifier string to the +/// validator, so `getblock "-1"` worked; the merged pre-parse used +/// `HashOrHeight::from_str`, which rejects negative heights. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block_resolves_negative_heights_against_the_tip() { + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let tip_height = (blocks.len() as u32) - 1; + wait_for_indexer_tip(&index_reader, tip_height).await; + + let by_negative_height = index_reader + .z_get_block("-1".to_string(), Some(1)) + .await + .expect("height -1 must resolve to the tip block"); + let by_tip_height = index_reader + .z_get_block(tip_height.to_string(), Some(1)) + .await + .expect("the tip height resolves"); + assert_eq!(by_negative_height, by_tip_height); +} + +/// An unparsable `getblock` identifier must carry zcashd's legacy +/// InvalidParameter code (-8) as a typed `RpcError` in the `source()` chain, +/// not be flattened into an internal-error string: the serve layer recovers +/// legacy codes by downcast-walking the chain. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block_invalid_identifier_keeps_legacy_error_code() { + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + wait_for_indexer_tip(&index_reader, (blocks.len() as u32) - 1).await; + + let error = index_reader + .z_get_block("notablockid".to_string(), Some(1)) + .await + .expect_err("an unparsable identifier must be rejected"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut rpc_error_code = None; + while let Some(source_error) = current { + if let Some(rpc_error) = + source_error.downcast_ref::() + { + rpc_error_code = Some(rpc_error.code); + break; + } + current = source_error.source(); + } + assert_eq!( + rpc_error_code, + Some(zebra_rpc::server::error::LegacyCode::InvalidParameter as i64), + "the typed RpcError (legacy code -8) must stay reachable via the source() chain" + ); +} + +/// During the initial finalised-state build there is no non-finalised +/// snapshot. Both pre-merge backends answered `getchaintips` in that window by +/// proxying the validator's own response; the merged service must fall back to +/// the source the same way, rather than serving UnavailableNotSyncedEnough for +/// the whole build (hours on mainnet). +#[tokio::test] +async fn get_chain_tips_falls_back_to_source_while_syncing() { + use crate::chain_index::non_finalised_state::ChainIndexSnapshot; + use crate::chain_index::tests::vectors::build_mockchain_source; + use crate::indexer::node_backed_indexer::chain_tips_for_snapshot; + + let blocks = load_test_vectors().unwrap().blocks; + let tip_height = (blocks.len() as u32) - 1; + let expected_tip_hash = blocks[tip_height as usize].zebra_block.hash().to_string(); + let mock = build_mockchain_source(blocks); + + let syncing_snapshot = ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height: crate::Height(tip_height), + }; + + let tips = chain_tips_for_snapshot(&syncing_snapshot, &mock) + .await + .expect("the syncing window must proxy the source's chain tips"); + + assert_eq!( + tips, + vec![zaino_fetch::jsonrpsee::response::chain_tips::ChainTip::new( + tip_height, + expected_tip_hash, + 0, + zaino_fetch::jsonrpsee::response::chain_tips::ChainTipStatus::Active, + )] + ); +} + +/// Dropping the service must not panic on a thread with no Tokio runtime. +/// `Drop` runs the blocking teardown, which previously called +/// `Handle::current()` unconditionally; outside a runtime that panics, and a +/// panic inside `Drop` during unwind aborts the process. +/// +/// multi_thread required: the harness's finalised-state validation uses +/// `block_in_place`. The drop under test happens on a separate plain thread. +#[tokio::test(flavor = "multi_thread")] +async fn service_drop_survives_thread_without_runtime() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerService; + use zaino_common::network::ActivationHeights; + + let (_blocks, indexer, _index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerService::new_for_test( + indexer, + ActivationHeights::default().to_regtest_network(), + ); + + std::thread::spawn(move || drop(service)) + .join() + .expect("dropping the service off-runtime must not panic"); +} + +/// Dropping the service must not panic on a current-thread Tokio runtime, +/// where `block_in_place` (the old unconditional teardown entry) aborts. +/// +/// multi_thread required: the harness's finalised-state validation uses +/// `block_in_place`. The drop under test happens inside a current-thread +/// runtime built on a separate thread. +#[tokio::test(flavor = "multi_thread")] +async fn service_drop_survives_current_thread_runtime() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerService; + use zaino_common::network::ActivationHeights; + + let (_blocks, indexer, _index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerService::new_for_test( + indexer, + ActivationHeights::default().to_regtest_network(), + ); + + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime builds") + .block_on(async move { drop(service) }); + }) + .join() + .expect("dropping the service on a current-thread runtime must not panic"); +} + +/// The `Rpc` connection has no local chain-tip-change stream, so requesting a +/// chain-tip subscriber over such a source must yield `None` rather than +/// panic. Before this method returned `Option`, it existed only in a +/// panicking form (`.expect("chaintip_update_subscriber requires the Direct +/// connection")`) reachable by any embedder configured with `backend = "rpc"`; +/// pre-merge the misuse was a compile error because only the State-backed +/// subscriber type had the method. +#[tokio::test(flavor = "multi_thread")] +async fn chaintip_update_subscriber_absent_without_tip_stream() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerServiceSubscriber; + use zaino_common::network::ActivationHeights; + + let (_blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerServiceSubscriber::new_for_test( + index_reader, + ActivationHeights::default().to_regtest_network(), + ); + + assert!( + service.chaintip_update_subscriber().is_none(), + "a source with no local tip-change stream must yield no subscriber, not panic" + ); +} + +/// `sendrawtransaction` rejections must carry zcashd's legacy error code: +/// zaino-serve forwards the code by downcast-walking the `source()` chain for +/// the typed `RpcError` (`sendrawtransaction_error_object_from_indexer_error`), +/// so stringifying it downgrades the legacy `-8` "invalid hex" rejection to a +/// generic `-32603` internal error. +/// +/// Invalid hex fails in local validation before the source is consulted, so +/// this also pins that the rejection happens without a validator round-trip +/// (the mock's `send_raw_transaction` would panic if reached). +#[tokio::test(flavor = "multi_thread")] +async fn send_raw_transaction_invalid_hex_keeps_legacy_error_code() { + let (_blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let error = index_reader + .send_raw_transaction("notahexstring".to_string()) + .await + .expect_err("invalid hex must be rejected"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut rpc_error_code = None; + while let Some(source_error) = current { + if let Some(rpc_error) = + source_error.downcast_ref::() + { + rpc_error_code = Some(rpc_error.code); + break; + } + current = source_error.source(); + } + assert_eq!( + rpc_error_code, + Some(zebra_rpc::server::error::LegacyCode::InvalidParameter as i64), + "the typed RpcError (legacy code -8) must stay reachable via the source() chain" + ); +} diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 8dde1281c..22c0da968 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -7,7 +7,7 @@ use proptest::{ }; use rand::seq::IndexedRandom; use tokio_stream::StreamExt as _; -use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; +use zaino_common::{network::ActivationHeights, DatabaseConfig, StorageConfig}; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; @@ -31,12 +31,20 @@ use crate::{ source::{BlockchainSourceResult, GetTransactionLocation}, tests::{init_tracing, poll::poll_until, proptest_blockgen::proptest_helpers::add_segment}, types::BestChainLocation, - NonFinalizedSnapshot, NON_FINALIZED_DEPTH, + NonFinalizedSnapshot, OPERATIONAL_NFS_DEPTH, }, - BlockHash, BlockchainSource, ChainIndex, ChainIndexConfig, NodeBackedChainIndex, - NodeBackedChainIndexSubscriber, TransactionHash, + BlockHash, BlockchainSource, ChainIndex, ChainIndexConfig, ChainIndexRpcExt, Height, + NodeBackedChainIndex, NodeBackedChainIndexSubscriber, TransactionHash, }; +use zaino_proto::proto::utils::PoolTypeFilter; + +/// Chain length per generated segment in the passthrough harness — long enough to +/// have some finalised blocks to play with. The best chain is twice this (genesis +/// segment plus one branch), so its expected tip height is +/// `2 * PASSTHROUGH_SEGMENT_LENGTH - 1`. +const PASSTHROUGH_SEGMENT_LENGTH: usize = OPERATIONAL_NFS_DEPTH as usize + 20; + /// Handle all the boilerplate for a passthrough fn passthrough_test( // The actual assertions. Takes as args: @@ -48,27 +56,54 @@ fn passthrough_test( // A snapshot, which will have only the genesis block &ChainIndexSnapshot, ), +) { + passthrough_test_on( + ActivationHeights::default().to_regtest_network(), + // Slow the source enough to hold the indexer in passthrough while the + // assertions run, without slowing passthrough more than necessary. + Some(Duration::from_millis(100)), + |_| {}, + test, + ) +} + +/// [`passthrough_test`] on an explicit network, with a per-segment chain mutator. +/// +/// The mutator exists because zebra's stock `Transaction` strategy never generates V6 +/// transactions (its NU6.3/NU7 arm produces only v4/v5), so ironwood-era content must +/// be injected after generation. Mutating a block's transactions is safe here: the +/// block hash covers only the header, so parent-hash continuity is untouched, and the +/// header's merkle root is already arbitrary — the passthrough path tolerates that by +/// construction. +fn passthrough_test_on( + network: zebra_chain::parameters::Network, + source_delay: Option, + mutate_segment: impl Fn(&mut Vec>), + test: impl AsyncFn( + &ProptestMockchain, + NodeBackedChainIndexSubscriber, + &ChainIndexSnapshot, + ), ) { init_tracing(); - let network = Network::Regtest(ActivationHeights::default()); - // Long enough to have some finalized blocks to play with - let segment_length = NON_FINALIZED_DEPTH as usize + 20; + let segment_length = PASSTHROUGH_SEGMENT_LENGTH; // No need to worry about non-best chains for this test let branch_count = 1; // from this line to `runtime.block_on(async {` are all // copy-pasted. Could a macro get rid of some of this boilerplate? - proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network))| { + proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network.clone()))| { let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_time().build().unwrap(); runtime.block_on(async { - let (genesis_segment, branching_segments) = segments; + let (mut genesis_segment, mut branching_segments) = segments; + mutate_segment(&mut genesis_segment.0); + for segment in &mut branching_segments { + mutate_segment(&mut segment.0); + } let mockchain = ProptestMockchain { genesis_segment, branching_segments, - // This number can be played with. We want to slow down - // sync enough to trigger passthrough without - // slowing down passthrough more than we need to - delay: Some(Duration::from_millis(100)), + delay: source_delay, best_branch_cache: Arc::new(std::sync::OnceLock::new()), tx_index: Arc::new(std::sync::OnceLock::new()), }; @@ -85,7 +120,7 @@ fn passthrough_test( }, ephemeral: true, db_version: 1, - network, + network: network.clone(), }; @@ -364,6 +399,386 @@ fn passthrough_get_block_range() { }) } +/// Upstream gap demonstration: zebra-chain's stock [`Transaction`] strategy never +/// generates V6 transactions, even for an NU6.3 ledger state — its NU6.3/NU7 arm is +/// `prop_oneof![v4_strategy, v5_strategy]` (zebra-chain `transaction/arbitrary.rs`). +/// V6 is therefore structurally impossible from the stock strategy, not merely rare, +/// which is why the `passthrough_metadata_consistency_*` walks must inject +/// `fake_v6_transaction` ironwood content instead of relying on generation. +/// +/// `should_panic` tracks the upstream gap: when a zebra upgrade starts generating V6, +/// this test flips, and the `#[should_panic]` should be removed together with the +/// fake-transaction injection in `inject_ironwood_transactions` (generation then covers +/// it natively). +/// +/// [`Transaction`]: zebra_chain::transaction::Transaction +#[test] +#[should_panic(expected = "zebra's stock Transaction strategy generated no V6")] +fn zebra_arbitrary_generates_v6_transactions_for_nu6_3() { + use proptest::strategy::ValueTree as _; + use proptest::test_runner::TestRunner; + use zebra_chain::parameters::NetworkUpgrade; + + let mut runner = TestRunner::default(); + + let ledger = LedgerState::arbitrary_with(LedgerStateOverride { + network_upgrade_override: Some(NetworkUpgrade::Nu6_3), + ..LedgerStateOverride::default() + }) + .new_tree(&mut runner) + .expect("ledger strategy yields a value") + .current(); + assert_eq!(ledger.network_upgrade(), NetworkUpgrade::Nu6_3); + + let transaction_strategy = + zebra_chain::transaction::Transaction::arbitrary_with(ledger.clone()); + + let mut generated_versions = std::collections::BTreeSet::new(); + for _ in 0..64 { + let transaction = transaction_strategy + .new_tree(&mut runner) + .expect("transaction strategy yields a value") + .current(); + generated_versions.insert(transaction.version()); + } + + assert!( + generated_versions.contains(&6), + "zebra's stock Transaction strategy generated no V6 transaction for an NU6.3 \ + ledger state in 64 samples (saw versions {generated_versions:?})" + ); +} + +/// NU6.3 active from height 2, so post-activation generated blocks carry V6 +/// transactions whose shielded data lands in the Ironwood pool. +const IRONWOOD_ONLY_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(2), + nu7: None, +}; + +/// Per-block consistency between served compact-block content and its chain metadata. +/// +/// A compact block's `chainMetadata` tree sizes are cumulative note-commitment counts; +/// a scanning wallet advances its trees by the actions/outputs each served block +/// carries, so a served block whose tree-size delta disagrees with its served +/// commitment count reads as a tree-size discontinuity — a phantom chain reorg. The +/// walk checks every serviceable height, for both the wire-decoded unfiltered request +/// (empty `poolTypes`, what real — including pre-Ironwood — clients send) and the +/// explicit all-pools filter, and cross-checks served counts against the mockchain +/// source of truth. +#[test] +fn passthrough_metadata_consistency_ironwood_only() { + metadata_consistency_for_era(IRONWOOD_ONLY_HEIGHTS, Some(2), false) +} + +/// Orchard-only heights: every upgrade through NU6.2 at height 2, NU6.3 never +/// activating. +const ORCHARD_ONLY_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: None, + nu7: None, +}; + +/// Orchard-only era (NU6.3 never activates): fake Orchard content from height 2, +/// and — since zebra's stock strategy cannot generate V6 — ironwood provably never +/// appears anywhere in the chain or the served form. +#[test] +fn passthrough_metadata_consistency_orchard_only() { + metadata_consistency_for_era(ORCHARD_ONLY_HEIGHTS, None, false) +} + +/// The transition: fake Orchard content below the NU6.3 boundary, fake Ironwood +/// content from it. The boundary is placed inside the walked non-finalised window so +/// both eras are actually observed by the walk. +#[test] +fn passthrough_metadata_consistency_orchard_to_ironwood_transition() { + let expected_tip = (2 * PASSTHROUGH_SEGMENT_LENGTH - 1) as u32; + let boundary = expected_tip - (OPERATIONAL_NFS_DEPTH / 2); + metadata_consistency_for_era( + ActivationHeights { + nu6_3: Some(boundary), + ..IRONWOOD_ONLY_HEIGHTS + }, + Some(boundary), + true, + ) +} + +/// A structurally-valid (cryptographically fake) V6 transaction carrying a two-action +/// Ironwood bundle. Injected because zebra's stock strategy never generates V6 +/// (demonstrated by [`zebra_arbitrary_generates_v6_transactions_for_nu6_3`]). +fn fake_ironwood_transaction() -> zebra_chain::transaction::Transaction { + use zebra_chain::amount::Amount; + use zebra_chain::orchard::{Flags, ShieldedDataV6}; + use zebra_chain::parameters::NetworkUpgrade; + use zebra_chain::transaction::arbitrary::{fake_v6_orchard_shielded_data, fake_v6_transaction}; + + let ironwood = zebra_chain::ironwood::ShieldedData::new(ShieldedDataV6::new( + fake_v6_orchard_shielded_data( + Flags::ENABLE_SPENDS, + Amount::try_from(0).expect("zero is a valid amount"), + 2, + ), + )); + fake_v6_transaction(NetworkUpgrade::Nu6_3, None, Some(ironwood)) +} + +/// A structurally-valid (cryptographically fake) V5 transaction carrying a two-action +/// Orchard bundle, for deterministic orchard-era content (the stock strategy's orchard +/// data is probabilistic). +fn fake_orchard_transaction() -> zebra_chain::transaction::Transaction { + use zebra_chain::amount::Amount; + use zebra_chain::orchard::Flags; + use zebra_chain::parameters::NetworkUpgrade; + use zebra_chain::transaction::arbitrary::fake_v6_orchard_shielded_data; + use zebra_chain::transaction::{LockTime, Transaction}; + + Transaction::V5 { + network_upgrade: NetworkUpgrade::Nu5, + lock_time: LockTime::unlocked(), + expiry_height: zebra_chain::block::Height(0), + inputs: Vec::new(), + outputs: Vec::new(), + sapling_shielded_data: None, + orchard_shielded_data: Some(fake_v6_orchard_shielded_data( + Flags::ENABLE_SPENDS, + Amount::try_from(0).expect("zero is a valid amount"), + 2, + )), + } +} + +/// Runs the metadata-consistency walk on a chain whose injected shielded content +/// follows the era layout: +/// +/// - `ironwood_boundary: None` — orchard era only: fake Orchard content from height 2, +/// and ironwood must never appear anywhere; +/// - `ironwood_boundary: Some(b)` — fake Ironwood content from height `b`; when +/// `orchard_below_boundary` is set, fake Orchard content fills heights 2..b (the +/// transition layout), otherwise heights below `b` carry only generated content. +fn metadata_consistency_for_era( + heights: ActivationHeights, + ironwood_boundary: Option, + orchard_below_boundary: bool, +) { + let inject = move |blocks: &mut Vec>| { + for block in blocks.iter_mut() { + let height = block + .coinbase_height() + .expect("generated blocks always have a coinbase height") + .0; + if height < 2 { + continue; + } + let fake_tx = match ironwood_boundary { + None => fake_orchard_transaction(), + Some(boundary) if height >= boundary => fake_ironwood_transaction(), + Some(_) if orchard_below_boundary => fake_orchard_transaction(), + Some(_) => continue, + }; + let mut new_block = (**block).clone(); + new_block.transactions.push(Arc::new(fake_tx)); + *block = Arc::new(new_block); + } + }; + + passthrough_test_on( + heights.to_regtest_network(), + // No artificial source delay: this test waits for the indexer to finish + // syncing, because compact blocks are not served while the finalised state + // is still syncing (get_compact_block's StillSyncingFinalizedState arm). + None, + inject, + async |mockchain, index_reader, _snapshot| { + // Source of truth: per-height shielded commitment counts from the mockchain + // blocks themselves (single branch, so arb branch order is chain order). + let source_counts: Vec<(u32, u32, u32)> = mockchain + .all_blocks_arb_branch_order() + .map(|block| { + let sapling = block + .transactions + .iter() + .map(|tx| tx.sapling_note_commitments().count() as u32) + .sum(); + let orchard = block + .transactions + .iter() + .map(|tx| tx.orchard_note_commitments().count() as u32) + .sum(); + let ironwood = block + .transactions + .iter() + .map(|tx| tx.ironwood_note_commitments().count() as u32) + .sum(); + (sapling, orchard, ironwood) + }) + .collect(); + + // Era-composition guards on the source chain, so no assertion below can go + // vacuously green (and no era leaks content into the other). + match ironwood_boundary { + None => { + let ironwood_total: u32 = source_counts.iter().map(|(_, _, i)| i).sum(); + assert_eq!( + ironwood_total, 0, + "orchard-only era must carry no ironwood commitments" + ); + let orchard_total: u32 = source_counts.iter().map(|(_, o, _)| o).sum(); + assert!( + orchard_total > 0, + "orchard-only era carries no orchard commitments; the orchard \ + assertions below would be vacuous" + ); + } + Some(boundary) => { + let below: u32 = source_counts[..boundary as usize] + .iter() + .map(|(_, _, i)| i) + .sum(); + assert_eq!( + below, 0, + "no ironwood commitments may exist below the activation boundary" + ); + let above: u32 = source_counts[boundary as usize..] + .iter() + .map(|(_, _, i)| i) + .sum(); + assert!( + above > 0, + "no ironwood commitments above the boundary; the ironwood \ + assertions below would be vacuous" + ); + if orchard_below_boundary { + let orchard_below: u32 = source_counts[2..boundary as usize] + .iter() + .map(|(_, o, _)| o) + .sum(); + assert!( + orchard_below > 0, + "transition layout carries no orchard commitments below the \ + boundary; the orchard-era half would be vacuous" + ); + } + } + } + + // Compact blocks are only served once the finalised state has caught up. + let snapshot = poll_until( + "indexer to finish syncing so compact blocks are served", + Duration::from_secs(60), + Duration::from_millis(50), + || async { + let snapshot = index_reader.snapshot_nonfinalized_state().await.ok()?; + matches!(snapshot, ChainIndexSnapshot::NonFinalizedStateExists { .. }) + .then_some(snapshot) + }, + ) + .await; + let snapshot = &snapshot; + + let tip = snapshot + .get_nfs_snapshot() + .expect("fully synced snapshot has a non-finalised state") + .best_tip + .height; + // The walk covers the non-finalised window; its absolute baseline is the + // cumulative source count below the window. + let first_walked = finalized_height_floor(tip.0).0 + 1; + let baseline = source_counts[..first_walked as usize].iter().fold( + (0u32, 0u32, 0u32), + |(sapling, orchard, ironwood), (s, o, i)| (sapling + s, orchard + o, ironwood + i), + ); + + for unfiltered_wire_request in [true, false] { + let (mut prev_sapling, mut prev_orchard, mut prev_ironwood) = baseline; + + for height_int in first_walked..=tip.0 { + // The empty slice is the wire shape unfiltered clients send; both + // filters include every shielded pool, which the delta assertions + // below rely on. + let filter = if unfiltered_wire_request { + PoolTypeFilter::new_from_slice(&[]).unwrap() + } else { + PoolTypeFilter::includes_all() + }; + let block = index_reader + .get_compact_block(snapshot, Height(height_int), filter) + .await + .unwrap() + .expect("serviceable heights must serve a compact block"); + let metadata = block + .chain_metadata + .as_ref() + .expect("served compact blocks carry chain metadata"); + + let served_sapling: u32 = + block.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(); + let served_orchard: u32 = + block.vtx.iter().map(|tx| tx.actions.len() as u32).sum(); + let served_ironwood: u32 = block + .vtx + .iter() + .map(|tx| tx.ironwood_actions.len() as u32) + .sum(); + + // Serving completeness: everything the source block carries is served. + let (source_sapling, source_orchard, source_ironwood) = + source_counts[height_int as usize]; + assert_eq!( + (served_sapling, served_orchard, served_ironwood), + (source_sapling, source_orchard, source_ironwood), + "served shielded counts must match the source block at height \ + {height_int} (unfiltered_wire_request: {unfiltered_wire_request})" + ); + + // Metadata consistency: tree-size deltas equal served counts. + assert_eq!( + metadata.sapling_commitment_tree_size, + prev_sapling + served_sapling, + "sapling tree-size delta must equal the served output count at \ + height {height_int}" + ); + assert_eq!( + metadata.orchard_commitment_tree_size, + prev_orchard + served_orchard, + "orchard tree-size delta must equal the served action count at \ + height {height_int}" + ); + assert_eq!( + metadata.ironwood_commitment_tree_size, + prev_ironwood + served_ironwood, + "ironwood tree-size delta must equal the served action count at \ + height {height_int}" + ); + + prev_sapling = metadata.sapling_commitment_tree_size; + prev_orchard = metadata.orchard_commitment_tree_size; + prev_ironwood = metadata.ironwood_commitment_tree_size; + } + } + }, + ) +} + // Ignored: this drives the full indexer over `partial_chain_strategy` blocks, whose headers carry // arbitrary (invalid) merkle roots. The finalised state now validates blocks on the write path // (cheap merkle + parent-continuity checks), so it correctly rejects these blocks once the indexer's @@ -376,14 +791,14 @@ fn passthrough_get_block_range() { #[test] fn make_chain() { init_tracing(); - let network = Network::Regtest(ActivationHeights::default()); + let network = ActivationHeights::default().to_regtest_network(); let segment_length = 12; let branch_count = 2; // default is 256. As each case takes multiple seconds, this seems too many. // TODO: this should be higher than 1. Currently set to 1 for ease of iteration - proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network))| { + proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network.clone()))| { let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_time().build().unwrap(); runtime.block_on(async { let (genesis_segment, branching_segments) = segments; @@ -407,7 +822,7 @@ fn make_chain() { }, ephemeral: true, db_version: 1, - network, + network: network.clone(), }; @@ -616,6 +1031,115 @@ impl BlockchainSource for ProptestMockchain { } } + async fn get_block_verbose( + &self, + _hash_or_height: HashOrHeight, + _verbosity: Option, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the verbose getblock RPC. + unimplemented!() + } + + async fn get_block_header( + &self, + _hash: String, + _verbose: bool, + ) -> BlockchainSourceResult + { + // ProptestMockchain exercises sync/reorg, not the getblockheader RPC. + unimplemented!() + } + + async fn get_block_deltas( + &self, + _hash: String, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getblockdeltas RPC. + unimplemented!() + } + + async fn get_difficulty(&self) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getdifficulty RPC. + unimplemented!() + } + + async fn get_blockchain_info( + &self, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getblockchaininfo RPC. + unimplemented!() + } + + async fn get_info(&self) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_peer_info( + &self, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + unimplemented!() + } + + async fn get_block_subsidy( + &self, + _height: u32, + ) -> BlockchainSourceResult + { + unimplemented!() + } + + async fn get_mining_info( + &self, + ) -> BlockchainSourceResult + { + unimplemented!() + } + + async fn get_tx_out( + &self, + _txid: String, + _n: u32, + _include_mempool: Option, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_spent_info( + &self, + _request: zaino_fetch::jsonrpsee::response::GetSpentInfoRequest, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_network_sol_ps( + &self, + _blocks: Option, + _height: Option, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn send_raw_transaction( + &self, + _raw_transaction_hex: String, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_treestate_by_id( + &self, + _hash_or_height: String, + ) -> BlockchainSourceResult { + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, @@ -623,6 +1147,7 @@ impl BlockchainSource for ProptestMockchain { ) -> BlockchainSourceResult<( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, )> { if let Some(delay) = self.delay { tokio::time::sleep(delay).await; @@ -630,44 +1155,57 @@ impl BlockchainSource for ProptestMockchain { let Some(chain_up_to_block) = self.get_block_and_all_preceeding(|block| block.hash().0 == id.0) else { - return Ok((None, None)); + return Ok((None, None, None)); }; - let (sapling, orchard) = - chain_up_to_block - .iter() - .fold((None, None), |(mut sapling, mut orchard), block| { - for transaction in &block.transactions { - for sap_commitment in transaction.sapling_note_commitments() { - let sap_commitment = - sapling_crypto::Node::from_bytes(sap_commitment.to_bytes()) - .unwrap(); - - sapling = Some(sapling.unwrap_or_else(|| { - incrementalmerkletree::frontier::Frontier::<_, 32>::empty() - })); - - sapling = sapling.map(|mut tree| { - tree.append(sap_commitment); - tree - }); - } - for orc_commitment in transaction.orchard_note_commitments() { - let orc_commitment = - zebra_chain::orchard::tree::Node::from(*orc_commitment); - - orchard = Some(orchard.unwrap_or_else(|| { - incrementalmerkletree::frontier::Frontier::<_, 32>::empty() - })); - - orchard = orchard.map(|mut tree| { - tree.append(orc_commitment); - tree - }); - } + let (sapling, orchard, ironwood) = chain_up_to_block.iter().fold( + (None, None, None), + |(mut sapling, mut orchard, mut ironwood), block| { + for transaction in &block.transactions { + for sap_commitment in transaction.sapling_note_commitments() { + let sap_commitment = + sapling_crypto::Node::from_bytes(sap_commitment.to_bytes()).unwrap(); + + sapling = Some(sapling.unwrap_or_else(|| { + incrementalmerkletree::frontier::Frontier::<_, 32>::empty() + })); + + sapling = sapling.map(|mut tree| { + tree.append(sap_commitment); + tree + }); } - (sapling, orchard) - }); + for orc_commitment in transaction.orchard_note_commitments() { + let orc_commitment = + zebra_chain::orchard::tree::Node::from(*orc_commitment); + + orchard = Some(orchard.unwrap_or_else(|| { + incrementalmerkletree::frontier::Frontier::<_, 32>::empty() + })); + + orchard = orchard.map(|mut tree| { + tree.append(orc_commitment); + tree + }); + } + // Ironwood reuses the Orchard tree/node types. + for irw_commitment in transaction.ironwood_note_commitments() { + let irw_commitment = + zebra_chain::orchard::tree::Node::from(*irw_commitment); + + ironwood = Some(ironwood.unwrap_or_else(|| { + incrementalmerkletree::frontier::Frontier::<_, 32>::empty() + })); + + ironwood = ironwood.map(|mut tree| { + tree.append(irw_commitment); + tree + }); + } + } + (sapling, orchard, ironwood) + }, + ); Ok(( sapling.map(|sap_front| { ( @@ -681,6 +1219,12 @@ impl BlockchainSource for ProptestMockchain { orc_front.tree_size(), ) }), + ironwood.map(|irw_front| { + ( + zebra_chain::orchard::tree::Root::from_bytes(irw_front.root().as_bytes()), + irw_front.tree_size(), + ) + }), )) } @@ -688,7 +1232,7 @@ impl BlockchainSource for ProptestMockchain { async fn get_treestate( &self, _id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>)> { + ) -> BlockchainSourceResult { // I don't think this is used for sync? unimplemented!() } @@ -822,9 +1366,9 @@ fn make_branching_chain( // The length of the initial segment, and of the branches // TODO: it would be useful to allow branches of different lengths. chain_size: usize, - network_override: Network, + network_override: zebra_chain::parameters::Network, ) -> BoxedStrategy<(ChainSegment, Vec)> { - let network_override = Some(network_override.to_zebra_network()); + let network_override = Some(network_override); add_segment( SummaryDebug(Vec::new()), network_override.clone(), diff --git a/packages/zaino-state/src/chain_index/tests/vectors.rs b/packages/zaino-state/src/chain_index/tests/vectors.rs index b0b5115fc..0e8a3ceae 100644 --- a/packages/zaino-state/src/chain_index/tests/vectors.rs +++ b/packages/zaino-state/src/chain_index/tests/vectors.rs @@ -97,6 +97,7 @@ pub(crate) fn indexed_block_chain( vector.sapling_tree_size as u32, vector.orchard_root, vector.orchard_tree_size as u32, + None, parent_chain_work, zebra_chain::parameters::Network::new_regtest( zebra_chain::parameters::testnet::ConfiguredActivationHeights { @@ -111,6 +112,7 @@ pub(crate) fn indexed_block_chain( // see https://zips.z.cash/#nu6-1-candidate-zips for info on NU6.1 nu6_1: None, nu6_2: None, + nu6_3: None, nu7: None, } .into(), diff --git a/packages/zaino-state/src/chain_index/types/db/block.rs b/packages/zaino-state/src/chain_index/types/db/block.rs index 186abcebc..cb55b4ba1 100644 --- a/packages/zaino-state/src/chain_index/types/db/block.rs +++ b/packages/zaino-state/src/chain_index/types/db/block.rs @@ -86,8 +86,16 @@ impl ZainoVersionedSerde for PersistentChainWork { } } +/// Fixed-length encoding metadata for `PersistentChainWork`. +/// +/// v1 consists of a single 32-byte value. impl FixedEncodedLen for PersistentChainWork { - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Database-adjacent persistence shape for [`CompactDifficulty`]. @@ -129,8 +137,16 @@ impl ZainoVersionedSerde for PersistentCompactDifficulty { } } +/// Fixed-length encoding metadata for `PersistentCompactDifficulty`. +/// +/// v1 consists of a single 4-byte LE u32. impl FixedEncodedLen for PersistentCompactDifficulty { - const ENCODED_LEN: usize = 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + _ => None, + } + } } /// Database-adjacent persistence shape for [`BlockContext`]. @@ -305,7 +321,7 @@ mod tests { impl ChainWork { /// Returns ChainWork as a U256. - pub(super) fn to_u256(&self) -> primitive_types::U256 { + pub(super) fn to_u256(self) -> primitive_types::U256 { primitive_types::U256::from_big_endian(&self.0) } diff --git a/packages/zaino-state/src/chain_index/types/db/commitment.rs b/packages/zaino-state/src/chain_index/types/db/commitment.rs index 53fbeca30..b552f6b5a 100644 --- a/packages/zaino-state/src/chain_index/types/db/commitment.rs +++ b/packages/zaino-state/src/chain_index/types/db/commitment.rs @@ -8,9 +8,12 @@ use corez::io::{self, Read, Write}; -use crate::chain_index::encoding::{ - read_fixed_le, read_u32_le, version, write_fixed_le, write_u32_le, FixedEncodedLen, - ZainoVersionedSerde, +use crate::{ + chain_index::encoding::{ + read_fixed_le, read_u32_le, version, write_fixed_le, write_u32_le, FixedEncodedLen, + ZainoVersionedSerde, + }, + read_option, write_option, }; /// Holds commitment tree metadata (roots and sizes) for a block. @@ -39,20 +42,20 @@ impl CommitmentTreeData { } impl ZainoVersionedSerde for CommitmentTreeData { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, w: &mut W) -> io::Result<()> { let mut w = w; - self.roots.serialize(&mut w)?; // carries its own tag - self.sizes.serialize(&mut w) + self.roots.serialize_with_version(&mut w, 1)?; + self.sizes.serialize_with_version(&mut w, 1) } fn decode_v1(r: &mut R) -> io::Result { @@ -61,14 +64,41 @@ impl ZainoVersionedSerde for CommitmentTreeData { let sizes = CommitmentTreeSizes::deserialize(&mut r)?; Ok(CommitmentTreeData::new(roots, sizes)) } + + fn encode_v2(&self, w: &mut W) -> io::Result<()> { + let mut w = w; + self.roots.serialize_with_version(&mut w, 2)?; + self.sizes.serialize_with_version(&mut w, 2) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let roots = CommitmentTreeRoots::deserialize(&mut r)?; + let sizes = CommitmentTreeSizes::deserialize(&mut r)?; + Ok(CommitmentTreeData::new(roots, sizes)) + } } -/// CommitmentTreeData: 74 bytes total impl FixedEncodedLen for CommitmentTreeData { - // 1 byte tag + 64 body for roots - // + 1 byte tag + 8 body for sizes - const ENCODED_LEN: usize = - (CommitmentTreeRoots::ENCODED_LEN + 1) + (CommitmentTreeSizes::ENCODED_LEN + 1); + /// Returns the fixed encoded body length for a specific + /// `CommitmentTreeData` version. + /// + /// v1 is fixed-length: + /// - versioned `CommitmentTreeRoots` v1 + /// - versioned `CommitmentTreeSizes` v1 + /// + /// v2 is variable-length because `CommitmentTreeRoots` v2 contains an + /// `Option<[u8; 32]>`. + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some( + CommitmentTreeRoots::versioned_len(version::V1)? + + CommitmentTreeSizes::versioned_len(version::V1)?, + ), + version::V2 => None, + _ => None, + } + } } /// Commitment tree roots for shielded transactions, enabling shielded wallet synchronization. @@ -79,12 +109,18 @@ pub struct CommitmentTreeRoots { sapling: [u8; 32], /// Orchard note-commitment tree root at this block. orchard: [u8; 32], + /// Ironwood note-commitment tree root at this block. + ironwood: Option<[u8; 32]>, } impl CommitmentTreeRoots { /// Reutns a new CommitmentTreeRoots instance. - pub fn new(sapling: [u8; 32], orchard: [u8; 32]) -> Self { - Self { sapling, orchard } + pub fn new(sapling: [u8; 32], orchard: [u8; 32], ironwood: Option<[u8; 32]>) -> Self { + Self { + sapling, + orchard, + ironwood, + } } /// Returns sapling commitment tree root. @@ -96,17 +132,25 @@ impl CommitmentTreeRoots { pub fn orchard(&self) -> &[u8; 32] { &self.orchard } + + /// returns orchard commitment tree root. + /// No production reader consumes the stored ironwood root yet; the regression test + /// for its None-preservation does. Un-gate when a production consumer appears. + #[cfg(test)] + pub(crate) fn ironwood(&self) -> &Option<[u8; 32]> { + &self.ironwood + } } impl ZainoVersionedSerde for CommitmentTreeRoots { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, w: &mut W) -> io::Result<()> { @@ -119,14 +163,44 @@ impl ZainoVersionedSerde for CommitmentTreeRoots { let mut r = r; let sapling = read_fixed_le::<32, _>(&mut r)?; let orchard = read_fixed_le::<32, _>(&mut r)?; - Ok(CommitmentTreeRoots::new(sapling, orchard)) + Ok(CommitmentTreeRoots::new(sapling, orchard, None)) + } + + fn encode_v2(&self, w: &mut W) -> io::Result<()> { + let mut w = w; + write_fixed_le::<32, _>(&mut w, &self.sapling)?; + write_fixed_le::<32, _>(&mut w, &self.orchard)?; + write_option(&mut w, &self.ironwood, |w, v| write_fixed_le(w, v)) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let sapling = read_fixed_le::<32, _>(&mut r)?; + let orchard = read_fixed_le::<32, _>(&mut r)?; + let ironwood = read_option(&mut r, |r| read_fixed_le(r))?; + + Ok(CommitmentTreeRoots::new(sapling, orchard, ironwood)) } } -/// CommitmentTreeRoots: 64 bytes total impl FixedEncodedLen for CommitmentTreeRoots { - /// 32 byte hash + 32 byte hash. - const ENCODED_LEN: usize = 32 + 32; + /// Returns the fixed encoded body length for a specific + /// `CommitmentTreeRoots` version. + /// + /// v1 is fixed-length: + /// - 32 bytes Sapling root + /// - 32 bytes Orchard root + /// + /// v2 is variable-length because `ironwood: Option<[u8; 32]>` encodes as: + /// - 1 byte option tag + /// - plus 32 bytes when present + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32 + 32), + version::V2 => None, + _ => None, + } + } } /// Sizes of commitment trees, indicating total number of shielded notes created. @@ -137,12 +211,18 @@ pub struct CommitmentTreeSizes { sapling: u32, /// Total notes in Orchard commitment tree. orchard: u32, + /// Total notes in Ironwood commitment tree. + ironwood: u32, } impl CommitmentTreeSizes { /// Creates a new CompactSaplingSizes instance. - pub fn new(sapling: u32, orchard: u32) -> Self { - Self { sapling, orchard } + pub fn new(sapling: u32, orchard: u32, ironwood: u32) -> Self { + Self { + sapling, + orchard, + ironwood, + } } /// Returns sapling commitment tree size @@ -154,17 +234,22 @@ impl CommitmentTreeSizes { pub fn orchard(&self) -> u32 { self.orchard } + + /// Returns orchard commitment tree size + pub fn ironwood(&self) -> u32 { + self.ironwood + } } impl ZainoVersionedSerde for CommitmentTreeSizes { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, w: &mut W) -> io::Result<()> { @@ -177,12 +262,42 @@ impl ZainoVersionedSerde for CommitmentTreeSizes { let mut r = r; let sapling = read_u32_le(&mut r)?; let orchard = read_u32_le(&mut r)?; - Ok(CommitmentTreeSizes::new(sapling, orchard)) + Ok(CommitmentTreeSizes::new(sapling, orchard, 0)) + } + + fn encode_v2(&self, w: &mut W) -> io::Result<()> { + let mut w = w; + write_u32_le(&mut w, self.sapling)?; + write_u32_le(&mut w, self.orchard)?; + write_u32_le(&mut w, self.ironwood) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let sapling = read_u32_le(&mut r)?; + let orchard = read_u32_le(&mut r)?; + let ironwood = read_u32_le(&mut r)?; + Ok(CommitmentTreeSizes::new(sapling, orchard, ironwood)) } } -/// CommitmentTreeSizes: 8 bytes total impl FixedEncodedLen for CommitmentTreeSizes { - /// 4 byte LE int32 + 4 byte LE int32 - const ENCODED_LEN: usize = 4 + 4; + /// Returns the fixed encoded body length for a specific + /// `CommitmentTreeSizes` version. + /// + /// v1 is fixed-length: + /// - 4 bytes Sapling size + /// - 4 bytes Orchard size + /// + /// v2 is also fixed-length: + /// - 4 bytes Sapling size + /// - 4 bytes Orchard size + /// - 4 bytes Ironwood size + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4 + 4), + version::V2 => Some(4 + 4 + 4), + _ => None, + } + } } diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index ee4074d5e..f6bdb88bd 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -200,10 +200,16 @@ impl ZainoVersionedSerde for BlockHash { } } -/// Hash = 32-byte body. +/// Fixed-length encoding metadata for `BlockHash`. +/// +/// v1 consists of a single 32-byte hash. impl FixedEncodedLen for BlockHash { - /// 32 bytes, LE - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Transaction hash. @@ -338,10 +344,16 @@ impl ZainoVersionedSerde for TransactionHash { } } -/// Hash = 32-byte body. +/// Fixed-length encoding metadata for `TransactionHash`. +/// +/// v1 consists of a single 32-byte hash. impl FixedEncodedLen for TransactionHash { - /// 32 bytes, LE - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Block height. @@ -352,6 +364,16 @@ impl FixedEncodedLen for TransactionHash { #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Height(pub(crate) u32); +impl Height { + /// Iterates every height from `start` through `end` inclusive. + /// + /// Both bounds are already-valid heights, so every intermediate value is a valid + /// height by construction — callers walking a range need no per-step validation. + pub(crate) fn range_inclusive(start: Self, end: Self) -> impl Iterator { + (start.0..=end.0).map(Self) + } +} + impl PartialOrd for Height { fn partial_cmp(&self, other: &zebra_chain::block::Height) -> Option { Some(self.0.cmp(&other.0)) @@ -480,10 +502,16 @@ impl ZainoVersionedSerde for Height { } } -/// Height = 4-byte big-endian body. +/// Fixed-length encoding metadata for `Height`. +/// +/// v1 consists of a single 4-byte big-endian u32. impl FixedEncodedLen for Height { - /// 4 bytes, BE - const ENCODED_LEN: usize = 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + _ => None, + } + } } /// Numerical index of subtree / shard roots. @@ -516,10 +544,16 @@ impl ZainoVersionedSerde for ShardIndex { } } -/// Index = 4-byte big-endian body. +/// Fixed-length encoding metadata for `ShardIndex`. +/// +/// v1 consists of a single 4-byte big-endian u32. impl FixedEncodedLen for ShardIndex { - /// 4 bytes (BE u32) - const ENCODED_LEN: usize = 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + _ => None, + } + } } /// A 20-byte hash160 *plus* a 1-byte ScriptType tag. @@ -645,10 +679,16 @@ impl ZainoVersionedSerde for AddrScript { } } -/// AddrScript = 21 bytes of body data. +/// Fixed-length encoding metadata for `AddrScript`. +/// +/// v1 consists of a 20 byte script (LE) + 1 byte script type impl FixedEncodedLen for AddrScript { - /// 20 bytes, LE + 1 byte script type - const ENCODED_LEN: usize = 21; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(21), + _ => None, + } + } } /// Reference to a spent transparent UTXO. @@ -712,10 +752,16 @@ impl ZainoVersionedSerde for Outpoint { } } -/// Outpoint = 32‐byte txid + 4-byte LE u32 index = 36 bytes +/// Fixed-length encoding metadata for `Outpoint`. +/// +/// v1 consists of a 32 byte txid + 4 byte tx index. impl FixedEncodedLen for Outpoint { - /// 32 byte txid + 4 byte tx index. - const ENCODED_LEN: usize = 32 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(36), + _ => None, + } + } } // *** Block Level Objects *** @@ -1071,6 +1117,7 @@ impl IndexedBlock { let sapling_commitment_tree_size = self.commitment_tree_data().sizes().sapling(); let orchard_commitment_tree_size = self.commitment_tree_data().sizes().orchard(); + let ironwood_commitment_tree_size = self.commitment_tree_data().sizes().ironwood(); zaino_proto::proto::compact_formats::CompactBlock { proto_version: 0, @@ -1083,6 +1130,7 @@ impl IndexedBlock { chain_metadata: Some(zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size, orchard_commitment_tree_size, + ironwood_commitment_tree_size, }), } } @@ -1137,6 +1185,8 @@ impl Option, [u8; 32], [u8; 32], + Option<[u8; 32]>, + u32, u32, u32, )> for IndexedBlock @@ -1149,13 +1199,17 @@ impl parent_chainwork, final_sapling_root, final_orchard_root, + final_ironwood_root, parent_sapling_size, parent_orchard_size, + parent_ironwood_size, ): ( zaino_fetch::chain::block::FullBlock, Option, [u8; 32], [u8; 32], + Option<[u8; 32]>, + u32, u32, u32, ), @@ -1211,6 +1265,7 @@ impl // --- Convert transactions --- let mut sapling_note_count = 0; let mut orchard_note_count = 0; + let mut ironwood_note_count = 0; let full_transactions = full_block.transactions(); let mut tx = Vec::with_capacity(full_transactions.len()); @@ -1221,6 +1276,7 @@ impl sapling_note_count += txdata.sapling().outputs().len(); orchard_note_count += txdata.orchard().actions().len(); + ironwood_note_count += txdata.ironwood().actions().len(); tx.push(txdata); } @@ -1228,12 +1284,14 @@ impl // --- Compute commitment trees --- let sapling_root = final_sapling_root; let orchard_root = final_orchard_root; + let ironwood_root = final_ironwood_root; let commitment_tree_data = CommitmentTreeData::new( - CommitmentTreeRoots::new(sapling_root, orchard_root), + CommitmentTreeRoots::new(sapling_root, orchard_root, ironwood_root), CommitmentTreeSizes::new( parent_sapling_size + sapling_note_count as u32, parent_orchard_size + orchard_note_count as u32, + parent_ironwood_size + ironwood_note_count as u32, ), ); @@ -1287,6 +1345,8 @@ pub struct CompactTxData { sapling: SaplingCompactTx, /// Compact representation of Orchard actions (shielded pool transactions). orchard: OrchardCompactTx, + /// Compact representation of Ironwood actions (shielded pool transactions). + ironwood: OrchardCompactTx, } impl CompactTxData { @@ -1297,6 +1357,7 @@ impl CompactTxData { transparent: TransparentCompactTx, sapling: SaplingCompactTx, orchard: OrchardCompactTx, + ironwood: OrchardCompactTx, ) -> Self { Self { index, @@ -1304,6 +1365,7 @@ impl CompactTxData { transparent, sapling, orchard, + ironwood, } } @@ -1337,6 +1399,11 @@ impl CompactTxData { &self.orchard } + /// Returns compact ironwood tx data. + pub(crate) fn ironwood(&self) -> &OrchardCompactTx { + &self.ironwood + } + /// Converts this `TxData` into a `CompactTx` protobuf message with an optional fee. pub fn to_compact_tx( &self, @@ -1348,38 +1415,28 @@ impl CompactTxData { .sapling() .spends() .iter() - .map( - |s| zaino_proto::proto::compact_formats::CompactSaplingSpend { - nf: s.nullifier().to_vec(), - }, - ) + .map(CompactSaplingSpend::into_compact) .collect(); let outputs = self .sapling() .outputs() .iter() - .map( - |o| zaino_proto::proto::compact_formats::CompactSaplingOutput { - cmu: o.cmu().to_vec(), - ephemeral_key: o.ephemeral_key().to_vec(), - ciphertext: o.ciphertext().to_vec(), - }, - ) + .map(CompactSaplingOutput::into_compact) .collect(); let actions = self .orchard() .actions() .iter() - .map( - |a| zaino_proto::proto::compact_formats::CompactOrchardAction { - nullifier: a.nullifier().to_vec(), - cmx: a.cmx().to_vec(), - ephemeral_key: a.ephemeral_key().to_vec(), - ciphertext: a.ciphertext().to_vec(), - }, - ) + .map(CompactOrchardAction::into_compact) + .collect(); + + let ironwood_actions = self + .ironwood() + .actions() + .iter() + .map(CompactOrchardAction::into_compact) .collect(); let vout = self.transparent().compact_vout(); @@ -1393,12 +1450,37 @@ impl CompactTxData { spends, outputs, actions, + ironwood_actions, vin, vout, } } } +/// Converts one RPC action tuple `(nullifier, cmx, ephemeral_key, ciphertext)` into a +/// [`CompactOrchardAction`], naming `pool` in each rejection so orchard and ironwood +/// failures are distinguishable. +fn compact_orchard_action_from_parts( + pool: &str, + (nf, cmx, epk, ct): (Vec, Vec, Vec, Vec), +) -> Result { + let nf: [u8; 32] = nf + .try_into() + .map_err(|_| format!("{pool} nullifier must be 32 bytes"))?; + let cmx: [u8; 32] = cmx + .try_into() + .map_err(|_| format!("{pool} cmx must be 32 bytes"))?; + let epk: [u8; 32] = epk + .try_into() + .map_err(|_| format!("{pool} ephemeral_key must be 32 bytes"))?; + let ct: [u8; 52] = ct + .get(..52) + .ok_or_else(|| format!("{pool} ciphertext must be at least 52 bytes"))? + .try_into() + .map_err(|_| format!("{pool} ciphertext must be 52 bytes"))?; + Ok(CompactOrchardAction::new(nf, cmx, epk, ct)) +} + /// TryFrom inputs: /// - Transaction Index /// - Full Transaction @@ -1414,7 +1496,7 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac .try_into() .map_err(|_| "txid must be 32 bytes".to_string())?; - let (sapling_balance, orchard_balance) = tx.value_balances(); + let (sapling_balance, orchard_balance, ironwood_balance) = tx.value_balances(); let vin: Vec = tx .transparent_inputs() @@ -1478,29 +1560,21 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac let sapling = SaplingCompactTx::new(sapling_balance, spends, outputs); - let actions: Vec = tx + let orchard_actions: Vec = tx .orchard_actions() .into_iter() - .map(|(nf, cmx, epk, ct)| { - let nf: [u8; 32] = nf - .try_into() - .map_err(|_| "orchard nullifier must be 32 bytes".to_string())?; - let cmx: [u8; 32] = cmx - .try_into() - .map_err(|_| "orchard cmx must be 32 bytes".to_string())?; - let epk: [u8; 32] = epk - .try_into() - .map_err(|_| "orchard ephemeral_key must be 32 bytes".to_string())?; - let ct: [u8; 52] = ct - .get(..52) - .ok_or("orchard ciphertext must be at least 52 bytes")? - .try_into() - .map_err(|_| "orchard ciphertext must be 52 bytes".to_string())?; - Ok::<_, String>(CompactOrchardAction::new(nf, cmx, epk, ct)) - }) + .map(|action| compact_orchard_action_from_parts("orchard", action)) .collect::>()?; - let orchard = OrchardCompactTx::new(orchard_balance, actions); + let orchard = OrchardCompactTx::new(orchard_balance, orchard_actions); + + let ironwood_actions: Vec = tx + .ironwood_actions() + .into_iter() + .map(|action| compact_orchard_action_from_parts("ironwood", action)) + .collect::>()?; + + let ironwood = OrchardCompactTx::new(ironwood_balance, ironwood_actions); Ok(CompactTxData::new( index, @@ -1509,19 +1583,20 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac transparent, sapling, orchard, + ironwood, )) } } impl ZainoVersionedSerde for CompactTxData { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, mut w: &mut W) -> io::Result<()> { @@ -1548,6 +1623,37 @@ impl ZainoVersionedSerde for CompactTxData { transparent, sapling, orchard, + OrchardCompactTx::empty(), + )) + } + + fn encode_v2(&self, mut w: &mut W) -> io::Result<()> { + write_u64_le(&mut w, self.index)?; + + self.txid.serialize_with_version(&mut w, 1)?; + self.transparent.serialize_with_version(&mut w, 1)?; + self.sapling.serialize_with_version(&mut w, 1)?; + self.orchard.serialize_with_version(&mut w, 1)?; + self.ironwood.serialize_with_version(&mut w, 1) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let index = read_u64_le(&mut r)?; + + let txid = TransactionHash::deserialize(&mut r)?; + let transparent = TransparentCompactTx::deserialize(&mut r)?; + let sapling = SaplingCompactTx::deserialize(&mut r)?; + let orchard = OrchardCompactTx::deserialize(&mut r)?; + let ironwood = OrchardCompactTx::deserialize(&mut r)?; + + Ok(CompactTxData::new( + index, + txid, + transparent, + sapling, + orchard, + ironwood, )) } } @@ -1717,10 +1823,16 @@ impl ZainoVersionedSerde for TxInCompact { } } -/// TxInCompact = 36 bytes +/// Fixed-length encoding metadata for `TxInCompact`. +/// +/// v1 consists of a 32-byte txid + 4-byte LE index impl FixedEncodedLen for TxInCompact { - /// 32-byte txid + 4-byte LE index - const ENCODED_LEN: usize = 32 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(36), + _ => None, + } + } } /// Identifies the type of transparent transaction output script. @@ -1782,10 +1894,16 @@ impl ZainoVersionedSerde for ScriptType { } } -/// ScriptType = 1 byte +/// Fixed-length encoding metadata for `ScriptType`. +/// +/// v1 consists of a single byte impl FixedEncodedLen for ScriptType { - /// 1 byte - const ENCODED_LEN: usize = 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(1), + _ => None, + } + } } /// Try to recognise a standard P2PKH / P2SH locking script. @@ -1960,10 +2078,16 @@ impl ZainoVersionedSerde for TxOutCompact { } } -/// TxOutCompact = 29 bytes +/// Fixed-length encoding metadata for `TxOutCompact`. +/// +/// v1 consists of a 8-byte LE value + 20-byte script hash + 1-byte type impl FixedEncodedLen for TxOutCompact { - /// 8-byte LE value + 20-byte script hash + 1-byte type - const ENCODED_LEN: usize = 8 + 20 + 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(29), + _ => None, + } + } } /// Compact representation of Sapling shielded transaction data for wallet scanning. @@ -2085,10 +2209,16 @@ impl ZainoVersionedSerde for CompactSaplingSpend { } } -/// 32-byte nullifier +/// Fixed-length encoding metadata for `CompactSaplingSpend`. +/// +/// v1 consists of a 32 byte nullifier impl FixedEncodedLen for CompactSaplingSpend { - /// 32 bytes - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Compact representation of a newly created Sapling shielded note output. @@ -2166,10 +2296,16 @@ impl ZainoVersionedSerde for CompactSaplingOutput { } } -/// 116 bytes +/// Fixed-length encoding metadata for `CompactSaplingOutput`. +/// +/// v1 consists of a 32-byte cmu + 32-byte ephemeral_key + 52-byte ciphertext impl FixedEncodedLen for CompactSaplingOutput { - /// 32-byte cmu + 32-byte ephemeral_key + 52-byte ciphertext - const ENCODED_LEN: usize = 32 + 32 + 52; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(116), + _ => None, + } + } } /// Compact summary of all shielded activity in a transaction. @@ -2197,6 +2333,14 @@ impl OrchardCompactTx { pub fn actions(&self) -> &[CompactOrchardAction] { &self.actions } + + /// Return an empty OrchardCompactTx + pub fn empty() -> Self { + Self { + value: None, + actions: Vec::new(), + } + } } impl ZainoVersionedSerde for OrchardCompactTx { @@ -2318,10 +2462,20 @@ impl ZainoVersionedSerde for CompactOrchardAction { } } -// CompactOrchardAction = 148 bytes +/// Fixed-length encoding metadata for `CompactOrchardAction`. +/// +/// v1 consists of a: +/// - 32-byte nullifier +/// - 32-byte cmx +/// - 32-byte ephemeral_key +/// - 52-byte ciphertext impl FixedEncodedLen for CompactOrchardAction { - /// 32-byte nullifier + 32-byte cmx + 32-byte ephemeral_key + 52-byte ciphertext - const ENCODED_LEN: usize = 32 + 32 + 32 + 52; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(148), + _ => None, + } + } } /// Identifies a transaction's location by block height and transaction index. @@ -2377,10 +2531,16 @@ impl ZainoVersionedSerde for TxLocation { } } -/// 6 bytes, BE encoded. +/// Fixed-length encoding metadata for `TxLocation`. +/// +/// v1 consists of a 4-byte big-endian block_index + 2-byte big-endian tx_index impl FixedEncodedLen for TxLocation { - /// 4-byte big-endian block_index + 2-byte big-endian tx_index - const ENCODED_LEN: usize = 4 + 2; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(6), + _ => None, + } + } } /// Single transparent-address activity record (input or output). @@ -2481,15 +2641,22 @@ impl ZainoVersionedSerde for AddrHistRecord { } } -/// 18 byte total +/// Fixed-length encoding metadata for `AddrHistRecord`. +/// +/// v1 consists of: +/// 1 byte: TxLocation tag +/// +6 bytes: TxLocation body (4 BE block_index + 2 BE tx_index) +/// +2 bytes: out_index (BE) +/// +8 bytes: value (LE) +/// +1 byte : flags +/// =18 bytes impl FixedEncodedLen for AddrHistRecord { - /// 1 byte: TxLocation tag - /// +6 bytes: TxLocation body (4 BE block_index + 2 BE tx_index) - /// +2 bytes: out_index (BE) - /// +8 bytes: value (LE) - /// +1 byte : flags - /// =18 bytes - const ENCODED_LEN: usize = (TxLocation::ENCODED_LEN + 1) + 2 + 8 + 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(18), + _ => None, + } + } } /// AddrHistRecord database byte array. @@ -2579,6 +2746,9 @@ impl ZainoVersionedSerde for AddrEventBytes { } } +/// Fixed-length encoding metadata for `AddrEventBytes`. +/// +/// v1 consists of: /// 17 byte body: /// /// ```text @@ -2589,7 +2759,12 @@ impl ZainoVersionedSerde for AddrEventBytes { /// [9..17] value (LE u64) | Amount in zatoshi, little-endian /// ``` impl FixedEncodedLen for AddrEventBytes { - const ENCODED_LEN: usize = 17; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(17), + _ => None, + } + } } // *** Sharding *** @@ -2659,10 +2834,16 @@ impl ZainoVersionedSerde for ShardRoot { } } -/// 68 byte body. +/// Fixed-length encoding metadata for `ShardRoot`. +/// +/// v1 consists of a 32 byte hash + 32 byte hash + 4 byte block height impl FixedEncodedLen for ShardRoot { - /// 32 byte hash + 32 byte hash + 4 byte block height - const ENCODED_LEN: usize = 32 + 32 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(68), + _ => None, + } + } } // *** Wrapper Objects *** diff --git a/packages/zaino-state/src/chain_index/types/db/metadata.rs b/packages/zaino-state/src/chain_index/types/db/metadata.rs index dd95d5ef4..efbb3880f 100644 --- a/packages/zaino-state/src/chain_index/types/db/metadata.rs +++ b/packages/zaino-state/src/chain_index/types/db/metadata.rs @@ -102,10 +102,16 @@ impl ZainoVersionedSerde for MempoolInfo { } } -/// 24 byte body. +/// Fixed-length encoding metadata for `MempoolInfo`. +/// +/// v1 consists of 8 byte size + 8 byte bytes + 8 byte usage impl FixedEncodedLen for MempoolInfo { - /// 8 byte size + 8 byte bytes + 8 byte usage - const ENCODED_LEN: usize = 8 + 8 + 8; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(24), + _ => None, + } + } } impl From for MempoolInfo { @@ -328,9 +334,16 @@ impl ZainoVersionedSerde for FinalisedTxOutSetInfoAccumulator { } } +/// Fixed-length encoding metadata for `FinalisedTxOutSetInfoAccumulator`. +/// +/// v1 consists of 8 + 8 + 8 + 32 + 8 = 64 bytes impl FixedEncodedLen for FinalisedTxOutSetInfoAccumulator { - /// 8 + 8 + 8 + 32 + 8 = 64 bytes. - const ENCODED_LEN: usize = 8 + 8 + 8 + 32 + 8; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(64), + _ => None, + } + } } #[cfg(test)] @@ -353,7 +366,7 @@ mod tests { assert_eq!( encoded_accumulator.len(), - FinalisedTxOutSetInfoAccumulator::VERSIONED_LEN + FinalisedTxOutSetInfoAccumulator::latest_versioned_len().unwrap() ); let decoded_accumulator = diff --git a/packages/zaino-state/src/chain_index/types/helpers.rs b/packages/zaino-state/src/chain_index/types/helpers.rs index d4e63db94..dbfb5297a 100644 --- a/packages/zaino-state/src/chain_index/types/helpers.rs +++ b/packages/zaino-state/src/chain_index/types/helpers.rs @@ -60,6 +60,8 @@ pub struct TreeRootData { pub sapling: Option<(zebra_chain::sapling::tree::Root, u64)>, /// Orchard tree root and size pub orchard: Option<(zebra_chain::orchard::tree::Root, u64)>, + /// Orchard tree root and size + pub ironwood: Option<(zebra_chain::orchard::tree::Root, u64)>, } impl TreeRootData { @@ -67,11 +69,20 @@ impl TreeRootData { pub fn new( sapling: Option<(zebra_chain::sapling::tree::Root, u64)>, orchard: Option<(zebra_chain::orchard::tree::Root, u64)>, + ironwood: Option<(zebra_chain::orchard::tree::Root, u64)>, ) -> Self { - Self { sapling, orchard } + Self { + sapling, + orchard, + ironwood, + } } /// Extract with defaults for genesis/sync use case + /// + /// Sapling and orchard roots default when absent (genesis). The ironwood component + /// passes through unchanged: `None` means the block has no ironwood treestate and + /// must be stored as `None`. pub fn extract_with_defaults( self, ) -> ( @@ -79,10 +90,17 @@ impl TreeRootData { u64, zebra_chain::orchard::tree::Root, u64, + Option<(zebra_chain::orchard::tree::Root, u64)>, ) { let (sapling_root, sapling_size) = self.sapling.unwrap_or_default(); let (orchard_root, orchard_size) = self.orchard.unwrap_or_default(); - (sapling_root, sapling_size, orchard_root, orchard_size) + ( + sapling_root, + sapling_size, + orchard_root, + orchard_size, + self.ironwood, + ) } } @@ -97,6 +115,9 @@ pub struct BlockMetadata { pub orchard_root: zebra_chain::orchard::tree::Root, /// Orchard tree size pub orchard_size: u32, + /// Ironwood commitment tree root and size; `None` when the block has no ironwood + /// treestate (below NU6.3 activation, or a network with no NU6.3 activation height) + pub ironwood: Option<(zebra_chain::orchard::tree::Root, u32)>, /// Parent block's chainwork (`None` for genesis). pub parent_chainwork: Option, /// Network for block validation @@ -110,6 +131,7 @@ impl BlockMetadata { sapling_size: u32, orchard_root: zebra_chain::orchard::tree::Root, orchard_size: u32, + ironwood: Option<(zebra_chain::orchard::tree::Root, u32)>, parent_chainwork: Option, network: zebra_chain::parameters::Network, ) -> Self { @@ -118,6 +140,7 @@ impl BlockMetadata { sapling_size, orchard_root, orchard_size, + ironwood, parent_chainwork, network, } @@ -172,9 +195,16 @@ impl<'a> BlockWithMetadata<'a> { let transparent = self.extract_transparent_data(txn)?; let sapling = self.extract_sapling_data(txn); let orchard = self.extract_orchard_data(txn); - - let txdata = - CompactTxData::new(i as u64, txn.hash().into(), transparent, sapling, orchard); + let ironwood = self.extract_ironwood_data(txn); + + let txdata = CompactTxData::new( + i as u64, + txn.hash().into(), + transparent, + sapling, + orchard, + ironwood, + ); transactions.push(txdata); } @@ -217,18 +247,41 @@ impl<'a> BlockWithMetadata<'a> { Ok(TransparentCompactTx::new(inputs, outputs)) } + /// Returns the first 52 bytes of a 580-byte encrypted note ciphertext: the prefix a + /// compact block carries, sufficient for trial decryption. + fn compact_ciphertext_prefix(enc_ciphertext: [u8; 580]) -> [u8; 52] { + std::array::from_fn(|i| enc_ciphertext[i]) + } + + /// Builds the Orchard-shaped compact transaction shared by the Orchard and Ironwood + /// pools from a value balance and the pool's actions. + fn extract_orchard_shaped_data<'t>( + value_balance: i64, + actions: impl Iterator, + ) -> OrchardCompactTx { + OrchardCompactTx::new( + (value_balance != 0).then_some(value_balance), + actions + .map(|action| { + CompactOrchardAction::new( + <[u8; 32]>::from(action.nullifier), + <[u8; 32]>::from(action.cm_x), + <[u8; 32]>::from(action.ephemeral_key), + Self::compact_ciphertext_prefix(<[u8; 580]>::from(action.enc_ciphertext)), + ) + }) + .collect(), + ) + } + /// Extract sapling transaction data fn extract_sapling_data( &self, txn: &zebra_chain::transaction::Transaction, ) -> SaplingCompactTx { let sapling_value = { - let val = txn.sapling_value_balance().sapling_amount(); - if val == 0 { - None - } else { - Some(i64::from(val)) - } + let value = i64::from(txn.sapling_value_balance().sapling_amount()); + (value != 0).then_some(value) }; SaplingCompactTx::new( @@ -238,13 +291,10 @@ impl<'a> BlockWithMetadata<'a> { .collect(), txn.sapling_outputs() .map(|output| { - let cipher: [u8; 52] = <[u8; 580]>::from(output.enc_ciphertext)[..52] - .try_into() - .unwrap(); // TODO: Remove unwrap CompactSaplingOutput::new( output.cm_u.to_bytes(), <[u8; 32]>::from(output.ephemeral_key), - cipher, + Self::compact_ciphertext_prefix(<[u8; 580]>::from(output.enc_ciphertext)), ) }) .collect::>(), @@ -256,30 +306,20 @@ impl<'a> BlockWithMetadata<'a> { &self, txn: &zebra_chain::transaction::Transaction, ) -> OrchardCompactTx { - let orchard_value = { - let val = txn.orchard_value_balance().orchard_amount(); - if val == 0 { - None - } else { - Some(i64::from(val)) - } - }; + Self::extract_orchard_shaped_data( + i64::from(txn.orchard_value_balance().orchard_amount()), + txn.orchard_actions(), + ) + } - OrchardCompactTx::new( - orchard_value, - txn.orchard_actions() - .map(|action| { - let cipher: [u8; 52] = <[u8; 580]>::from(action.enc_ciphertext)[..52] - .try_into() - .unwrap(); // TODO: Remove unwrap - CompactOrchardAction::new( - <[u8; 32]>::from(action.nullifier), - <[u8; 32]>::from(action.cm_x), - <[u8; 32]>::from(action.ephemeral_key), - cipher, - ) - }) - .collect::>(), + /// Extract ironwood transaction data + fn extract_ironwood_data( + &self, + txn: &zebra_chain::transaction::Transaction, + ) -> OrchardCompactTx { + Self::extract_orchard_shaped_data( + i64::from(txn.ironwood_value_balance().ironwood_amount()), + txn.ironwood_actions(), ) } @@ -307,17 +347,21 @@ impl<'a> BlockWithMetadata<'a> { Ok(BlockContext::new(hash, parent_hash, chainwork, height)) } +} - /// Create commitment tree data from metadata +impl BlockMetadata { + /// Create the stored commitment tree data for this block's metadata. fn create_commitment_tree_data(&self) -> super::db::CommitmentTreeData { let commitment_tree_roots = super::db::CommitmentTreeRoots::new( - <[u8; 32]>::from(self.metadata.sapling_root), - <[u8; 32]>::from(self.metadata.orchard_root), + <[u8; 32]>::from(self.sapling_root), + <[u8; 32]>::from(self.orchard_root), + self.ironwood.map(|(root, _)| <[u8; 32]>::from(root)), ); let commitment_tree_size = super::db::CommitmentTreeSizes::new( - self.metadata.sapling_size, - self.metadata.orchard_size, + self.sapling_size, + self.orchard_size, + self.ironwood.map_or(0, |(_, size)| size), ); super::db::CommitmentTreeData::new(commitment_tree_roots, commitment_tree_size) @@ -332,7 +376,7 @@ impl TryFrom> for IndexedBlock { let data = block_with_metadata.extract_block_data()?; let transactions = block_with_metadata.extract_transactions()?; let context = block_with_metadata.create_block_context()?; - let commitment_tree_data = block_with_metadata.create_commitment_tree_data(); + let commitment_tree_data = block_with_metadata.metadata.create_commitment_tree_data(); Ok(IndexedBlock { context, @@ -342,3 +386,38 @@ impl TryFrom> for IndexedBlock { }) } } + +#[cfg(test)] +mod create_commitment_tree_data { + use super::*; + + /// Regression test: a block whose source reported no ironwood treestate (pre-NU6.3, + /// or a network with no NU6.3 activation height) must store its ironwood root as + /// `None` — the encoding the v1.2.1->v1.3.0 migration and the CommitmentTreeRoots V1 + /// decode produce for the same state. The write path instead erased the `Option` via + /// `extract_with_defaults` and stored `Some([0; 32])`, so a freshly synced database + /// and a migrated database encoded identical pre-activation heights differently. + /// + #[test] + fn absent_ironwood_root_is_stored_as_none() { + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood) = + TreeRootData::new(None, None, None).extract_with_defaults(); + let metadata = BlockMetadata::new( + sapling_root, + sapling_size as u32, + orchard_root, + orchard_size as u32, + ironwood.map(|(root, size)| (root, size as u32)), + None, + zebra_chain::parameters::Network::Mainnet, + ); + + let commitment_tree_data = metadata.create_commitment_tree_data(); + + assert_eq!( + commitment_tree_data.roots().ironwood(), + &None, + "ironwood root must be stored as None when the source reported no ironwood treestate" + ); + } +} diff --git a/packages/zaino-state/src/config.rs b/packages/zaino-state/src/config.rs index 0083dbcd4..3cfff0d76 100644 --- a/packages/zaino-state/src/config.rs +++ b/packages/zaino-state/src/config.rs @@ -42,31 +42,37 @@ impl std::fmt::Display for DonationAddress { } } -/// Type of backend to be used. +/// How the [`NodeBackedIndexerService`](crate::NodeBackedIndexerService) connects to its +/// validator to source blockchain data. /// -/// Determines how Zaino fetches blockchain data from the validator. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "lowercase")] -pub enum BackendType { - /// Uses Zebra's ReadStateService for direct state access. - /// - /// More efficient but requires running on the same machine as Zebra. - State, - /// Uses JSON-RPC client to fetch data. +/// Carries the connection-specific configuration inline: the `Direct` variant owns the +/// Zebra `ReadStateService` settings, while `Rpc` needs only the JSON-RPC connection bits +/// already held in [`CommonBackendConfig`]. +#[derive(Debug, Clone)] +pub enum ValidatorConnectionType { + /// JSON-RPC connection (formerly `Fetch`). /// /// Compatible with Zcashd, Zebra, or another Zaino instance. - #[default] - Fetch, + Rpc, + /// Direct Zebra `ReadStateService` connection (formerly `State`). + /// + /// More efficient but requires running alongside a Zebra whose state DB and gRPC + /// sync endpoint we own. + Direct(DirectConnectionConfig), } -/// Unified backend configuration enum. +/// Connection parameters for the [`ValidatorConnectionType::Direct`] backend. +/// +/// Bundles the Zebra `ReadStateService` DB config with the gRPC sync endpoint used to +/// drive it, so the whole "direct connection" description travels as one value. #[derive(Debug, Clone)] -#[allow(deprecated)] -pub enum BackendConfig { - /// StateService config. - State(StateServiceConfig), - /// Fetchservice config. - Fetch(FetchServiceConfig), +pub struct DirectConnectionConfig { + /// Zebra [`zebra_state::ReadStateService`] config data (DB cache dir etc.). + pub validator_state_config: zebra_state::Config, + /// Validator gRPC address (requires ip:port format for Zebra state sync). + pub validator_grpc_address: std::net::SocketAddr, + /// Whether validator cookie authentication is enabled. + pub validator_cookie_auth: bool, } /// Configuration shared by every backend variant. @@ -109,29 +115,55 @@ pub struct CommonBackendConfig { pub indexer_version: String, } -/// Holds config data for [crate::StateService]. +impl CommonBackendConfig { + /// Builds a [`CommonBackendConfig`], applying the default RPC user/password + /// placeholder (`"xxxxxx"`) and this crate's `CARGO_PKG_VERSION` as the + /// indexer version. Shared by the [`NodeBackedIndexerServiceConfig`] constructors. + #[allow(clippy::too_many_arguments)] + pub fn new( + validator_rpc_address: String, + validator_cookie_path: Option, + validator_rpc_user: Option, + validator_rpc_password: Option, + service: ServiceConfig, + storage: StorageConfig, + ephemeral_finalised_state: bool, + network: Network, + donation_address: Option, + ) -> Self { + CommonBackendConfig { + validator_rpc_address, + validator_cookie_path, + validator_rpc_user: validator_rpc_user.unwrap_or_else(|| "xxxxxx".to_string()), + validator_rpc_password: validator_rpc_password.unwrap_or_else(|| "xxxxxx".to_string()), + service, + storage, + ephemeral_finalised_state, + network, + donation_address, + indexer_version: env!("CARGO_PKG_VERSION").to_string(), + } + } +} + +/// Holds config data for [`crate::NodeBackedIndexerService`]. +/// +/// Replaces the former per-backend `FetchServiceConfig` / `StateServiceConfig`: the +/// shared bits live in [`CommonBackendConfig`] and the backend-specific bits (only the +/// `Direct` backend has any) live in [`ValidatorConnectionType`]. #[derive(Debug, Clone)] -// #[deprecated] -pub struct StateServiceConfig { - /// Settings shared with [`FetchServiceConfig`]. +pub struct NodeBackedIndexerServiceConfig { + /// Connection-independent settings (validator RPC, storage, network, ...). pub common: CommonBackendConfig, - /// Zebra [`zebra_state::ReadStateService`] config data - pub validator_state_config: zebra_state::Config, - /// Validator gRPC address (requires ip:port format for Zebra state sync). - pub validator_grpc_address: std::net::SocketAddr, - /// Validator cookie auth. - pub validator_cookie_auth: bool, + /// Which validator connection to use, and its connection-specific config. + pub connection: ValidatorConnectionType, } -#[allow(deprecated)] -impl StateServiceConfig { - /// Returns a new instance of [`StateServiceConfig`]. +impl NodeBackedIndexerServiceConfig { + /// Returns a JSON-RPC (`Rpc`) service config (formerly `FetchServiceConfig::new`). #[allow(clippy::too_many_arguments)] - pub fn new( - validator_state_config: zebra_state::Config, + pub fn new_rpc( validator_rpc_address: String, - validator_grpc_address: std::net::SocketAddr, - validator_cookie_auth: bool, validator_cookie_path: Option, validator_rpc_user: Option, validator_rpc_password: Option, @@ -141,44 +173,30 @@ impl StateServiceConfig { network: Network, donation_address: Option, ) -> Self { - tracing::trace!( - activations = ?network.to_zebra_network().full_activation_list(), - "state service expecting NU activations" - ); - StateServiceConfig { - common: CommonBackendConfig { + NodeBackedIndexerServiceConfig { + common: CommonBackendConfig::new( validator_rpc_address, validator_cookie_path, - validator_rpc_user: validator_rpc_user.unwrap_or("xxxxxx".to_string()), - validator_rpc_password: validator_rpc_password.unwrap_or("xxxxxx".to_string()), + validator_rpc_user, + validator_rpc_password, service, storage, ephemeral_finalised_state, network, donation_address, - indexer_version: env!("CARGO_PKG_VERSION").to_string(), - }, - validator_state_config, - validator_grpc_address, - validator_cookie_auth, + ), + connection: ValidatorConnectionType::Rpc, } } -} -/// Holds config data for [crate::FetchService]. -#[derive(Debug, Clone)] -#[deprecated] -pub struct FetchServiceConfig { - /// Settings shared with [`StateServiceConfig`]. - pub common: CommonBackendConfig, -} - -#[allow(deprecated)] -impl FetchServiceConfig { - /// Returns a new instance of [`FetchServiceConfig`]. + /// Returns a direct-`ReadStateService` (`Direct`) service config + /// (formerly `StateServiceConfig::new`). #[allow(clippy::too_many_arguments)] - pub fn new( + pub fn new_direct( + validator_state_config: zebra_state::Config, validator_rpc_address: String, + validator_grpc_address: std::net::SocketAddr, + validator_cookie_auth: bool, validator_cookie_path: Option, validator_rpc_user: Option, validator_rpc_password: Option, @@ -188,19 +206,25 @@ impl FetchServiceConfig { network: Network, donation_address: Option, ) -> Self { - FetchServiceConfig { - common: CommonBackendConfig { + // The config carries only the network kind; the activation schedule + // is adopted from the validator at spawn and logged there (#1076). + NodeBackedIndexerServiceConfig { + common: CommonBackendConfig::new( validator_rpc_address, validator_cookie_path, - validator_rpc_user: validator_rpc_user.unwrap_or("xxxxxx".to_string()), - validator_rpc_password: validator_rpc_password.unwrap_or("xxxxxx".to_string()), + validator_rpc_user, + validator_rpc_password, service, storage, ephemeral_finalised_state, network, donation_address, - indexer_version: env!("CARGO_PKG_VERSION").to_string(), - }, + ), + connection: ValidatorConnectionType::Direct(DirectConnectionConfig { + validator_state_config, + validator_grpc_address, + validator_cookie_auth, + }), } } } @@ -212,8 +236,10 @@ pub struct ChainIndexConfig { pub storage: StorageConfig, /// Database version selected to be run. pub db_version: u32, - /// Network type. - pub network: Network, + /// The runtime network, carrying the activation schedule adopted from + /// the validator (zaino#1076) — or, in fixtures that are their own + /// chain, the fixture's schedule. + pub network: zebra_chain::parameters::Network, /// Ephemeral finalised state: /// /// If true, FinalisedState does not write data to disk, @@ -225,44 +251,24 @@ pub struct ChainIndexConfig { } impl ChainIndexConfig { - /// Returns a new instance of [`BlockCacheConfig`]. - #[allow(dead_code)] - pub fn new(storage: StorageConfig, db_version: u32, network: Network, ephemeral: bool) -> Self { - ChainIndexConfig { - storage, - db_version, - network, - ephemeral, - } - } -} - -impl From for ChainIndexConfig { - fn from(value: CommonBackendConfig) -> Self { + /// Builds the chain-index config from the backend config plus the + /// runtime network. The backend config carries only a network kind, so + /// the adopted runtime network arrives as its own argument — there is + /// no conversion from a service config alone. + pub fn from_backend_config( + common: &CommonBackendConfig, + network: zebra_chain::parameters::Network, + ) -> Self { Self { - storage: value.storage, + storage: common.storage.clone(), // TODO: update zaino configs to include db version. db_version: 1, - network: value.network, - ephemeral: value.ephemeral_finalised_state, + network, + ephemeral: common.ephemeral_finalised_state, } } } -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: StateServiceConfig) -> Self { - value.common.into() - } -} - -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: FetchServiceConfig) -> Self { - value.common.into() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/packages/zaino-state/src/error.rs b/packages/zaino-state/src/error.rs index 839c17045..b0eb87b31 100644 --- a/packages/zaino-state/src/error.rs +++ b/packages/zaino-state/src/error.rs @@ -10,11 +10,11 @@ use std::{any::type_name, fmt::Display}; use zaino_fetch::jsonrpsee::connector::RpcRequestError; use zaino_proto::proto::utils::GetBlockRangeError; -/// Errors related to the `StateService`. -// #[deprecated] +/// Errors returned by the [`NodeBackedIndexerService`](crate::NodeBackedIndexerService) +/// subscriber's `ZcashIndexer` / `LightWalletIndexer` methods. #[derive(Debug, thiserror::Error)] #[allow(clippy::result_large_err)] -pub enum StateServiceError { +pub enum NodeBackedIndexerServiceError { /// Critical Errors, Restart Zaino. #[error("Critical error: {0}")] Critical(String), @@ -88,7 +88,7 @@ pub enum StateServiceError { UnavailableNotSyncedEnough, } -impl From for StateServiceError { +impl From for NodeBackedIndexerServiceError { fn from(value: GetBlockRangeError) -> Self { match value { GetBlockRangeError::StartHeightOutOfRange => { @@ -115,55 +115,59 @@ impl From for StateServiceError { } #[allow(deprecated)] -impl From for tonic::Status { - fn from(error: StateServiceError) -> Self { +impl From for tonic::Status { + fn from(error: NodeBackedIndexerServiceError) -> Self { match error { - StateServiceError::Critical(message) => tonic::Status::internal(message), - StateServiceError::Custom(message) => tonic::Status::internal(message), - StateServiceError::JoinError(err) => { + NodeBackedIndexerServiceError::Critical(message) => tonic::Status::internal(message), + NodeBackedIndexerServiceError::Custom(message) => tonic::Status::internal(message), + NodeBackedIndexerServiceError::JoinError(err) => { tonic::Status::internal(format!("Join error: {err}")) } - StateServiceError::JsonRpcConnectorError(err) => { + NodeBackedIndexerServiceError::JsonRpcConnectorError(err) => { tonic::Status::internal(format!("JsonRpcConnector error: {err}")) } - StateServiceError::RpcError(err) => { + NodeBackedIndexerServiceError::RpcError(err) => { tonic::Status::internal(format!("RPC error: {err:?}")) } - StateServiceError::ChainIndexError(err) => match err.kind { + NodeBackedIndexerServiceError::ChainIndexError(err) => match err.kind { ChainIndexErrorKind::InternalServerError => tonic::Status::internal(err.message), ChainIndexErrorKind::InvalidSnapshot => { tonic::Status::failed_precondition(err.message) } }, - StateServiceError::BlockCacheError(err) => { + NodeBackedIndexerServiceError::BlockCacheError(err) => { tonic::Status::internal(format!("BlockCache error: {err:?}")) } - StateServiceError::MempoolError(err) => { + NodeBackedIndexerServiceError::MempoolError(err) => { tonic::Status::internal(format!("Mempool error: {err:?}")) } - StateServiceError::TonicStatusError(err) => err, - StateServiceError::SerializationError(err) => { + NodeBackedIndexerServiceError::TonicStatusError(err) => err, + NodeBackedIndexerServiceError::SerializationError(err) => { tonic::Status::internal(format!("Serialization error: {err}")) } - StateServiceError::TryFromIntError(err) => { + NodeBackedIndexerServiceError::TryFromIntError(err) => { tonic::Status::internal(format!("Integer conversion error: {err}")) } - StateServiceError::IoError(err) => tonic::Status::internal(format!("IO error: {err}")), - StateServiceError::Generic(err) => { + NodeBackedIndexerServiceError::IoError(err) => { + tonic::Status::internal(format!("IO error: {err}")) + } + NodeBackedIndexerServiceError::Generic(err) => { tonic::Status::internal(format!("Generic error: {err}")) } - ref err @ StateServiceError::ZebradVersionMismatch { .. } => { + ref err @ NodeBackedIndexerServiceError::ZebradVersionMismatch { .. } => { tonic::Status::internal(err.to_string()) } - StateServiceError::UnhandledRpcError(e) => tonic::Status::internal(e.to_string()), - StateServiceError::UnavailableNotSyncedEnough => { + NodeBackedIndexerServiceError::UnhandledRpcError(e) => { + tonic::Status::internal(e.to_string()) + } + NodeBackedIndexerServiceError::UnavailableNotSyncedEnough => { tonic::Status::failed_precondition("zaino not yet synced".to_string()) } } } } -impl From> for StateServiceError { +impl From> for NodeBackedIndexerServiceError { fn from(value: RpcRequestError) -> Self { match value { RpcRequestError::Transport(transport_error) => { @@ -184,122 +188,6 @@ impl From> for StateServiceError { } } -/// Errors related to the `FetchService`. -#[deprecated] -#[derive(Debug, thiserror::Error)] -pub enum FetchServiceError { - /// Critical Errors, Restart Zaino. - #[error("Critical error: {0}")] - Critical(String), - - /// Error from JsonRpcConnector. - #[error("JsonRpcConnector error: {0}")] - JsonRpcConnectorError(#[from] zaino_fetch::jsonrpsee::error::TransportError), - - /// Chain index error. - #[error("Chain index error: {0}")] - ChainIndexError(#[from] ChainIndexError), - - /// RPC error in compatibility with zcashd. - #[error("RPC error: {0:?}")] - RpcError(#[from] zaino_fetch::jsonrpsee::connector::RpcError), - - /// Tonic gRPC error. - #[error("Tonic status error: {0}")] - TonicStatusError(#[from] tonic::Status), - - /// Serialization error. - #[error("Serialization error: {0}")] - SerializationError(#[from] zebra_chain::serialization::SerializationError), - #[error("Zaino has not synced high enough to serve this data")] - /// Zaino has not yet synced. - UnavailableNotSyncedEnough, -} - -impl From for tonic::Status { - fn from(error: FetchServiceError) -> Self { - match error { - FetchServiceError::Critical(message) => tonic::Status::internal(message), - FetchServiceError::JsonRpcConnectorError(err) => { - tonic::Status::internal(format!("JsonRpcConnector error: {err}")) - } - FetchServiceError::ChainIndexError(err) => match err.kind { - ChainIndexErrorKind::InternalServerError => tonic::Status::internal(err.message), - ChainIndexErrorKind::InvalidSnapshot => { - tonic::Status::failed_precondition(err.message) - } - }, - FetchServiceError::RpcError(err) => { - tonic::Status::internal(format!("RPC error: {err:?}")) - } - FetchServiceError::TonicStatusError(err) => err, - FetchServiceError::SerializationError(err) => { - tonic::Status::internal(format!("Serialization error: {err}")) - } - FetchServiceError::UnavailableNotSyncedEnough => { - tonic::Status::failed_precondition("zaino not yet synced".to_string()) - } - } - } -} - -impl From> for FetchServiceError { - fn from(value: RpcRequestError) -> Self { - match value { - RpcRequestError::Transport(transport_error) => { - FetchServiceError::JsonRpcConnectorError(transport_error) - } - RpcRequestError::JsonRpc(error) => { - FetchServiceError::Critical(format!("argument failed to serialze: {error}")) - } - RpcRequestError::InternalUnrecoverable(e) => { - FetchServiceError::Critical(format!("Internal unrecoverable error: {e}")) - } - RpcRequestError::ServerWorkQueueFull => FetchServiceError::Critical( - "Server queue full. Handling for this not yet implemented".to_string(), - ), - RpcRequestError::Method(e) => FetchServiceError::Critical(format!( - "unhandled rpc-specific {} error: {}", - type_name::(), - e.to_string() - )), - RpcRequestError::UnexpectedErrorResponse(error) => { - FetchServiceError::Critical(format!( - "unhandled rpc-specific {} error: {}", - type_name::(), - error - )) - } - } - } -} - -impl From for FetchServiceError { - fn from(value: GetBlockRangeError) -> Self { - match value { - GetBlockRangeError::StartHeightOutOfRange => { - FetchServiceError::TonicStatusError(tonic::Status::out_of_range( - "Error: Start height out of range. Failed to convert to u32.", - )) - } - GetBlockRangeError::NoStartHeightProvided => FetchServiceError::TonicStatusError( - tonic::Status::out_of_range("Error: No start height given"), - ), - GetBlockRangeError::EndHeightOutOfRange => { - FetchServiceError::TonicStatusError(tonic::Status::out_of_range( - "Error: End height out of range. Failed to convert to u32.", - )) - } - GetBlockRangeError::NoEndHeightProvided => FetchServiceError::TonicStatusError( - tonic::Status::out_of_range("Error: No end height given."), - ), - GetBlockRangeError::PoolTypeArgumentError(_) => FetchServiceError::TonicStatusError( - tonic::Status::invalid_argument("Error: invalid pool type"), - ), - } - } -} - /// These aren't the best conversions, but the MempoolError should go away /// in favor of a new type with the new chain cache is complete impl From> for MempoolError { @@ -622,6 +510,18 @@ impl ChainIndexError { } } + /// Constructs an `InternalServerError`-kind error from a typed error, + /// preserving it as `source` so zaino-serve's RPC-error-code recovery + /// walks can downcast to it (e.g. the legacy `-8` code carried by an + /// [`RpcError`](zaino_fetch::jsonrpsee::connector::RpcError)). + pub(crate) fn internal_from(error: impl std::error::Error + Send + Sync + 'static) -> Self { + Self { + kind: ChainIndexErrorKind::InternalServerError, + message: error.to_string(), + source: Some(Box::new(error)), + } + } + pub(crate) fn backing_validator(value: impl std::error::Error + Send + Sync + 'static) -> Self { Self { kind: ChainIndexErrorKind::InternalServerError, diff --git a/packages/zaino-state/src/indexer.rs b/packages/zaino-state/src/indexer.rs index acdf57a43..42199f193 100644 --- a/packages/zaino-state/src/indexer.rs +++ b/packages/zaino-state/src/indexer.rs @@ -1,5 +1,10 @@ -//! Holds the Indexer trait containing the zcash RPC definitions served by zaino -//! and generic wrapper structs for the various backend options available. +//! Zaino's indexer frontend: the `ZcashIndexer` / `LightWalletIndexer` RPC trait +//! definitions served by zaino, the generic [`IndexerService`] / [`IndexerSubscriber`] +//! wrappers, and the concrete [`node_backed_indexer::NodeBackedIndexerService`] — the +//! single validator-backed service (JSON-RPC `Rpc` or direct `ReadStateService` +//! connection, selected at runtime). + +pub(crate) mod node_backed_indexer; use crate::SendFut; use tokio::{sync::mpsc, time::timeout}; @@ -12,9 +17,12 @@ use zaino_fetch::jsonrpsee::response::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::ZValidateAddressResponse, + z_validate_address::{ + InvalidZValidateAddress, KnownZValidateAddress, ZValidateAddressResponse, + DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, + }, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, - GetTxOutSetInfoResponse, + GetSubtreesResponse, GetTxOutSetInfoResponse, }; use zaino_proto::proto::{ compact_formats::CompactBlock, @@ -29,7 +37,9 @@ use zebra_chain::{ block::Height, serialization::BytesInDisplayOrder as _, subtree::NoteCommitmentSubtreeIndex, }; use zebra_rpc::{ - client::{GetSubtreesByIndexResponse, GetTreestateResponse, ValidateAddressResponse}, + client::{ + GetSubtreesByIndexResponse, GetTreestateResponse, SubtreeRpcData, ValidateAddressResponse, + }, methods::{ AddressBalance, GetAddressBalanceRequest, GetAddressTxIdsRequest, GetAddressUtxos, GetBlock, GetBlockHash, GetBlockchainInfoResponse, GetInfo, GetRawTransaction, @@ -43,10 +53,10 @@ use crate::{ AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, SubtreeRootReplyStream, UtxoReplyStream, }, - BackendType, }; -/// Wrapper Struct for a ZainoState chain-fetch service (StateService, FetchService) +/// Wrapper struct for a ZainoState chain-fetch service (currently the single +/// [`node_backed_indexer::NodeBackedIndexerService`]). /// /// The future plan is to also add a TonicService and DarksideService to this to enable /// wallets to use a single unified chain fetch service. @@ -87,9 +97,6 @@ where /// Implementors automatically gain [`Liveness`](zaino_common::probing::Liveness) and /// [`Readiness`](zaino_common::probing::Readiness) via the [`Status`] supertrait. pub trait ZcashService: Sized + Status { - /// Backend type. Read state or fetch service. - const BACKEND_TYPE: BackendType; - /// A subscriber to the service, used to fetch chain data. type Subscriber: Clone + ZcashIndexer + LightWalletIndexer + Status; @@ -108,7 +115,8 @@ pub trait ZcashService: Sized + Status { fn close(&mut self); } -/// Wrapper Struct for a ZainoState chain-fetch service subscriber (StateServiceSubscriber, FetchServiceSubscriber) +/// Wrapper struct for a ZainoState chain-fetch service subscriber (currently the single +/// [`node_backed_indexer::NodeBackedIndexerServiceSubscriber`]). /// /// The future plan is to also add a TonicServiceSubscriber and DarksideServiceSubscriber to this to enable wallets to use a single unified chain fetch service. #[derive(Clone)] @@ -983,3 +991,523 @@ pub(crate) async fn handle_raw_transaction( } } } + +/// Maps a Zebra network to the `zcash_protocol` network type used for address decoding. +fn address_network_type( + network: &zebra_chain::parameters::Network, +) -> zcash_protocol::consensus::NetworkType { + use zcash_protocol::consensus::NetworkType; + use zebra_chain::parameters::NetworkKind; + match network.kind() { + NetworkKind::Mainnet => NetworkType::Main, + NetworkKind::Testnet => NetworkType::Test, + NetworkKind::Regtest => NetworkType::Regtest, + } +} + +/// Validates a Zcash address for the `validateaddress` RPC. +/// +/// Pure address parsing over `network`; no chain data required, so both backends share +/// this implementation. +pub(crate) fn validate_address( + raw_address: String, + network: &zebra_chain::parameters::Network, +) -> ValidateAddressResponse { + use zcash_keys::address::Address; + use zcash_transparent::address::TransparentAddress; + + let Ok(address) = raw_address.parse::() else { + return ValidateAddressResponse::invalid(); + }; + + let address = match address.convert_if_network::
(address_network_type(network)) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return ValidateAddressResponse::invalid(); + } + }; + + match address { + Address::Transparent(taddr) => ValidateAddressResponse::new( + true, + Some(raw_address), + Some(matches!(taddr, TransparentAddress::ScriptHash(_))), + ), + _ => ValidateAddressResponse::invalid(), + } +} + +/// Validates a Zcash address for the deprecated `z_validateaddress` RPC. +/// +/// Pure address parsing over `network`; shared by both backends. +pub(crate) fn z_validate_address( + address: String, + network: &zebra_chain::parameters::Network, +) -> ZValidateAddressResponse { + use zcash_keys::address::Address; + use zcash_keys::encoding::AddressCodec as _; + use zcash_transparent::address::TransparentAddress; + + tracing::warn!("{}", Z_VALIDATE_DEPRECATION); + + let invalid = || { + ZValidateAddressResponse::Known(KnownZValidateAddress::Invalid( + InvalidZValidateAddress::new(), + )) + }; + + let Ok(parsed_address) = address.parse::() else { + return invalid(); + }; + + let converted_address = + match parsed_address.convert_if_network::
(address_network_type(network)) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return invalid(); + } + }; + + // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. + match converted_address { + Address::Transparent(TransparentAddress::PublicKeyHash(_)) => { + ZValidateAddressResponse::p2pkh(address) + } + Address::Transparent(TransparentAddress::ScriptHash(_)) => { + ZValidateAddressResponse::p2sh(address) + } + Address::Unified(u) => ZValidateAddressResponse::unified(u.encode(network)), + Address::Sapling(s) => { + let (diversifier, pk_d) = sapling_key_bytes(&s); + ZValidateAddressResponse::sapling( + s.encode(network), + Some(hex::encode(diversifier)), + Some(hex::encode(pk_d)), + ) + } + _ => invalid(), + } +} + +/// Extracts the diversifier and pk_d bytes from a validated Sapling +/// [`sapling_crypto::PaymentAddress`], returning pk_d in zcashd's big-endian byte order. +/// +/// # Deprecation +/// +/// See [`Z_VALIDATE_DEPRECATION`]. This function exists to support the `z_validateaddress` +/// RPC endpoint, which itself exists solely for zcashd compatibility. The pk_d bytes are +/// reversed from `sapling-crypto`'s native little-endian representation to match zcashd's +/// big-endian hex output. +/// +/// # Precondition +/// +/// The caller must have obtained `s` through `PaymentAddress::from_bytes` or equivalent +/// (e.g. `ZcashAddress::convert_if_network`), which guarantees the diversifier has a valid +/// `g_d()` and pk_d is a non-identity Jubjub subgroup point. No additional validation is +/// performed here. +/// +/// # Layout +/// +/// `PaymentAddress::to_bytes()` returns 43 bytes: `diversifier (11) || pk_d (32)`. +pub(crate) fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11], [u8; 32]) { + let bytes = s.to_bytes(); + let diversifier: [u8; 11] = bytes[..11] + .try_into() + .expect("PaymentAddress::to_bytes always returns 43 bytes: diversifier is the first 11"); + let mut pk_d: [u8; 32] = bytes[11..] + .try_into() + .expect("PaymentAddress::to_bytes always returns 43 bytes: pk_d is the last 32"); + pk_d.reverse(); + (diversifier, pk_d) +} + +/// Shapes a `z_getsubtreesbyindex` JSON-RPC response from raw subtree roots. +/// +/// The `(root, end_height)` pairs from [`ChainIndex::get_subtree_roots`] are already in +/// the byte order the JSON-RPC uses (sapling `to_bytes`, orchard `to_repr` — zcashd's +/// `z_getsubtreesbyindex` does not reverse orchard subtree roots), so they are hex-encoded +/// as-is. Shared by both backends so the shaping lives in one place. +/// +/// [`ChainIndex::get_subtree_roots`]: crate::ChainIndex::get_subtree_roots +pub(crate) fn build_subtrees_by_index_response( + pool: String, + start_index: NoteCommitmentSubtreeIndex, + roots: Vec<([u8; 32], u32)>, +) -> GetSubtreesByIndexResponse { + use hex::ToHex as _; + + let subtrees = roots + .into_iter() + .map(|(root, end_height)| { + SubtreeRpcData { + root: root.encode_hex(), + end_height: Height(end_height), + } + .into() + }) + .collect(); + + GetSubtreesResponse { + pool, + start_index, + subtrees, + } + .into() +} + +/// Builds the gRPC [`TreeState`](zaino_proto::proto::service::TreeState) from a +/// `z_gettreestate` response: hex-encoded per-pool final states (the ironwood field is +/// the empty string below NU6.3 activation, matching lightwalletd behaviour). +fn tree_state_from_treestate_response( + network: String, + treestate_response: zebra_rpc::client::GetTreestateResponse, +) -> zaino_proto::proto::service::TreeState { + let sapling_tree = hex::encode( + treestate_response + .sapling() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let orchard_tree = hex::encode( + treestate_response + .orchard() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let ironwood_tree = treestate_response + .ironwood() + .clone() + .and_then(|treestate| treestate.commitments().final_state().clone()) + .map(hex::encode) + .unwrap_or_default(); + + zaino_proto::proto::service::TreeState { + network, + height: treestate_response.height().0 as u64, + hash: treestate_response.hash().to_string(), + time: treestate_response.time(), + sapling_tree, + orchard_tree, + ironwood_tree, + } +} + +/// Builds the `z_gettreestate` response from the per-pool treestates the chain index +/// reported. +/// +/// `Commitments::new(final_root, final_state)`: the note-commitment tree is the +/// `final_state`. The ironwood treestate is `Some` only from NU6.3 activation, so +/// pre-NU6.3 responses omit the field exactly as zebrad does. +fn build_treestate_response( + hash: zebra_chain::block::Hash, + height: zebra_chain::block::Height, + time: u32, + (sapling, orchard, ironwood): ( + Option, + Option, + Option, + ), +) -> zebra_rpc::client::GetTreestateResponse { + fn treestate( + pool: Option, + ) -> zebra_rpc::client::Treestate { + let (final_root, final_state) = match pool { + Some(pool) => (pool.final_root, Some(pool.final_state)), + None => (None, None), + }; + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + final_root, + final_state, + )) + } + + let sprout_treestate = None; + let ironwood_treestate = ironwood.map(|pool| treestate(Some(pool))); + zebra_rpc::client::GetTreestateResponse::new( + hash, + height, + time, + sprout_treestate, + treestate(sapling), + treestate(orchard), + ironwood_treestate, + ) +} + +fn latest_network_upgrade( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, +) -> Result<&zebra_rpc::methods::NetworkUpgradeInfo, tonic::Status> { + upgrades.last().map(|(_, upgrade)| upgrade).ok_or_else(|| { + tonic::Status::failed_precondition("validator returned no network upgrade metadata") + }) +} + +/// Maximum number of addresses a single `get_address_utxos` / `get_address_utxos_stream` +/// request may carry. +/// +/// The service resolves the full backend UTXO set before applying `max_entries` / +/// `start_height` (issue #974). A complete pushdown fix needs upstream interface changes +/// the caller-supplied entry cap cannot reach today, so until then this bounds the one +/// input the service controls locally: the address fan-out. It stops an unauthenticated +/// caller forcing an unbounded number of backend address lookups in a single request, and +/// is set well above realistic wallet usage. +/// +/// TODO: make this deployment-configurable rather than a fixed constant. +const UTXO_MAX_ADDRESSES: usize = 1000; + +/// Reject a `get_address_utxos` request whose address list exceeds [`UTXO_MAX_ADDRESSES`]. +/// +/// `max_entries` bounds the response size, not the backend work; this guard bounds the +/// address fan-out, the part the service can cap without upstream changes. +fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { + if count > UTXO_MAX_ADDRESSES { + return Err(tonic::Status::invalid_argument(format!( + "Error: too many addresses in request: {count} exceeds the maximum of {UTXO_MAX_ADDRESSES}." + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn latest_network_upgrade_rejects_empty_metadata() { + let upgrades = indexmap::IndexMap::new(); + let err = super::latest_network_upgrade(&upgrades).expect_err("empty upgrades must fail"); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert_eq!( + err.message(), + "validator returned no network upgrade metadata" + ); + } + + #[test] + fn utxo_address_count_within_limit_is_accepted() { + assert!(super::validate_utxo_address_count(0).is_ok()); + assert!(super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES).is_ok()); + } + + #[test] + fn utxo_address_count_over_limit_is_rejected() { + let err = super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES + 1) + .expect_err("over-limit address count must fail"); + + assert_eq!(err.code(), tonic::Code::InvalidArgument); + } + + #[test] + fn build_subtrees_by_index_response_hex_encodes_roots() { + let roots = vec![([0xabu8; 32], 100u32), ([0xcdu8; 32], 200u32)]; + let response = build_subtrees_by_index_response( + "orchard".to_string(), + NoteCommitmentSubtreeIndex(5), + roots, + ); + + assert_eq!(response.pool().as_str(), "orchard"); + assert_eq!(response.start_index(), NoteCommitmentSubtreeIndex(5)); + + let subtrees = response.subtrees(); + assert_eq!(subtrees.len(), 2); + assert_eq!(subtrees[0].root, hex::encode([0xabu8; 32])); + assert_eq!(subtrees[0].end_height, Height(100)); + assert_eq!(subtrees[1].root, hex::encode([0xcdu8; 32])); + assert_eq!(subtrees[1].end_height, Height(200)); + } + + /// Classifies the byte-level relationship between two slices. + #[derive(Debug, PartialEq)] + enum ByteRelation { + /// The slices are identical. + Equal, + /// `actual` fully byte-reversed equals `expected` (endian swap). + FullByteReversal, + /// Each byte's bits reversed maps `actual` to `expected`. + PerByteBitReversal, + /// Reversing bytes within 16-bit chunks maps `actual` to `expected`. + ChunkSwap16, + /// Reversing bytes within 32-bit chunks maps `actual` to `expected`. + ChunkSwap32, + /// Reversing bytes within 64-bit chunks maps `actual` to `expected`. + ChunkSwap64, + /// No recognized transformation. + Unrecognized, + } + + impl std::fmt::Display for ByteRelation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Equal => write!(f, "equal"), + Self::FullByteReversal => write!(f, "full byte-reversal (endian swap)"), + Self::PerByteBitReversal => write!(f, "per-byte bit-reversal"), + Self::ChunkSwap16 => write!(f, "16-bit pairwise byte-swap"), + Self::ChunkSwap32 => write!(f, "32-bit chunk byte-reversal"), + Self::ChunkSwap64 => write!(f, "64-bit chunk byte-reversal"), + Self::Unrecognized => write!(f, "unrecognized mismatch"), + } + } + } + + /// Applies each candidate byte transformation to `actual` and returns + /// the first that produces `expected`, or [`ByteRelation::Unrecognized`]. + // `u32::is_multiple_of` is only stable from Rust 1.87; keep `% n == 0` for our older MSRV. + #[allow(clippy::manual_is_multiple_of)] + fn classify_byte_relation(actual: &[u8], expected: &[u8]) -> ByteRelation { + if actual == expected { + return ByteRelation::Equal; + } + + let chunk_swap = |size: usize| -> Vec { + actual + .chunks(size) + .flat_map(|c| c.iter().rev()) + .copied() + .collect() + }; + + let mut reversed = actual.to_vec(); + reversed.reverse(); + if reversed == expected { + return ByteRelation::FullByteReversal; + } + + let bit_reversed: Vec = actual.iter().map(|b| b.reverse_bits()).collect(); + if bit_reversed == expected { + return ByteRelation::PerByteBitReversal; + } + + if actual.len() % 2 == 0 && chunk_swap(2) == expected { + return ByteRelation::ChunkSwap16; + } + if actual.len() % 4 == 0 && chunk_swap(4) == expected { + return ByteRelation::ChunkSwap32; + } + if actual.len() % 8 == 0 && chunk_swap(8) == expected { + return ByteRelation::ChunkSwap64; + } + + ByteRelation::Unrecognized + } + + /// Verifies that our Sapling address parsing logic produces the same + /// diversifier and diversified transmission key (pk_d) hex strings as + /// zcashd's `z_validateaddress` RPC. + /// + /// # Guarantees + /// + /// - Exercises the production `sapling_key_bytes` function directly. + /// - The 11-byte diversifier matches the zcashd-derived test vector. + /// - The 32-byte pk_d (after the endian reversal inside `sapling_key_bytes`) + /// matches the zcashd-derived test vector. + /// - If the upstream serialization changes, the failure message + /// classifies the mismatch (endian swap, bit-reversal, chunk swap, + /// or unrecognized) to aid diagnosis. + /// + /// # Non-guarantees + /// + /// - Does not prove the test vector constants themselves are correct; + /// they were captured from zcashd and are trusted as ground truth. + /// - Does not exercise the full `z_validate_address` RPC path through + /// `StateService` — only the `sapling_key_bytes` extraction function. + /// - Does not verify behavior for malformed Sapling addresses or + /// addresses on other networks (mainnet, testnet). + #[test] + fn sapling_pk_d_byte_order_matches_test_vector() { + use crate::indexer::sapling_key_bytes; + use zcash_keys::address::Address; + use zcash_protocol::consensus::NetworkType; + + // Canonical source: live-tests/clientless/src/lib.rs::rpc::json_rpc + // Tracked for DRY consolidation: https://github.com/zingolabs/zaino/issues/988 + const SAPLING_ADDRESS: &str = "zregtestsapling1jalqhycwumq3unfxlzyzcktq3n478n82k2wacvl8gwfxk6ahshkxmtp2034qj28n7gl92ka5wca"; + const EXPECTED_DIVERSIFIER: &str = "977e0b930ee6c11e4d26f8"; + const EXPECTED_PK_D: &str = + "553ef2f328096a7c2aac6dec85b76b6b9243e733dc9db2eacce3eb8c60592c88"; + + let parsed: zcash_address::ZcashAddress = SAPLING_ADDRESS.parse().unwrap(); + let converted = parsed + .convert_if_network::
(NetworkType::Regtest) + .unwrap(); + + let Address::Sapling(s) = converted else { + panic!("expected Sapling address"); + }; + + let (diversifier, pk_d) = sapling_key_bytes(&s); + + let expected_diversifier = hex::decode(EXPECTED_DIVERSIFIER).unwrap(); + let expected_pk_d = hex::decode(EXPECTED_PK_D).unwrap(); + + // Diversifier + match classify_byte_relation(&diversifier, &expected_diversifier) { + ByteRelation::Equal => {} + relation => panic!( + "diversifier mismatch.\n relation: {relation}\n actual: {}\n expected: {}", + hex::encode(diversifier), + hex::encode(expected_diversifier), + ), + } + + // pk_d (sapling_key_bytes already applies the endian reversal) + match classify_byte_relation(&pk_d, &expected_pk_d) { + ByteRelation::Equal => {} + relation => panic!( + "pk_d mismatch — upstream serialization may have changed.\ + \n relation: {relation}\n actual: {}\n expected: {}", + hex::encode(pk_d), + hex::encode(expected_pk_d), + ), + } + } + + #[test] + fn classify_byte_relation_detects_known_transforms() { + let original = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + + assert_eq!( + classify_byte_relation(&original, &original), + ByteRelation::Equal, + ); + + let mut reversed = original.to_vec(); + reversed.reverse(); + assert_eq!( + classify_byte_relation(&original, &reversed), + ByteRelation::FullByteReversal, + ); + + let bit_rev: Vec = original.iter().map(|b| b.reverse_bits()).collect(); + assert_eq!( + classify_byte_relation(&original, &bit_rev), + ByteRelation::PerByteBitReversal, + ); + + let swapped_16: Vec = original + .chunks(2) + .flat_map(|c| c.iter().rev()) + .copied() + .collect(); + assert_eq!( + classify_byte_relation(&original, &swapped_16), + ByteRelation::ChunkSwap16, + ); + + let garbage = [0xFF; 8]; + assert_eq!( + classify_byte_relation(&original, &garbage), + ByteRelation::Unrecognized, + ); + } +} diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs similarity index 77% rename from packages/zaino-state/src/backends/fetch.rs rename to packages/zaino-state/src/indexer/node_backed_indexer.rs index 058004802..0aeaa446d 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -24,8 +24,7 @@ use zebra_rpc::{ use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ - connector::{JsonRpSeeConnector, RpcError}, - raw_transaction::validate_raw_transaction_hex, + connector::RpcError, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -34,9 +33,7 @@ use zaino_fetch::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::{ - ZValidateAddressResponse, DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, - }, + z_validate_address::ZValidateAddressResponse, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutResponse, GetTxOutSetInfoResponse, }, @@ -60,9 +57,15 @@ use zaino_proto::proto::{ #[allow(deprecated)] use crate::{ chain_index::chain_tips_from_nonfinalized_snapshot, - chain_index::{source::ValidatorConnector, types}, - config::{DonationAddress, FetchServiceConfig}, - error::FetchServiceError, + chain_index::{ + source::{BlockchainSource, ValidatorConnector}, + types, + }, + config::{ + ChainIndexConfig, CommonBackendConfig, DonationAddress, NodeBackedIndexerServiceConfig, + ValidatorConnectionType, + }, + error::NodeBackedIndexerServiceError, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, }, @@ -72,96 +75,123 @@ use crate::{ UtxoReplyStream, }, utils::{get_build_info, ServiceMetadata}, - BackendType, }; use crate::{ chain_index::{non_finalised_state::ChainIndexSnapshot, NonFinalizedSnapshot}, - ChainIndex, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, }; -/// Chain fetch service backed by Zcashd's JsonRPC engine. +/// A single node-backed chain-fetch + tx-submission service, generic over its +/// [`BlockchainSource`]. /// -/// This service is a central service, [`FetchServiceSubscriber`] should be created to fetch data. -/// This is done to enable large numbers of concurrent subscribers without significant slowdowns. +/// Replaces the former `FetchService` / `StateService` split: the JSON-RPC (`Rpc`) and +/// direct-`ReadStateService` (`Direct`) backends are now one type selected at runtime +/// via [`NodeBackedIndexerServiceConfig`]. Production instantiates the default +/// `Source = ValidatorConnector` (which itself carries the `Rpc`/`Direct` arm); tests +/// may instantiate over a mock source. /// -/// NOTE: We currently do not implement clone for chain fetch services as this service is responsible for maintaining and closing its child processes. -/// ServiceSubscribers are used to create separate chain fetch processes while allowing central state processes to be managed in a single place. -/// If we want the ability to clone Service all JoinHandle's should be converted to Arc\. +/// This service is a central service — create a [`NodeBackedIndexerServiceSubscriber`] +/// (via [`ZcashService::get_subscriber`]) to fetch data. This lets large numbers of +/// concurrent subscribers run without contending on the single central process. +/// +/// NOTE: We do not implement `Clone` for the central service: it owns and closes its +/// child processes. Subscribers are the clone-safe read handles. #[derive(Debug)] -#[deprecated = "Will be eventually replaced by `BlockchainSource`"] -pub struct FetchService { - /// JsonRPC Client. - /// - /// NOTE: DEPRCATED, USE INDEXER OR VALIDATOR_CONNECTOR. - fetcher: JsonRpSeeConnector, +pub struct NodeBackedIndexerService { /// Core indexer. - indexer: NodeBackedChainIndex, + indexer: NodeBackedChainIndex, /// Service metadata. data: ServiceMetadata, - - /// StateService config data. - #[allow(deprecated)] - config: FetchServiceConfig, + /// Connection-independent config (validator RPC, storage, network, ...). + config: CommonBackendConfig, } -#[allow(deprecated)] -impl Status for FetchService { +impl Status for NodeBackedIndexerService { fn status(&self) -> StatusType { self.indexer.status() } } -#[allow(deprecated)] -impl ZcashService for FetchService { - const BACKEND_TYPE: BackendType = BackendType::Fetch; +impl NodeBackedIndexerService { + /// Tears down the indexer (sync loop, finalised DB, mempool, and any source-owned + /// syncer task) from a synchronous context. Shared by [`ZcashService::close`] and + /// [`Drop`]. + /// + /// `block_in_place` is only legal on a multi-thread runtime; on a + /// current-thread runtime or a thread with no runtime it panics, and a + /// panic inside `Drop` during unwind aborts the process. Those contexts + /// fall back to the synchronous best-effort teardown, which skips only + /// the finalised DB's async shutdown step (the DB's own `Drop` still + /// releases its resources). + fn shutdown_blocking(&self) { + use tokio::runtime::{Handle, RuntimeFlavor}; + + match Handle::try_current() { + Ok(handle) if handle.runtime_flavor() == RuntimeFlavor::MultiThread => { + tokio::task::block_in_place(|| { + handle.block_on(async { + let _ = self.indexer.shutdown().await; + }); + }); + } + _ => self.indexer.shutdown_sync_best_effort(), + } + } +} - type Subscriber = FetchServiceSubscriber; - type Config = FetchServiceConfig; +impl ZcashService for NodeBackedIndexerService { + type Subscriber = NodeBackedIndexerServiceSubscriber; + type Config = NodeBackedIndexerServiceConfig; - /// Initializes a new FetchService instance and starts sync process. - #[instrument(name = "FetchService::spawn", skip(config), fields(network = %config.common.network))] - async fn spawn(config: FetchServiceConfig) -> Result { + /// Initializes a new [`NodeBackedIndexerService`] and starts its sync process. + #[instrument(name = "NodeBackedIndexerService::spawn", skip(config), fields(network = %config.common.network))] + async fn spawn( + config: NodeBackedIndexerServiceConfig, + ) -> Result { info!( rpc_address = %config.common.validator_rpc_address, network = %config.common.network, - "Launching Fetch Service" + "Launching NodeBackedIndexerService" ); - let fetcher = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), - ) - .await?; + // Select the validator connection from config; both arms return the built source, + // the validator `getinfo` used for service metadata, and the runtime network + // (activation schedule adopted from the validator at first contact, zaino#1076). + let (source, zebra_build_data, network) = match &config.connection { + ValidatorConnectionType::Rpc => ValidatorConnector::spawn_fetch(&config.common).await, + ValidatorConnectionType::Direct(direct) => { + ValidatorConnector::spawn_state(&config.common, direct).await + } + } + .map_err(|error| NodeBackedIndexerServiceError::Critical(error.to_string()))?; - let zebra_build_data = fetcher.get_info().await?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), + network.clone(), zebra_build_data.build, zebra_build_data.subversion, ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - let source = ValidatorConnector::Fetch(fetcher.clone()); - let indexer = NodeBackedChainIndex::new(source, config.clone().into()) - .await - .unwrap(); + let indexer = NodeBackedChainIndex::new( + source, + ChainIndexConfig::from_backend_config(&config.common, network), + ) + .await + .map_err(|error| NodeBackedIndexerServiceError::Critical(error.to_string()))?; - let fetch_service = Self { - fetcher, + let service = Self { indexer, data, - config, + config: config.common, }; // wait for sync to complete, return error on sync fail. loop { - match fetch_service.status() { + match service.status() { StatusType::Ready | StatusType::Closing => break, StatusType::CriticalError => { - return Err(FetchServiceError::Critical( + return Err(NodeBackedIndexerServiceError::Critical( "ChainIndex initial sync failed, check full log for details.".to_string(), )); } @@ -171,78 +201,210 @@ impl ZcashService for FetchService { } } - Ok(fetch_service) + Ok(service) } - /// Returns a [`FetchServiceSubscriber`]. - fn get_subscriber(&self) -> IndexerSubscriber { - IndexerSubscriber::new(FetchServiceSubscriber { - fetcher: self.fetcher.clone(), + /// Returns a [`NodeBackedIndexerServiceSubscriber`]. + fn get_subscriber( + &self, + ) -> IndexerSubscriber> { + IndexerSubscriber::new(NodeBackedIndexerServiceSubscriber { indexer: self.indexer.subscriber(), data: self.data.clone(), config: self.config.clone(), }) } - /// Shuts down the StateService. + /// Shuts down the service, tearing down its indexer (sync loop, finalised DB, + /// mempool, and any source-owned syncer task). fn close(&mut self) { - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - let _ = self.indexer.shutdown().await; - }); - }); + self.shutdown_blocking(); } } -#[allow(deprecated)] -impl Drop for FetchService { +impl Drop for NodeBackedIndexerService { fn drop(&mut self) { - self.close() + self.shutdown_blocking(); } } -/// A fetch service subscriber. -/// -/// Subscribers should be +/// A clone-safe, read-only subscriber to a [`NodeBackedIndexerService`]. #[derive(Debug, Clone)] -#[allow(deprecated)] -pub struct FetchServiceSubscriber { - /// JsonRPC Client. - /// - /// NOTE: DEPRCATED, USE INDEXER OR VALIDATOR_CONNECTOR. - fetcher: JsonRpSeeConnector, +pub struct NodeBackedIndexerServiceSubscriber { /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, + pub indexer: NodeBackedChainIndexSubscriber, /// Service metadata. pub data: ServiceMetadata, - /// StateService config data. - #[allow(deprecated)] - config: FetchServiceConfig, + /// Connection-independent config (validator RPC, storage, network, ...). + config: CommonBackendConfig, } -impl Status for FetchServiceSubscriber { +impl Status for NodeBackedIndexerServiceSubscriber { fn status(&self) -> StatusType { self.indexer.status() } } -impl FetchServiceSubscriber { +impl NodeBackedIndexerServiceSubscriber { /// Fetches the current status #[deprecated(note = "Use the Status trait method instead")] pub fn get_status(&self) -> StatusType { self.indexer.status() } - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network + /// Returns the runtime network, carrying the activation schedule + /// adopted from the validator (zaino#1076). + pub fn network(&self) -> zebra_chain::parameters::Network { + self.data.network() } } -impl ZcashIndexer for FetchServiceSubscriber { - #[allow(deprecated)] - type Error = FetchServiceError; +/// `getchaintips` served from the non-finalised snapshot when it exists, +/// falling back to the validator's own response during the initial +/// finalised-state build — matching both pre-merge backends, which proxied +/// the validator for that window instead of erroring. +pub(crate) async fn chain_tips_for_snapshot( + snapshot: &ChainIndexSnapshot, + source: &Source, +) -> Result { + match snapshot.get_nfs_snapshot() { + Some(non_finalized_snapshot) => Ok(chain_tips_from_nonfinalized_snapshot( + non_finalized_snapshot, + )), + None => Ok(source + .get_chain_tips() + .await + .map_err(crate::error::ChainIndexError::backing_validator)?), + } +} + +/// Placeholder metadata and config for test-only service construction. Takes the +/// runtime (zebra) network directly — tests are their own chain, so they stand in +/// for the validator the production path adopts the schedule from. +#[cfg(test)] +fn test_service_parts( + network: zebra_chain::parameters::Network, +) -> (ServiceMetadata, CommonBackendConfig) { + let network_kind = zaino_common::Network::from(network.clone()); + ( + ServiceMetadata::new( + get_build_info("test".to_string()), + network, + "test-build".to_string(), + "test-subversion".to_string(), + ), + CommonBackendConfig::new( + "127.0.0.1:0".to_string(), + None, + None, + None, + zaino_common::ServiceConfig::default(), + zaino_common::StorageConfig::default(), + true, + network_kind, + None, + ), + ) +} + +#[cfg(test)] +impl NodeBackedIndexerService { + /// Wraps a chain index in a service for tests, with placeholder + /// metadata/config. Lets unit tests exercise the service lifecycle over a + /// mock source (no real validator). Production builds go through + /// [`ZcashService::spawn`]. + pub(crate) fn new_for_test( + indexer: NodeBackedChainIndex, + network: zebra_chain::parameters::Network, + ) -> Self { + let (data, config) = test_service_parts(network); + Self { + data, + config, + indexer, + } + } +} + +#[cfg(test)] +impl NodeBackedIndexerServiceSubscriber { + /// Wraps a chain-index subscriber in a service subscriber for tests, with placeholder + /// metadata/config. Lets unit tests drive the service RPC layer over a mock source + /// (no real validator). Production builds go through [`ZcashService::get_subscriber`]. + pub(crate) fn new_for_test( + indexer: NodeBackedChainIndexSubscriber, + network: zebra_chain::parameters::Network, + ) -> Self { + let (data, config) = test_service_parts(network); + Self { + data, + config, + indexer, + } + } +} + +/// A subscriber to any chaintip updates. +#[derive(Clone)] +pub struct ChainTipSubscriber { + monitor: zebra_state::ChainTipChange, +} + +impl ChainTipSubscriber { + /// Waits until the tip hash has changed (relative to the last time this method + /// was called), then returns the best tip's block hash. + pub async fn next_tip_hash( + &mut self, + ) -> Result { + self.monitor + .wait_for_tip_change() + .await + .map(|tip| tip.best_tip_hash()) + } +} + +impl NodeBackedIndexerServiceSubscriber { + /// A subscriber to chain-tip updates, when the backing source exposes a + /// local tip-change stream. `Some` only on the `Direct` connection; the + /// `Rpc` connection (and any other stream-less source) observes tips by + /// polling the validator and yields `None`. + pub fn chaintip_update_subscriber(&self) -> Option { + Some(ChainTipSubscriber { + monitor: self.indexer.source().chain_tip_change()?, + }) + } +} + +/// Methods available only on the production (`ValidatorConnector`-backed) subscriber: +/// the `Direct` connection owns a `ReadStateService` that the generic source +/// abstraction does not expose. +impl NodeBackedIndexerServiceSubscriber { + /// The backing Zebra [`zebra_state::ReadStateService`] (`Direct` connection only). + /// + /// Test-only escape hatch: live tests recompute expected chain data directly off the + /// `ReadStateService`. Production code goes through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub fn read_state_service(&self) -> zebra_state::ReadStateService { + self.indexer + .source() + .read_state_service() + .expect("read_state_service requires the Direct connection") + .clone() + } + + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` values + /// directly off the mempool's entries. Production code goes through the `ChainIndex` + /// mempool API. + #[cfg(feature = "test_dependencies")] + pub fn mempool(&self) -> &crate::chain_index::mempool::MempoolSubscriber { + self.indexer.mempool_subscriber() + } +} + +impl ZcashIndexer for NodeBackedIndexerServiceSubscriber { + type Error = NodeBackedIndexerServiceError; /// Returns information about all changes to the given transparent addresses within the given inclusive block-height range. /// @@ -282,7 +444,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// in Zebra's [`GetInfo`]. Zebra uses the field names and formats from the /// [zcashd code](https://github.com/zcash/zcash/blob/v4.6.0-1/src/rpc/misc.cpp#L86-L87). async fn get_info(&self) -> Result { - Ok(self.fetcher.get_info().await?.into()) + Ok(self.indexer.get_info().await?) } /// Returns blockchain state information, as a [`GetBlockchainInfoResponse`] JSON struct. @@ -296,19 +458,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// Some fields from the zcashd reference are missing from Zebra's [`GetBlockchainInfoResponse`]. It only contains the fields /// [required for lightwalletd support.](https://github.com/zcash/lightwalletd/blob/v0.4.9/common/common.go#L72-L89) async fn get_blockchain_info(&self) -> Result { - Ok(self - .fetcher - .get_blockchain_info() - .await? - .try_into() - .map_err(|_e| { - #[allow(deprecated)] - FetchServiceError::SerializationError( - zebra_chain::serialization::SerializationError::Parse( - "chainwork not hex-encoded integer", - ), - ) - })?) + Ok(self.indexer.get_blockchain_info().await?) } /// Returns details on the active state of the TX memory pool. @@ -325,7 +475,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn get_peer_info(&self) -> Result { - Ok(self.fetcher.get_peer_info().await?) + Ok(self.indexer.get_peer_info().await?) } /// Returns the proof-of-work difficulty as a multiple of the minimum difficulty. @@ -334,11 +484,11 @@ impl ZcashIndexer for FetchServiceSubscriber { /// method: post /// tags: blockchain async fn get_difficulty(&self) -> Result { - Ok(self.fetcher.get_difficulty().await?.0) + Ok(self.indexer.get_difficulty().await?) } async fn get_block_subsidy(&self, height: u32) -> Result { - Ok(self.fetcher.get_block_subsidy(height).await?) + Ok(self.indexer.get_block_subsidy(height).await?) } /// Returns the total balance of a provided `addresses` in an [`AddressBalance`] instance. @@ -389,12 +539,10 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, raw_transaction_hex: String, ) -> Result { - validate_raw_transaction_hex(&raw_transaction_hex)?; Ok(self - .fetcher + .indexer .send_raw_transaction(raw_transaction_hex) - .await? - .into()) + .await?) } /// Returns the requested block by hash or height, as a [`GetBlock`] JSON string. @@ -426,11 +574,7 @@ impl ZcashIndexer for FetchServiceSubscriber { hash_or_height: String, verbosity: Option, ) -> Result { - Ok(self - .fetcher - .get_block(hash_or_height, verbosity) - .await? - .try_into()?) + Ok(self.indexer.z_get_block(hash_or_height, verbosity).await?) } /// Returns information about the given block and its transactions. @@ -441,7 +585,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// /// Note: This method has only been implemented in `zcashd`. Zebra has no intention of supporting it. async fn get_block_deltas(&self, hash: String) -> Result { - Ok(self.fetcher.get_block_deltas(hash).await?) + Ok(self.indexer.get_block_deltas(hash).await?) } async fn get_block_header( @@ -449,11 +593,11 @@ impl ZcashIndexer for FetchServiceSubscriber { hash: String, verbose: bool, ) -> Result { - Ok(self.fetcher.get_block_header(hash, verbose).await?) + Ok(self.indexer.get_block_header(hash, verbose).await?) } async fn get_mining_info(&self) -> Result { - Ok(self.fetcher.get_mining_info().await?) + Ok(self.indexer.get_mining_info().await?) } /// Returns statistics about the unspent transaction output set. @@ -485,7 +629,9 @@ impl ZcashIndexer for FetchServiceSubscriber { /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` async fn get_best_blockhash(&self) -> Result { - Ok(self.fetcher.get_best_blockhash().await?.into()) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(GetBlockHashResponse::new(tip.hash.into())) } /// Returns the current block count in the best valid block chain. @@ -494,18 +640,15 @@ impl ZcashIndexer for FetchServiceSubscriber { /// method: post /// tags: blockchain async fn get_block_count(&self) -> Result { - Ok(self.fetcher.get_block_count().await?.into()) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(tip.height.into()) } + #[allow(deprecated)] async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Ok(self.fetcher.get_chain_tips().await?); - }; - - Ok(chain_tips_from_nonfinalized_snapshot( - non_finalized_snapshot, - )) + chain_tips_for_snapshot(&snapshot, self.indexer.source()).await } /// Return information about the given Zcash address. @@ -520,7 +663,9 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, address: String, ) -> Result { - Ok(self.fetcher.validate_address(address).await?) + #[allow(deprecated)] + let network = self.data.network(); + Ok(crate::indexer::validate_address(address, &network)) } #[allow(deprecated)] @@ -528,8 +673,9 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, address: String, ) -> Result { - tracing::warn!("{}", Z_VALIDATE_DEPRECATION); - Ok(self.fetcher.z_validate_address(address).await?) + #[allow(deprecated)] + let network = self.data.network(); + Ok(crate::indexer::z_validate_address(address, &network)) } /// Returns all transaction ids in the memory pool, as a JSON array. @@ -568,7 +714,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// NOTE: This method currently has to fetch data from 2 places (get_treestate and get_indexed_block_by_*), /// If `ValidatorConnector::GetTreeState` was updated to return the additional information /// required, this second call could be removed, improving the performance of this method. - // Pre-existing lint: `FetchServiceError` is a large error type; returning it by value here is + // Pre-existing lint: `NodeBackedIndexerServiceError` is a large error type; returning it by value here is // flagged by `result_large_err`. Suppressed to satisfy `-D warnings` without an invasive // boxing refactor of the shared error enum. #[allow(clippy::result_large_err)] @@ -588,7 +734,7 @@ impl ZcashIndexer for FetchServiceSubscriber { .await? .ok_or( #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch block data.", )), @@ -599,29 +745,27 @@ impl ZcashIndexer for FetchServiceSubscriber { .await? .ok_or( #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch block data.", )), )?, }; - let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; + let treestates = self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Block time is out of range for u32.", )) })?; - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( + Ok(super::build_treestate_response( (*block_data.hash()).into(), block_data.height().into(), time, - sapling, - orchard, + treestates, )) } .await; @@ -639,25 +783,10 @@ impl ZcashIndexer for FetchServiceSubscriber { return local_result; } - self.fetcher - .get_treestate(fallback_hash_or_height) - .await - .map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch treestate.", - )) - }) - .and_then(|treestate| { - treestate.try_into().map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to parse treestate.", - )) - }) - }) + Ok(self + .indexer + .get_treestate_by_id(fallback_hash_or_height) + .await?) } /// Returns information about a range of Sapling or Orchard subtrees. @@ -678,17 +807,36 @@ impl ZcashIndexer for FetchServiceSubscriber { /// starting at the chain tip. This RPC will return an empty list if the `start_index` subtree /// exists, but has not been rebuilt yet. This matches `zcashd`'s behaviour when subtrees aren't /// available yet. (But `zcashd` does its rebuild before syncing any blocks.) + #[allow(deprecated)] async fn z_get_subtrees_by_index( &self, pool: String, start_index: NoteCommitmentSubtreeIndex, limit: Option, ) -> Result { - Ok(self - .fetcher - .get_subtrees_by_index(pool, start_index.0, limit.map(|limit_index| limit_index.0)) - .await? - .into()) + let shielded_pool = match pool.as_str() { + "sapling" => crate::chain_index::ShieldedPool::Sapling, + "orchard" => crate::chain_index::ShieldedPool::Orchard, + otherwise => { + return Err(NodeBackedIndexerServiceError::RpcError( + RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::Misc, + format!( + "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + ), + ), + )) + } + }; + let roots = self + .indexer + .get_subtree_roots(shielded_pool, start_index.0, limit.map(|index| index.0)) + .await?; + Ok(crate::indexer::build_subtrees_by_index_response( + pool, + start_index, + roots, + )) } /// Returns the raw transaction data, as a [`GetRawTransaction`] JSON string or structure. @@ -717,7 +865,7 @@ impl ZcashIndexer for FetchServiceSubscriber { ) -> Result { #[allow(deprecated)] let txid = types::TransactionHash::from_hex(&txid_hex).map_err(|error| { - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, error.to_string(), )) @@ -725,7 +873,7 @@ impl ZcashIndexer for FetchServiceSubscriber { #[allow(deprecated)] let not_found_error = || { - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, "No such mempool or main chain transaction", )) @@ -757,11 +905,12 @@ impl ZcashIndexer for FetchServiceSubscriber { let (height, confirmations, block_hash, in_best_chain) = match best_chain_location { Some(types::BestChainLocation::Block(block_hash, height)) => { - let confirmations = snapshot + let confirmations: i64 = snapshot .max_serviceable_height() .0 .saturating_sub(height.0) - .saturating_add(1); + .saturating_add(1) + .into(); ( Some(zebra_chain::block::Height::from(height)), @@ -780,7 +929,7 @@ impl ZcashIndexer for FetchServiceSubscriber { height, confirmations, #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), + &self.data.network(), None, block_hash, in_best_chain, @@ -800,24 +949,19 @@ impl ZcashIndexer for FetchServiceSubscriber { n: u32, include_mempool: Option, ) -> Result { - Ok(self.fetcher.get_tx_out(txid, n, include_mempool).await?) + Ok(self.indexer.get_tx_out(txid, n, include_mempool).await?) } async fn get_spent_info( &self, request: GetSpentInfoRequest, ) -> Result { - Ok(self.fetcher.get_spent_info(request).await?) + Ok(self.indexer.get_spent_info(request).await?) } async fn chain_height(&self) -> Result { - Ok(Height( - self.indexer - .snapshot_nonfinalized_state() - .await? - .max_serviceable_height() - .0, - )) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + Ok(self.indexer.best_chaintip(&snapshot).await?.height.into()) } /// Returns the transaction ids made by the provided transparent addresses. /// @@ -888,12 +1032,12 @@ impl ZcashIndexer for FetchServiceSubscriber { blocks: Option, height: Option, ) -> Result { - Ok(self.fetcher.get_network_sol_ps(blocks, height).await?) + Ok(self.indexer.get_network_sol_ps(blocks, height).await?) } } #[allow(deprecated)] -impl LightWalletIndexer for FetchServiceSubscriber { +impl LightWalletIndexer for NodeBackedIndexerServiceSubscriber { /// Return the height of the tip of the best chain async fn get_latest_block(&self) -> Result { match self.indexer.snapshot_nonfinalized_state().await? { @@ -904,7 +1048,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of reporting // the genesis block - Err(FetchServiceError::UnavailableNotSyncedEnough) + Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough) } } // dbg!(&tip); @@ -913,7 +1057,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// Return the compact block corresponding to the given block identifier async fn get_block(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Failed to convert to u32.", )), )?; @@ -925,12 +1069,12 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self.indexer.get_block_height(&snapshot, hash.into()).await { Ok(Some(height)) => height.0, Ok(None) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + return Err(NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Hash not founf in chain", ))); } Err(_e) => { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::internal("Error: Internal db error."), )); } @@ -942,7 +1086,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; match self @@ -958,32 +1102,38 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(None) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), + )), + )) + } + _otherwise => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Failed to retrieve block from state."), + )), } } Err(e) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } _otherwise => // TODO: Hide server error from clients before release. Currently useful for dev purposes. { - Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!( + "Error: Failed to retrieve block from node. Server Error: {e}", + )), + )) } } } @@ -995,7 +1145,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// NOTE: Currently this only returns Orchard nullifiers to follow Lightwalletd functionality but Sapling could be added if required by wallets. async fn get_block_nullifiers(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Failed to convert to u32.", )), )?; @@ -1006,12 +1156,12 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self.indexer.get_block_height(&snapshot, hash.into()).await { Ok(Some(height)) => height.0, Ok(None) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + return Err(NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Hash not founf in chain", ))); } Err(_e) => { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::internal("Error: Internal db error."), )); } @@ -1022,7 +1172,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; match self .indexer @@ -1037,40 +1187,44 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(None) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } HashOrHeight::Height(height) if height > self.data.network().sapling_activation_height() => { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::out_of_range(format!( "Error: Height out of range [{hash_or_height}]. Height requested \ is below sapling activation height [{chain_height}].", )), )) } - _otherwise => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), + _otherwise => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Failed to retrieve block from state."), + )), } } Err(e) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } HashOrHeight::Height(height) if height > self.data.network().sapling_activation_height() => { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::out_of_range(format!( "Error: Height out of range [{hash_or_height}]. Height requested \ is below sapling activation height [{chain_height}].", @@ -1080,9 +1234,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { _otherwise => // TODO: Hide server error from clients before release. Currently useful for dev purposes. { - Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!( + "Error: Failed to retrieve block from node. Server Error: {e}", + )), + )) } } } @@ -1096,24 +1252,20 @@ impl LightWalletIndexer for FetchServiceSubscriber { request: BlockRange, ) -> Result { let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] let start = validated_request.start() as u32; let end = validated_request.end() as u32; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = fetch_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); + let snapshot = service_clone.indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout_result = timeout( @@ -1136,7 +1288,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // Use the snapshot tip directly, as this function doesn't support passthrough let chain_height = non_finalized_snapshot.best_tip.height.0; - match fetch_service_clone + match service_clone .indexer .get_compact_block_stream( &snapshot, @@ -1229,24 +1381,20 @@ impl LightWalletIndexer for FetchServiceSubscriber { request: BlockRange, ) -> Result { let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] let start = validated_request.start() as u32; let end = validated_request.end() as u32; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = fetch_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); + let snapshot = service_clone.indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout_result = timeout( @@ -1270,7 +1418,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // Use the snapshot tip directly, as this function doesn't support passthrough let chain_height = non_finalized_snapshot.best_tip.height.0; - match fetch_service_clone + match service_clone .indexer .get_compact_block_stream( &snapshot, @@ -1380,7 +1528,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { let (hex, height) = if let GetRawTransaction::Object(tx_object) = tx { (tx_object.hex().clone(), tx_object.height()) } else { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::not_found("Error: Transaction not received"), )); }; @@ -1393,7 +1541,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; non_finalized_snapshot.best_tip.height.0 as u64 } @@ -1404,7 +1552,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { height, }) } else { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::invalid_argument("Error: Transaction hash incorrect"), )) } @@ -1428,17 +1576,15 @@ impl LightWalletIndexer for FetchServiceSubscriber { ) -> Result { let chain_height = self.chain_height().await?; let txids = self.get_taddress_txids_helper(request).await?; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (transmitter, receiver) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (transmitter, receiver) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), async { for txid in txids { - let transaction = - fetch_service_clone.get_raw_transaction(txid, Some(1)).await; + let transaction = service_clone.get_raw_transaction(txid, Some(1)).await; if handle_raw_transaction::( chain_height.0 as u64, transaction, @@ -1485,9 +1631,9 @@ impl LightWalletIndexer for FetchServiceSubscriber { let checked_balance: i64 = match i64::try_from(balance.balance()) { Ok(balance) => balance, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Error converting balance from u64 to i64."), + )); } }; Ok(Balance { @@ -1501,10 +1647,10 @@ impl LightWalletIndexer for FetchServiceSubscriber { &self, mut request: AddressStream, ) -> Result { - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; let (channel_tx, mut channel_rx) = - mpsc::channel::(self.config.common.service.channel_size as usize); + mpsc::channel::(self.config.service.channel_size as usize); let fetcher_task_handle = tokio::spawn(async move { let fetcher_timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), @@ -1514,8 +1660,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { match channel_rx.recv().await { Some(taddr) => { let taddrs = GetAddressBalanceRequest::new(vec![taddr]); - let balance = - fetch_service_clone.z_get_address_balance(taddrs).await?; + let balance = service_clone.z_get_address_balance(taddrs).await?; total_balance += balance.balance(); } None => { @@ -1563,11 +1708,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(Ok(())) => {} Ok(Err(e)) => { fetcher_task_handle.abort(); - return Err(FetchServiceError::TonicStatusError(e)); + return Err(NodeBackedIndexerServiceError::TonicStatusError(e)); } Err(_) => { fetcher_task_handle.abort(); - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::deadline_exceeded( "Error: get_taddress_balance_stream request timed out in address loop.", ), @@ -1581,21 +1726,23 @@ impl LightWalletIndexer for FetchServiceSubscriber { Err(_) => { // TODO: Hide server error from clients before release. // Currently useful for dev purposes. - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Error converting balance from u64 to i64.", + ), + )); } }; Ok(Balance { value_zat: checked_balance, }) } - Ok(Err(e)) => Err(FetchServiceError::TonicStatusError(e)), + Ok(Err(e)) => Err(NodeBackedIndexerServiceError::TonicStatusError(e)), // TODO: Hide server error from clients before release. // Currently useful for dev purposes. - Err(e) => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Fetcher Task failed: {e}"), - ))), + Err(e) => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!("Fetcher Task failed: {e}")), + )), } } @@ -1620,7 +1767,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { for (i, excluded_id) in request.exclude_txid_suffixes.iter().enumerate() { if excluded_id.len() > 32 { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::invalid_argument(format!( "Error: excluded txid {} is larger than 32 bytes", i @@ -1636,9 +1783,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { } let mempool = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( @@ -1737,9 +1883,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { #[allow(deprecated)] async fn get_mempool_stream(&self) -> Result { let indexer = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); let snapshot = indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout = timeout( @@ -1820,32 +1965,18 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// The block can be specified by either height or hash. async fn get_tree_state(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - crate::error::FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Invalid hash or height", - )), + crate::error::NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::invalid_argument("Invalid hash or height"), + ), )?; - #[allow(deprecated)] - let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); - Ok(TreeState { - network: self - .config - .common - .network - .to_zebra_network() - .bip70_network_name(), - height: height.0 as u64, - hash: hash.to_string(), - time, - sapling_tree: sapling.map(hex::encode).unwrap_or_default(), - orchard_tree: orchard.map(hex::encode).unwrap_or_default(), - }) + let treestate_response = + ::z_get_treestate(self, hash_or_height.to_string()).await?; + + Ok(super::tree_state_from_treestate_response( + self.data.network().bip70_network_name(), + treestate_response, + )) } /// GetLatestTreeState returns the note commitment tree state corresponding to the chain tip. @@ -1861,8 +1992,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { #[allow(deprecated)] fn timeout_channel_size(&self) -> (u32, u32) { ( - self.config.common.service.timeout, - self.config.common.service.channel_size, + self.config.service.timeout, + self.config.service.channel_size, ) } @@ -1893,17 +2024,21 @@ impl LightWalletIndexer for FetchServiceSubscriber { let checked_index = match i32::try_from(output_index.index()) { Ok(index) => index, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Index out of range. Failed to convert to i32.", + ), + )); } }; let checked_satoshis = match i64::try_from(satoshis) { Ok(satoshis) => satoshis, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Satoshis out of range. Failed to convert to i64.", + ), + )); } }; let utxo_reply = GetAddressUtxosReply { @@ -1933,9 +2068,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { super::validate_utxo_address_count(request.addresses.len())?; let taddrs = GetAddressBalanceRequest::new(request.addresses); let utxos = self.z_get_address_utxos(taddrs).await?; - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), @@ -2027,7 +2161,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { .to_string(); let latest_upgrade = super::latest_network_upgrade(blockchain_info.upgrades()) - .map_err(FetchServiceError::TonicStatusError)? + .map_err(NodeBackedIndexerServiceError::TonicStatusError)? .into_parts(); let nu_name = latest_upgrade.0; @@ -2050,14 +2184,13 @@ impl LightWalletIndexer for FetchServiceSubscriber { zcashd_subversion: self.data.zebra_subversion(), donation_address: self .config - .common .donation_address .as_ref() .map(DonationAddress::encode) .unwrap_or_default(), upgrade_name: nu_name.to_string(), upgrade_height: nu_height.0 as u64, - lightwallet_protocol_version: "v0.4.0".to_string(), + lightwallet_protocol_version: "v0.5.0".to_string(), }) } @@ -2065,7 +2198,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// /// NOTE: Currently unimplemented in Zaino. async fn ping(&self, _request: Duration) -> Result { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::unimplemented( "Ping not yet implemented. If you require this RPC \ please open an issue or PR at the Zaino github \ diff --git a/packages/zaino-state/src/lib.rs b/packages/zaino-state/src/lib.rs index 548c1da16..a2c0d67a6 100644 --- a/packages/zaino-state/src/lib.rs +++ b/packages/zaino-state/src/lib.rs @@ -56,18 +56,16 @@ pub use indexer::{ ZcashService, }; -pub(crate) mod backends; - -#[allow(deprecated)] -pub use backends::{ - fetch::{FetchService, FetchServiceSubscriber}, - state::{StateService, StateServiceSubscriber}, +pub use indexer::node_backed_indexer::{ + ChainTipSubscriber, NodeBackedIndexerService, NodeBackedIndexerServiceSubscriber, }; pub mod chain_index; // Core ChainIndex trait and implementations -pub use chain_index::{ChainIndex, NodeBackedChainIndex, NodeBackedChainIndexSubscriber}; +pub use chain_index::{ + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, +}; // Source types for ChainIndex backends pub use chain_index::source::{BlockchainSource, State, ValidatorConnector}; // Supporting types @@ -102,16 +100,14 @@ pub mod test_dependencies { pub(crate) mod config; -#[allow(deprecated)] pub use config::{ - BackendConfig, BackendType, ChainIndexConfig, CommonBackendConfig, DonationAddress, - FetchServiceConfig, StateServiceConfig, + ChainIndexConfig, CommonBackendConfig, DirectConnectionConfig, DonationAddress, + NodeBackedIndexerServiceConfig, ValidatorConnectionType, }; pub(crate) mod error; -#[allow(deprecated)] -pub use error::{FetchServiceError, StateServiceError}; +pub use error::NodeBackedIndexerServiceError; pub(crate) mod status; diff --git a/packages/zainod/CHANGELOG.md b/packages/zainod/CHANGELOG.md index df864132a..130408b7d 100644 --- a/packages/zainod/CHANGELOG.md +++ b/packages/zainod/CHANGELOG.md @@ -9,6 +9,11 @@ and this crate adheres to Rust's notion of ## [Unreleased] ### Added +- Ironwood (NU6.3) / V6 transaction support, end to end through the + workspace crates: V6 parsing and ironwood extraction (`zaino-fetch`), + `ironwoodActions` in served compact blocks (`zaino-proto`, on by + default), and ironwood treestate roots in the chain index + (`zaino-state`). - `[storage.database]` config gains `sync_checkpoint_interval` (seconds, default 120) — the bulk-sync write-batch flush interval, which also bounds the window of unflushed (`NO_SYNC`) writes at risk on a hard kill / eviction. @@ -16,6 +21,9 @@ and this crate adheres to Rust's notion of default 8) — a dedicated heap budget for the txout-set accumulator rebuild, separate from `sync_write_batch_size`. ### Changed +- Version marked `0.4.3-ironwood.1` so Ironwood feature builds identify + themselves at `zainod --version`; stock `0.4.2` binaries are otherwise + indistinguishable from this branch's builds. - **Breaking** — `[storage.database] sync_write_batch_bytes` (bytes) is renamed to `sync_write_batch_size` and is now given in **GiB** (default 8). It now budgets only the bulk-sync block buffer; the accumulator rebuild uses the new diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index b544e55ff..4e2c7d501 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zainod" -version = "0.5.1" +version = "0.6.0" description = "Crate containing the Zaino Indexer binary." authors = { workspace = true } repository = { workspace = true } @@ -70,7 +70,6 @@ zaino-state = { workspace = true } zaino-serve = { workspace = true } # Zebra -zebra-chain = { workspace = true } zebra-state = { workspace = true } # Runtime @@ -81,7 +80,6 @@ clap = { workspace = true, features = ["derive"] } # Tracing tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "time"] } # Network / RPC http = { workspace = true } @@ -97,8 +95,8 @@ thiserror = { workspace = true } # Formats toml = { workspace = true } config = { workspace = true } -tempfile = { workspace = true } [dev-dependencies] +tempfile = { workspace = true } zcash_address = { workspace = true } zcash_protocol = { workspace = true } diff --git a/packages/zainod/src/config.rs b/packages/zainod/src/config.rs index aeb97dbbe..889359c65 100644 --- a/packages/zainod/src/config.rs +++ b/packages/zainod/src/config.rs @@ -21,11 +21,32 @@ use zaino_common::{ try_resolve_address, AddressResolution, Network, ServiceConfig, StorageConfig, ValidatorConfig, }; use zaino_serve::server::config::{GrpcServerConfig, JsonRpcServerConfig}; -#[allow(deprecated)] use zaino_state::{ - BackendType, CommonBackendConfig, DonationAddress, FetchServiceConfig, StateServiceConfig, + CommonBackendConfig, DirectConnectionConfig, DonationAddress, NodeBackedIndexerServiceConfig, + ValidatorConnectionType, }; +/// On-disk selector for the validator connection (`backend = "direct" | "rpc"` in the +/// config file), mapped to [`zaino_state::ValidatorConnectionType`] at spawn. +/// +/// The legacy values `"state"` / `"fetch"` remain accepted as aliases for backward +/// compatibility with existing `zainod.toml` files. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum BackendType { + /// Direct Zebra `ReadStateService` access (formerly `state`). + /// + /// More efficient but requires running on the same machine as Zebra. + #[serde(alias = "state")] + Direct, + /// JSON-RPC access (formerly `fetch`). + /// + /// Compatible with Zcashd, Zebra, or another Zaino instance. + #[default] + #[serde(alias = "fetch")] + Rpc, +} + /// Header for generated configuration files. pub const GENERATED_CONFIG_HEADER: &str = r#"# Zaino Configuration # @@ -82,7 +103,8 @@ pub struct ZainodConfig { /// When enabled, Zaino does not use a persistent on-disk finalised-state database. Finalised /// state reads are served from the configured validator/source instead. pub ephemeral_finalised_state: bool, - /// Network to connect to (Mainnet, Testnet, or Regtest). + /// Network to connect to (Mainnet, PubTestnet — The Public Testnet — or Regtest; + /// `"Testnet"` is accepted as a legacy spelling of PubTestnet). pub network: Network, /// Prometheus metrics endpoint listen address. /// @@ -222,11 +244,6 @@ impl ZainodConfig { Ok(()) } - - /// Returns the network type currently being used by the server. - pub fn get_network(&self) -> Result { - Ok(self.network.to_zebra_network()) - } } impl Default for ZainodConfig { @@ -250,7 +267,7 @@ impl Default for ZainodConfig { storage: StorageConfig::default(), ephemeral_finalised_state: false, zebra_db_path: default_zebra_db_path(), - network: Network::Testnet, + network: Network::PubTestnet, donation_address: None, } } @@ -363,59 +380,56 @@ pub fn load_config_with_env( Ok(parsed_config) } -#[allow(deprecated)] -impl TryFrom for StateServiceConfig { +impl TryFrom for NodeBackedIndexerServiceConfig { type Error = IndexerError; fn try_from(cfg: ZainodConfig) -> Result { - let grpc_listen_address = cfg - .validator_settings - .validator_grpc_listen_address - .as_ref() - .ok_or_else(|| { - IndexerError::ConfigError( - "Missing validator_grpc_listen_address in configuration".to_string(), - ) - })?; - - let validator_grpc_address = - fetch_socket_addr_from_hostname(grpc_listen_address).map_err(|e| { - let msg = match e { - IndexerError::ConfigError(msg) => msg, - other => other.to_string(), + let connection = match cfg.backend { + BackendType::Rpc => ValidatorConnectionType::Rpc, + BackendType::Direct => { + let grpc_listen_address = cfg + .validator_settings + .validator_grpc_listen_address + .as_ref() + .ok_or_else(|| { + IndexerError::ConfigError( + "Missing validator_grpc_listen_address in configuration".to_string(), + ) + })?; + + let validator_grpc_address = fetch_socket_addr_from_hostname(grpc_listen_address) + .map_err(|e| { + let msg = match e { + IndexerError::ConfigError(msg) => msg, + other => other.to_string(), + }; + IndexerError::ConfigError(format!( + "Invalid validator_grpc_listen_address '{grpc_listen_address}': {msg}" + )) + })?; + + let validator_state_config = zebra_state::Config { + cache_dir: cfg.zebra_db_path.clone(), + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + should_backup_non_finalized_state: true, + debug_skip_non_finalized_state_backup_task: false, }; - IndexerError::ConfigError(format!( - "Invalid validator_grpc_listen_address '{grpc_listen_address}': {msg}" - )) - })?; - - let validator_state_config = zebra_state::Config { - cache_dir: cfg.zebra_db_path.clone(), - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - should_backup_non_finalized_state: true, - debug_skip_non_finalized_state_backup_task: false, - }; - let validator_cookie_auth = cfg.validator_settings.validator_cookie_path.is_some(); - - Ok(StateServiceConfig { - common: build_common(cfg), - validator_state_config, - validator_grpc_address, - validator_cookie_auth, - }) - } -} + let validator_cookie_auth = cfg.validator_settings.validator_cookie_path.is_some(); -#[allow(deprecated)] -impl TryFrom for FetchServiceConfig { - type Error = IndexerError; + ValidatorConnectionType::Direct(DirectConnectionConfig { + validator_state_config, + validator_grpc_address, + validator_cookie_auth, + }) + } + }; - fn try_from(cfg: ZainodConfig) -> Result { - Ok(FetchServiceConfig { + Ok(NodeBackedIndexerServiceConfig { common: build_common(cfg), + connection, }) } } @@ -555,7 +569,8 @@ key_path = "{}" let config_path = create_test_config_file(&temp_dir, &toml_content, "full_config.toml"); let config = load_config(&config_path).expect("load_config failed"); - assert_eq!(config.backend, BackendType::Fetch); + // legacy `backend = "fetch"` still parses via the serde alias + assert_eq!(config.backend, BackendType::Rpc); assert!(config.json_server_settings.is_some()); assert_eq!( config @@ -588,7 +603,7 @@ key_path = "{}" let toml_content = r#" backend = "state" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/opt/zebra/data" [storage.database] @@ -605,7 +620,9 @@ listen_address = "127.0.0.1:8137" let config = load_config(&config_path).expect("load_config failed"); let default_values = ZainodConfig::default(); - assert_eq!(config.backend, BackendType::State); + // legacy `backend = "state"` still parses via the serde alias + assert_eq!(config.backend, BackendType::Direct); + assert_eq!(config.network, Network::PubTestnet); assert!(config.json_server_settings.is_none()); assert_eq!( config.validator_settings.validator_user, @@ -617,6 +634,33 @@ listen_address = "127.0.0.1:8137" ); } + /// The pre-rename config spelling of The Public Testnet still parses, + /// via the `#[serde(alias = "Testnet")]` on `Network::PubTestnet`. + #[test] + fn legacy_testnet_spelling_parses_as_the_pub_testnet() { + let _guard = EnvGuard::new(); + let temp_dir = TempDir::new().unwrap(); + + let toml_content = r#" +backend = "state" +network = "Testnet" +zebra_db_path = "/opt/zebra/data" + +[storage.database] +path = "/opt/zaino/data" + +[validator_settings] +validator_jsonrpc_listen_address = "127.0.0.1:18232" + +[grpc_settings] +listen_address = "127.0.0.1:8137" +"#; + + let config_path = create_test_config_file(&temp_dir, toml_content, "legacy_testnet.toml"); + let config = load_config(&config_path).expect("load_config failed"); + assert_eq!(config.network, Network::PubTestnet); + } + #[test] fn test_cookie_dir_logic() { let _guard = EnvGuard::new(); @@ -625,7 +669,7 @@ listen_address = "127.0.0.1:8137" // Scenario 1: auth enabled, cookie_dir empty (should use default ephemeral path) let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -655,7 +699,7 @@ listen_address = "127.0.0.1:8137" // Scenario 2: auth enabled, cookie_dir specified let toml_content2 = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -682,7 +726,7 @@ listen_address = "127.0.0.1:8137" // Scenario 3: cookie_dir not specified (should be None) let toml_content3 = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -793,7 +837,7 @@ listen_address = "127.0.0.1:8137" let temp_dir = TempDir::new().unwrap(); let toml_content = r#" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "127.0.0.1:18232" @@ -884,7 +928,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "192.168.1.10:18232" @@ -915,7 +959,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "8.8.8.8:18232" @@ -1138,66 +1182,56 @@ listen_address = "127.0.0.1:8137" /// `CARGO_PKG_VERSION` at the boundary so the wire reflects the /// deployed binary, not zaino-state's library version. #[test] - #[allow(deprecated)] fn indexer_version_is_zainod_pkg_version() { let _guard = EnvGuard::new(); - let cfg = ZainodConfig::default(); - - let state_cfg = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for default ZainodConfig"); - assert_eq!(state_cfg.common.indexer_version, env!("CARGO_PKG_VERSION")); - - let fetch_cfg = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for default ZainodConfig"); - assert_eq!(fetch_cfg.common.indexer_version, env!("CARGO_PKG_VERSION")); + let service_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig::default()) + .expect("service config conversion should succeed for default ZainodConfig"); + assert_eq!( + service_cfg.common.indexer_version, + env!("CARGO_PKG_VERSION") + ); } - /// `StateServiceConfig::try_from` and `FetchServiceConfig::try_from` - /// share a single `build_common` helper, so the two backends can - /// never quietly disagree on the common payload they hand to a - /// service. Locks that property in across every field: a future - /// hand-rolled divergence (e.g. one path stops applying the - /// missing-credentials sentinel, or a new common field gets - /// populated on only one side) makes this fail. Pretty-Debug - /// equality is used because not every constituent of - /// `CommonBackendConfig` derives `PartialEq`, and a single - /// stringified compare future-proofs the test against fields added - /// later. + /// The `Rpc` and `Direct` connections share a single `build_common` helper, so the + /// common payload handed to the service is connection-independent. Locks that in + /// across every field: a future divergence (e.g. one path stops applying the + /// missing-credentials sentinel, or a new common field gets populated on only one + /// side) makes this fail. Pretty-Debug equality is used because not every constituent + /// of `CommonBackendConfig` derives `PartialEq`, and a single stringified compare + /// future-proofs the test against fields added later. #[test] - #[allow(deprecated)] - fn state_and_fetch_common_payloads_agree() { + fn common_payload_is_connection_independent() { let _guard = EnvGuard::new(); - let cfg = ZainodConfig::default(); + let rpc_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { + backend: BackendType::Rpc, + ..ZainodConfig::default() + }) + .expect("Rpc conversion should succeed for default ZainodConfig"); + let direct_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { + backend: BackendType::Direct, + ..ZainodConfig::default() + }) + .expect("Direct conversion should succeed for default ZainodConfig"); - let state_config = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for default ZainodConfig"); - let fetch_config = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for default ZainodConfig"); + assert!(matches!(rpc_cfg.connection, ValidatorConnectionType::Rpc)); + assert!(matches!( + direct_cfg.connection, + ValidatorConnectionType::Direct(_) + )); assert_eq!( - format!("{:#?}", state_config.common), - format!("{:#?}", fetch_config.common), + format!("{:#?}", rpc_cfg.common), + format!("{:#?}", direct_cfg.common), ); - let cfg = ZainodConfig { + let ephemeral_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { ephemeral_finalised_state: true, ..ZainodConfig::default() - }; - - let state_config = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for ephemeral finalised state"); - let fetch_config = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for ephemeral finalised state"); - - assert!(state_config.common.ephemeral_finalised_state); - assert!(fetch_config.common.ephemeral_finalised_state); - - assert_eq!( - format!("{:#?}", state_config.common), - format!("{:#?}", fetch_config.common), - ); + }) + .expect("conversion should succeed for ephemeral finalised state"); + assert!(ephemeral_cfg.common.ephemeral_finalised_state); } /// Builds a default config with the JSON-RPC server bound to `addr`. @@ -1286,7 +1320,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" ephemeral_finalised_state = true [validator_settings] @@ -1305,12 +1339,9 @@ listen_address = "127.0.0.1:8137" assert!(config.ephemeral_finalised_state); - #[allow(deprecated)] - { - let fetch_config = FetchServiceConfig::try_from(config) - .expect("FetchServiceConfig conversion should succeed"); + let service_config = NodeBackedIndexerServiceConfig::try_from(config) + .expect("service config conversion should succeed"); - assert!(fetch_config.common.ephemeral_finalised_state); - } + assert!(service_config.common.ephemeral_finalised_state); } } diff --git a/packages/zainod/src/error.rs b/packages/zainod/src/error.rs index 38a2433e4..0c27cecac 100644 --- a/packages/zainod/src/error.rs +++ b/packages/zainod/src/error.rs @@ -3,12 +3,10 @@ use zaino_fetch::jsonrpsee::error::TransportError; use zaino_serve::server::error::ServerError; -#[allow(deprecated)] -use zaino_state::{FetchServiceError, StateServiceError}; +use zaino_state::NodeBackedIndexerServiceError; /// Zingo-Indexer errors. #[derive(Debug, thiserror::Error)] -#[allow(deprecated)] pub enum IndexerError { /// Server based errors. #[error("Server error: {0}")] @@ -19,12 +17,9 @@ pub enum IndexerError { /// JSON RPSee connector errors. #[error("JSON RPSee connector error: {0}")] TransportError(#[from] TransportError), - /// FetchService errors. - #[error("FetchService error: {0}")] - FetchServiceError(Box), - /// FetchService errors. - #[error("StateService error: {0}")] - StateServiceError(Box), + /// NodeBackedIndexerService errors. + #[error("NodeBackedIndexerService error: {0}")] + NodeBackedIndexerServiceError(Box), /// HTTP related errors due to invalid URI. #[error("HTTP error: Invalid URI {0}")] HttpError(#[from] http::Error), @@ -43,16 +38,8 @@ pub enum IndexerError { Restart, } -#[allow(deprecated)] -impl From for IndexerError { - fn from(value: StateServiceError) -> Self { - IndexerError::StateServiceError(Box::new(value)) - } -} - -#[allow(deprecated)] -impl From for IndexerError { - fn from(value: FetchServiceError) -> Self { - IndexerError::FetchServiceError(Box::new(value)) +impl From for IndexerError { + fn from(value: NodeBackedIndexerServiceError) -> Self { + IndexerError::NodeBackedIndexerServiceError(Box::new(value)) } } diff --git a/packages/zainod/src/indexer.rs b/packages/zainod/src/indexer.rs index 1b23120a8..cfcdc2064 100644 --- a/packages/zainod/src/indexer.rs +++ b/packages/zainod/src/indexer.rs @@ -8,10 +8,9 @@ use zaino_serve::{ rpc::grpc_routes, server::{config::GrpcServerConfig, grpc::TonicServer, jsonrpc::JsonRpcServer}, }; -#[allow(deprecated)] use zaino_state::{ - BackendType, FetchService, FetchServiceConfig, IndexerService, LightWalletService, - StateService, StateServiceConfig, StatusType, ZcashIndexer, ZcashService, + IndexerService, LightWalletService, NodeBackedIndexerService, NodeBackedIndexerServiceConfig, + StatusType, ZcashIndexer, ZcashService, }; use crate::{config::ZainodConfig, error::IndexerError}; @@ -61,21 +60,13 @@ pub async fn spawn_indexer( info!(uri = %zebrad_uri, "Connected to node via JsonRPSee"); - #[allow(deprecated)] - match config.backend { - BackendType::State => { - let state_config = StateServiceConfig::try_from(config.clone())?; - Indexer::::launch_inner(state_config, config) - .await - .map(|res| res.0) - } - BackendType::Fetch => { - let fetch_config = FetchServiceConfig::try_from(config.clone())?; - Indexer::::launch_inner(fetch_config, config) - .await - .map(|res| res.0) - } - } + // Both the JSON-RPC (`Rpc`) and direct-`ReadStateService` (`Direct`) connections are + // now served by the single `NodeBackedIndexerService`; the connection is selected + // inside the config conversion from `config.backend`. + let service_config = NodeBackedIndexerServiceConfig::try_from(config.clone())?; + Indexer::::launch_inner(service_config, config) + .await + .map(|res| res.0) } impl Indexer diff --git a/tools/makefiles/lints.toml b/tools/makefiles/lints.toml index cac50b8d3..e6824f01f 100644 --- a/tools/makefiles/lints.toml +++ b/tools/makefiles/lints.toml @@ -82,6 +82,11 @@ description = "Assert Dockerfile.deterministic's stagex/pallet-rust tag matches command = "cargo" args = ["run", "-q", "--manifest-path", "tools/workbench/Cargo.toml", "--bin", "check-toolchain-pin"] +[tasks.lint-crypto-provider] +description = "Assert rustls resolves with aws-lc-rs + prefer-post-quantum and that the only ring-selecting feature edges are the tolerated zebra path (ADR-0006). Fails loud in both directions: new edges and vanished edges." +command = "cargo" +args = ["run", "-q", "--manifest-path", "tools/workbench/Cargo.toml", "--bin", "check-crypto-provider"] + [tasks.lint-no-openssl-apt] description = "Fail if any Dockerfile installs a libssl*/openssl* apt package. TLS is rustls; the system-package companion to deny.toml's openssl crate ban." command = "cargo" @@ -106,6 +111,7 @@ dependencies = [ "lint-boundary-conversions", "lint-workbench", "lint-toolchain-pin", + "lint-crypto-provider", "lint-no-openssl-apt", "deny", "lint-shell", diff --git a/tools/scripts/build-image.sh b/tools/scripts/build-image.sh index 9d3f44185..1233965e2 100755 --- a/tools/scripts/build-image.sh +++ b/tools/scripts/build-image.sh @@ -40,11 +40,20 @@ info "Files in tools/scripts/: $(ls -la tools/scripts/ | head -5)" DEVTOOL_REV=$(resolve_devtool_rev "$DEVTOOL_VERSION") info "Resolved DEVTOOL_VERSION=$DEVTOOL_VERSION to DEVTOOL_REV=$DEVTOOL_REV" +# Resolve ZEBRA_VERSION (canonically the Docker Hub tag, e.g. "6.0.0-rc.0") +# to the git ref the zebra-builder source stage checks out — zebra's release +# tags are `v`-prefixed in git. Fails here, before the build, on an +# unresolvable value. +ZEBRA_GIT_REF=$(cargo run -q --manifest-path tools/workbench/Cargo.toml \ + --bin get-zebra-git-ref -- "$ZEBRA_VERSION") +info "Resolved ZEBRA_VERSION=$ZEBRA_VERSION to ZEBRA_GIT_REF=$ZEBRA_GIT_REF" + cd live-tests/test_environment && \ podman build -f Containerfile \ --target "$TARGET" \ --build-arg "ZCASH_VERSION=$ZCASH_VERSION" \ --build-arg "ZEBRA_VERSION=$ZEBRA_VERSION" \ + --build-arg "ZEBRA_GIT_REF=$ZEBRA_GIT_REF" \ --build-arg "DEVTOOL_VERSION=$DEVTOOL_VERSION" \ --build-arg "DEVTOOL_REV=$DEVTOOL_REV" \ --build-arg "RUST_VERSION=$RUST_VERSION" \ diff --git a/tools/scripts/functions.sh b/tools/scripts/functions.sh index 7b28fd427..533ce4278 100755 --- a/tools/scripts/functions.sh +++ b/tools/scripts/functions.sh @@ -21,4 +21,4 @@ resolve_devtool_rev() { local ref="$1" rev rev=$(git ls-remote https://github.com/zingolabs/zcash-devtool "$ref" 2>/dev/null | awk 'NR==1{print $1}') echo "${rev:-$ref}" -} \ No newline at end of file +} diff --git a/tools/scripts/help.sh b/tools/scripts/help.sh index 524c80b6c..7f962451c 100755 --- a/tools/scripts/help.sh +++ b/tools/scripts/help.sh @@ -12,7 +12,7 @@ echo "" echo "Common usage:" echo " makers test # packages/* tests, no live validator (default)" echo " makers test live # both live partitions + combined summary" -echo " makers test all # everything: package then live" +echo " makers test all # everything: packages then live" echo "" echo "If you modify '.env.testing-artifacts', the test command will \ automatically:" @@ -21,15 +21,15 @@ echo " - Build a new local container image if needed" echo "" echo "Available commands:" echo "" -echo " test [SET] Front door. SET = package (default) | e2e | \ +echo " test [SET] Front door. SET = packages (default) | e2e | \ clientless | live | all" -echo " package packages/* tests, no live \ +echo " packages packages/* tests, no live \ validator" echo " e2e the e2e live partition" echo " clientless the clientless live partition" echo " live both live partitions + \ combined summary" -echo " all package then live (everything)" +echo " all packages then live (everything)" echo " container-test Engine: run nextest in the container \ (used by the front door; invoke directly to forward engine flags)" echo " live-clientless Engine: run the clientless live-test \ diff --git a/tools/scripts/init-podman-volumes.sh b/tools/scripts/init-podman-volumes.sh index cd9d6f92e..8ae163ded 100755 --- a/tools/scripts/init-podman-volumes.sh +++ b/tools/scripts/init-podman-volumes.sh @@ -1,8 +1,19 @@ #!/usr/bin/env bash # Initialize named podman volumes for container builds. +# `podman volume inspect` reads podman's database, not the filesystem, so a +# volume can "exist" in the DB while its backing directory is gone (e.g. after +# a partial ~/.local/share/containers cleanup or an interrupted `podman system +# reset`). Podman then refuses to mount it with "failed to validate if host +# path is dangerous: lstat .../volumes/: no such file or directory". Guard +# on the actual Mountpoint directory, and force-recreate a stale record so the +# DB and on-disk storage stay in sync. for vol in zaino-container-target zaino-cargo-git zaino-cargo-registry; do - if ! podman volume inspect "$vol" >/dev/null 2>&1; then + mountpoint="$(podman volume inspect --format '{{.Mountpoint}}' "$vol" 2>/dev/null)" + if [[ -z "$mountpoint" || ! -d "$mountpoint" ]]; then + # -z: no DB record. `! -d`: DB record present but backing dir gone; + # rm --force drops the dangling record before we recreate it. + podman volume rm --force "$vol" >/dev/null 2>&1 || true podman volume create "$vol" echo "Created podman volume: $vol" fi diff --git a/tools/test-runner/src/bin/live-summary.rs b/tools/test-runner/src/bin/live-summary.rs index 47d3ff3bd..2ca93c47e 100644 --- a/tools/test-runner/src/bin/live-summary.rs +++ b/tools/test-runner/src/bin/live-summary.rs @@ -1,10 +1,12 @@ -//! `live-summary` — run both live-test partitions and print a combined summary. +//! `live-summary` — run the test partitions and print a combined summary. //! -//! Invoked from `makers test live` (and `makers test all`) as -//! `cargo run --bin live-summary -- `. Runs the `clientless` then `e2e` -//! partition (each in its own CI container via its own `makers` task), streams -//! each run's output while capturing it, parses the nextest summary line, and -//! aggregates the totals. +//! Invoked from `makers test live` as `cargo run --bin live-summary -- ` +//! and from `makers test all` with the extra `--all` flag. The live set runs +//! the `clientless` then `e2e` partition (each in its own CI container via its +//! own `makers` task); `--all` first runs the `packages` set (`makers +//! container-test`) and adds its row to the table. The runner streams each +//! run's output while capturing it, parses the nextest summary line for +//! tallies, and measures wall-clock duration via `Instant`. //! //! Unlike a cargo-make `dependencies` list (which is fail-fast), this runs BOTH //! partitions even when the first fails, so the summary reflects the whole @@ -18,6 +20,15 @@ use std::error::Error; use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Which partitions this invocation runs and summarizes. +enum TestSet { + /// The two live partitions (`makers test live`). + Live, + /// The packages set plus the two live partitions (`makers test all`). + All, +} /// One nextest run's tallies, zero where the summary line was absent. #[derive(Default)] @@ -26,6 +37,8 @@ struct Summary { passed: u64, failed: u64, skipped: u64, + /// Wall-clock duration measured by the runner (includes build + test time). + elapsed: Duration, } impl Summary { @@ -35,6 +48,7 @@ impl Summary { passed: self.passed + other.passed, failed: self.failed + other.failed, skipped: self.skipped + other.skipped, + elapsed: self.elapsed + other.elapsed, } } } @@ -115,8 +129,7 @@ fn parse_summary(log: &str) -> Summary { let line = log .lines() .map(strip_ansi) - .filter(|l| l.contains("run:") && l.contains("test")) - .last() + .rfind(|l| l.contains("run:") && l.contains("test")) .unwrap_or_default(); Summary { @@ -125,37 +138,82 @@ fn parse_summary(log: &str) -> Summary { passed: count_before(&line, "passed"), failed: count_before(&line, "failed"), skipped: count_before(&line, "skipped"), + ..Summary::default() } } fn print_row(label: &str, s: &Summary) { + let total_secs = s.elapsed.as_secs(); + let millis = s.elapsed.subsec_millis(); + let secs = format!("{total_secs}.{millis:03}"); println!( - " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped", + " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped, {secs:>10}s", s.run, s.passed, s.failed, s.skipped ); } fn main() -> Result<(), Box> { let with_zcashd = std::env::args().any(|a| a == "--with-zcashd"); + let set = if std::env::args().any(|a| a == "--all") { + TestSet::All + } else { + TestSet::Live + }; + + let packages = match set { + TestSet::All => { + println!(">>> all: running packages set"); + let start = Instant::now(); + let (rc, log) = run_partition("container-test", with_zcashd)?; + let elapsed = start.elapsed(); + let mut s = parse_summary(&log); + s.elapsed = elapsed; + Some((rc, s)) + } + TestSet::Live => None, + }; println!(">>> live: running clientless partition"); + let cl_start = Instant::now(); let (cl_rc, cl_log) = run_partition("live-clientless", with_zcashd)?; + let cl_elapsed = cl_start.elapsed(); println!(">>> live: running e2e partition"); + let e2e_start = Instant::now(); let (e2e_rc, e2e_log) = run_partition("live-e2e", with_zcashd)?; + let e2e_elapsed = e2e_start.elapsed(); - let cl = parse_summary(&cl_log); - let e2e = parse_summary(&e2e_log); + let mut cl = parse_summary(&cl_log); + cl.elapsed = cl_elapsed; + let mut e2e = parse_summary(&e2e_log); + e2e.elapsed = e2e_elapsed; + + let header = match packages { + Some(_) => "test summary", + None => "live summary", + }; + let mut total = cl.add(&e2e); + if let Some((_, p)) = &packages { + total = total.add(p); + } println!(); - println!("====================== live summary =========================="); + println!("====================== {header} =========================="); + if let Some((_, p)) = &packages { + print_row("packages:", p); + } print_row("clientless:", &cl); print_row("e2e:", &e2e); - print_row("TOTAL:", &cl.add(&e2e)); + print_row("TOTAL:", &total); println!("=============================================================="); // A partition that errored without producing a summary line likely failed // to build; call it out so the zeros above aren't read as "all clear". + if let Some((pkg_rc, p)) = &packages { + if *pkg_rc != 0 && p.run == 0 { + println!(" warning: packages produced no nextest summary (build failure?) — see output above."); + } + } if cl_rc != 0 && cl.run == 0 { println!(" warning: clientless produced no nextest summary (build failure?) — see output above."); } @@ -163,8 +221,9 @@ fn main() -> Result<(), Box> { println!(" warning: e2e produced no nextest summary (build failure?) — see output above."); } - // Fail the front door if either partition failed. - if cl_rc != 0 || e2e_rc != 0 { + // Fail the front door if any set failed. + let pkg_rc = packages.map_or(0, |(rc, _)| rc); + if pkg_rc != 0 || cl_rc != 0 || e2e_rc != 0 { std::process::exit(1); } Ok(()) diff --git a/tools/workbench/src/bin/check-crypto-provider.rs b/tools/workbench/src/bin/check-crypto-provider.rs new file mode 100644 index 000000000..b3926a1cf --- /dev/null +++ b/tools/workbench/src/bin/check-crypto-provider.rs @@ -0,0 +1,206 @@ +//! Guard: one rustls CryptoProvider — aws-lc-rs — plus the single tolerated +//! ring path through zebra (ADR-0006). +//! +//! Two checks against the production workspace's resolved dependency graph: +//! +//! 1. rustls must resolve with `aws_lc_rs` and `prefer-post-quantum`. +//! 2. The set of feature edges that select ring anywhere in the graph must +//! equal [`RING_EDGE_ALLOWLIST`] — today, the single chain rooted in +//! zebra-node-services' `rpc-client` feature (its reqwest `rustls-tls`; +//! reqwest 0.12 offers no aws-lc alternative). That ring is compiled but +//! dormant: reqwest consults the process-default provider first and zaino +//! installs aws-lc-rs at both TLS boundaries. +//! +//! Loud in both directions. A NEW edge means a second provider path is +//! creeping back in via feature unification — the regression this guard +//! exists to catch (zingolabs/zaino#1360). A MISSING edge — in particular +//! ring vanishing from the graph entirely — means zebra dropped its ring +//! dependence: delete [`RING_EDGE_ALLOWLIST`], make this check assert ring's +//! absence, and close the classical-deprecation tracking issue's zebra item. + +use std::collections::BTreeSet; +use std::path::Path; +use std::process::Command; + +use workbench::{repo_root, run}; + +/// Feature names that select the ring provider when enabled on any crate. +const RING_SELECTING_FEATURES: [&str; 3] = ["ring", "__rustls-ring", "tls-ring"]; + +/// Every feature edge expected to select ring, all downstream of +/// zebra-node-services' reqwest `rustls-tls` (see module docs). +const RING_EDGE_ALLOWLIST: [&str; 5] = [ + "hyper-rustls feature \"ring\"", + "reqwest feature \"__rustls-ring\"", + "rustls feature \"ring\"", + "rustls-webpki feature \"ring\"", + "tokio-rustls feature \"ring\"", +]; + +fn main() { + run("check-crypto-provider", check, |()| { + println!( + "check-crypto-provider: ok — rustls resolves aws-lc-rs + prefer-post-quantum; \ + ring edges match the tolerated zebra path" + ); + }) +} + +fn check() -> Result<(), Vec> { + let root = repo_root()?; + + let rustls_features = cargo_tree(&root, &["--package", "rustls", "--depth", "0"], "{f}")?; + check_preferred_provider(&rustls_features)?; + + let ring_tree = match cargo_tree(&root, &["--invert", "ring", "--edges", "features"], "{p}") { + Ok(out) => out, + Err(diag) + if diag + .iter() + .any(|l| l.contains("did not match any packages")) => + { + return Err(vec![ + "ring is GONE from the dependency graph — zebra dropped it!".to_string(), + "Tighten this guard: delete RING_EDGE_ALLOWLIST and assert ring's absence," + .to_string(), + "and close the zebra item on the classical-deprecation tracking issue.".to_string(), + ]); + } + Err(diag) => return Err(diag), + }; + check_ring_edges(&ring_tree) +} + +/// Assert the resolved rustls feature set contains the preferred-provider pair. +fn check_preferred_provider(features_line: &str) -> Result<(), Vec> { + let features: BTreeSet<&str> = features_line.trim().split(',').collect(); + let missing: Vec<&str> = ["aws_lc_rs", "prefer-post-quantum"] + .into_iter() + .filter(|f| !features.contains(f)) + .collect(); + if missing.is_empty() { + Ok(()) + } else { + Err(vec![format!( + "rustls resolved without {} (got: {}) — the preferred provider per \ + ADR-0006 is aws-lc-rs with prefer-post-quantum; check \ + packages/zaino-common/Cargo.toml's rustls features", + missing.join(" + "), + features_line.trim(), + )]) + } +} + +/// Assert the ring-selecting feature edges equal the tolerated allowlist. +fn check_ring_edges(ring_tree: &str) -> Result<(), Vec> { + let found = ring_selecting_edges(ring_tree); + let expected: BTreeSet = RING_EDGE_ALLOWLIST + .iter() + .map(ToString::to_string) + .collect(); + + if found == expected { + return Ok(()); + } + + let mut msg = Vec::new(); + for new in found.difference(&expected) { + msg.push(format!( + "NEW ring-selecting feature edge: {new} — a second CryptoProvider path is \ + creeping back in (zingolabs/zaino#1360); remove the enabling feature", + )); + } + for gone in expected.difference(&found) { + msg.push(format!( + "tolerated ring edge disappeared: {gone} — if zebra dropped ring, tighten \ + this guard (see the module docs in check-crypto-provider.rs)", + )); + } + msg.push("see docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md".to_string()); + Err(msg) +} + +/// The deduplicated ` feature ""` edges in a +/// `cargo tree --edges features` output whose feature name selects ring. +fn ring_selecting_edges(tree: &str) -> BTreeSet { + tree.lines() + .map(|line| { + line.trim_start_matches(['│', '├', '└', '─', ' ']) + .trim_end_matches(" (*)") + }) + .filter(|node| { + node.split_once(" feature ").is_some_and(|(_, feature)| { + RING_SELECTING_FEATURES + .iter() + .any(|f| feature == format!("\"{f}\"")) + }) + }) + .map(ToString::to_string) + .collect() +} + +/// Run `cargo tree --format ` against the production workspace +/// at `root`, returning stdout; diagnostics carry stderr so callers can +/// distinguish "package not in graph" from other failures. +fn cargo_tree(root: &Path, args: &[&str], format: &str) -> Result> { + let output = Command::new("cargo") + .arg("tree") + .args(args) + .args(["--format", format]) + .current_dir(root) + .output() + .map_err(|e| vec![format!("failed to run cargo tree: {e}")])?; + if !output.status.success() { + return Err(vec![format!( + "`cargo tree {}` failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim(), + )]); + } + String::from_utf8(output.stdout).map_err(|e| vec![format!("cargo output not utf-8: {e}")]) +} + +#[cfg(test)] +mod ring_selecting_edges { + use super::*; + + #[test] + fn extracts_and_dedups_ring_edges_from_a_feature_tree() { + let tree = "\ +ring v0.17.14 +├── rustls v0.23.41 +│ ├── rustls feature \"ring\" +│ │ └── reqwest feature \"__rustls-ring\" +│ └── rustls feature \"ring\" (*) +└── rustls-webpki v0.103.13 + └── rustls-webpki feature \"ring\" +"; + let edges = ring_selecting_edges(tree); + assert_eq!( + edges.into_iter().collect::>(), + [ + "reqwest feature \"__rustls-ring\"", + "rustls feature \"ring\"", + "rustls-webpki feature \"ring\"", + ] + ); + } + + #[test] + fn ignores_package_nodes_and_non_ring_features() { + let tree = "\ +rustls v0.23.41 +├── rustls feature \"aws_lc_rs\" +├── reqwest feature \"rustls-tls\" +└── tokio-rustls feature \"logging\" +"; + assert!(ring_selecting_edges(tree).is_empty()); + } + + #[test] + fn matches_feature_names_exactly_not_by_substring() { + // "tls-ring-extra" must not match the "tls-ring" selector. + let tree = "└── tonic feature \"tls-ring-extra\"\n"; + assert!(ring_selecting_edges(tree).is_empty()); + } +} diff --git a/tools/workbench/src/bin/check-published-versions.rs b/tools/workbench/src/bin/check-published-versions.rs new file mode 100644 index 000000000..8d7bf8fd7 --- /dev/null +++ b/tools/workbench/src/bin/check-published-versions.rs @@ -0,0 +1,470 @@ +//! Guard: no publishable crate reuses an already-published version number +//! with different content. +//! +//! For every crate `cargo package --workspace` produces, look its exact +//! version up on the crates.io sparse index. A version that is not on the +//! index is fine (a fresh number, or a never-published crate). A published +//! version is fine only when the packaged content is byte-identical to the +//! published `.crate` — an unchanged crate legitimately keeps its number and +//! is simply skipped at release. Anything else is a version-reuse violation: +//! the tree cannot be released until that crate's version is bumped. +//! +//! `.cargo_vcs_info.json` is excluded from the comparison — it embeds the +//! packaging commit's SHA, so it differs on every commit even when the +//! source is identical. Yanked versions count as published: crates.io never +//! frees a version number. +//! +//! `--mode advisory` reports violations as warnings and exits 0 (feature +//! branches, where unbumped-but-changed crates are the normal +//! bump-at-release state). `--mode blocking` reports them as errors and +//! exits 1 (`rc/**` and `stable` release gates). +//! +//! Std-only by crate design: network, extraction, and diffing go through +//! `curl`, `tar`, and `diff` subprocesses. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use workbench::{repo_root, run}; + +const PROG: &str = "check-published-versions"; +const DIFF_LINE_CAP: usize = 120; + +/// Failure disposition for version-reuse violations. Infrastructure errors +/// (network, subprocess) fail loudly in either mode — a check that could not +/// run must not read as a pass. +enum Mode { + Advisory, + Blocking, +} + +struct PackagedCrate { + name: String, + version: String, +} + +impl PackagedCrate { + fn id(&self) -> String { + format!("{}@{}", self.name, self.version) + } +} + +fn main() { + run(PROG, check, |summary| println!("{PROG}: {summary}")) +} + +fn check() -> Result> { + let mode = mode_from_args(std::env::args().skip(1))?; + let root = repo_root()?; + let scratch = create_scratch_dir()?; + + let packaged = package_workspace(&root)?; + let mut violations = Vec::new(); + for krate in &packaged { + if let Comparison::Differs(diff) = compare_against_published(&root, &scratch, krate)? { + violations.push((krate, diff)); + } + } + let _ = std::fs::remove_dir_all(&scratch); + write_github_output(!violations.is_empty())?; + + if violations.is_empty() { + return Ok(format!( + "ok — none of the {} publishable crates reuses a published version with different content", + packaged.len() + )); + } + + let offenders: Vec = violations.iter().map(|(k, _)| k.id()).collect(); + let summary = format!( + "version-reuse violation(s): {} — already published with different content; \ + bump these versions before the next release", + offenders.join(", ") + ); + match mode { + Mode::Advisory => { + for (krate, diff) in &violations { + println!( + "{PROG}: {} is published with different content:", + krate.id() + ); + println!("{diff}"); + } + println!("::warning::{summary}"); + Ok(summary) + } + Mode::Blocking => { + let mut lines = vec![format!("::error::{summary}")]; + for (krate, diff) in violations { + lines.push(format!( + "{} is published with different content:", + krate.id() + )); + lines.push(diff); + } + Err(lines) + } + } +} + +/// Publish a `violations=` step output when running under GitHub +/// Actions (the `GITHUB_OUTPUT` file is set). Downstream steps use it to +/// degrade the publish dry-run to `--no-verify`: a crate that reuses a +/// published version is *shadowed* by the registry during verify builds +/// (cargo resolves the already-published artifact over the workspace +/// overlay), so packaged-form verification of its dependents is meaningless +/// until versions are bumped. Outside GitHub Actions this is a no-op. +fn write_github_output(violations: bool) -> Result<(), Vec> { + let Ok(path) = std::env::var("GITHUB_OUTPUT") else { + return Ok(()); + }; + let line = format!("violations={violations}\n"); + std::fs::OpenOptions::new() + .append(true) + .open(&path) + .and_then(|mut file| std::io::Write::write_all(&mut file, line.as_bytes())) + .map_err(|e| vec![format!("cannot append to GITHUB_OUTPUT ({path}): {e}")]) +} + +/// A per-process scratch directory for downloads and extractions, removed +/// (best-effort) at the end of the check. +fn create_scratch_dir() -> Result> { + let dir = std::env::temp_dir().join(format!("{PROG}-{}", std::process::id())); + std::fs::create_dir_all(&dir) + .map_err(|e| vec![format!("cannot create scratch dir {}: {e}", dir.display())])?; + Ok(dir) +} + +fn mode_from_args(mut args: impl Iterator) -> Result> { + match (args.next().as_deref(), args.next().as_deref(), args.next()) { + (Some("--mode"), Some("advisory"), None) => Ok(Mode::Advisory), + (Some("--mode"), Some("blocking"), None) => Ok(Mode::Blocking), + _ => Err(vec![format!("usage: {PROG} --mode ")]), + } +} + +/// Package every publishable workspace member (their `.crate` files land in +/// `/target/package/`) and return the packaged name/version pairs. +/// +/// This goes through `cargo publish --dry-run` rather than `cargo package`: +/// only the former skips `publish = false` members (`cargo package +/// --workspace` fails on the live-test crates' versionless path +/// dependencies). `--workspace` resolves sibling dependencies against a +/// local overlay of the registry, so this succeeds even when members depend +/// on unpublished sibling changes. `--allow-dirty` keeps local +/// (uncommitted-tree) runs useful; CI checkouts are clean regardless. +fn package_workspace(root: &Path) -> Result, Vec> { + let output = Command::new("cargo") + .current_dir(root) + .args([ + "publish", + "--workspace", + "--dry-run", + "--no-verify", + "--allow-dirty", + ]) + .output() + .map_err(|e| vec![format!("failed to run cargo publish --dry-run: {e}")])?; + let stderr = String::from_utf8_lossy(&output.stderr); + if !output.status.success() { + let mut lines = + vec!["`cargo publish --workspace --dry-run --no-verify` failed:".to_string()]; + lines.extend(stderr.lines().map(str::to_string)); + return Err(lines); + } + let packaged: Vec = stderr.lines().filter_map(packaged_crate_line).collect(); + if packaged.is_empty() { + return Err(vec![ + "cargo package reported no packaged crates — nothing to check".to_string(), + ]); + } + Ok(packaged) +} + +/// Parse a cargo status line `" Packaging zaino-common v0.2.0 (/path)"`. +fn packaged_crate_line(line: &str) -> Option { + let rest = line.trim_start().strip_prefix("Packaging ")?; + let mut words = rest.split_whitespace(); + let name = words.next()?.to_string(); + let version = words.next()?.strip_prefix('v')?.to_string(); + Some(PackagedCrate { name, version }) +} + +enum Comparison { + /// The exact version is not on the index (or the crate never published). + NotPublished, + /// Published, and the packaged content is byte-identical. + Identical, + /// Published with different content — the violation. Carries the + /// (possibly truncated) unified diff, published → local. + Differs(String), +} + +fn compare_against_published( + root: &Path, + scratch: &Path, + krate: &PackagedCrate, +) -> Result> { + let index_body = match fetch_index(scratch, &krate.name)? { + IndexLookup::NeverPublished => return Ok(Comparison::NotPublished), + IndexLookup::Versions(body) => body, + }; + if !index_lists_version(&index_body, &krate.version) { + return Ok(Comparison::NotPublished); + } + + let published_crate = download_published(scratch, krate)?; + let local_crate = local_crate_path(root, krate)?; + + let published_dir = scratch.join(krate.id()).join("published"); + let local_dir = scratch.join(krate.id()).join("local"); + extract(&published_crate, &published_dir)?; + extract(&local_crate, &local_dir)?; + + diff_trees(&published_dir, &local_dir) +} + +/// The `.crate` file [`package_workspace`] produced for `krate`. A +/// single-package `cargo package` writes to `target/package/`; the +/// multi-package `cargo publish --workspace --dry-run` stages its output in +/// `target/package/tmp-crate/` instead, so both locations are probed. +fn local_crate_path(root: &Path, krate: &PackagedCrate) -> Result> { + let file = format!("{}-{}.crate", krate.name, krate.version); + let candidates = [ + root.join("target/package").join(&file), + root.join("target/package/tmp-crate").join(&file), + ]; + candidates + .iter() + .find(|p| p.is_file()) + .cloned() + .ok_or_else(|| { + vec![format!( + "packaged {} not found at {} or {}", + krate.id(), + candidates[0].display(), + candidates[1].display() + )] + }) +} + +enum IndexLookup { + NeverPublished, + Versions(String), +} + +fn fetch_index(scratch: &Path, name: &str) -> Result> { + let body_file = scratch.join(format!("{name}.index")); + let url = format!("https://index.crates.io/{}", index_path(name)); + let status_code = curl_with_status(&url, &body_file)?; + match status_code.as_str() { + "200" => Ok(IndexLookup::Versions(workbench::read(&body_file)?)), + "404" => Ok(IndexLookup::NeverPublished), + other => Err(vec![format!( + "sparse-index lookup for {name} returned HTTP {other} ({url})" + )]), + } +} + +/// Download `url` to `out`, returning the HTTP status code. Transport-level +/// failures (DNS, TLS, timeouts) are errors; HTTP status is the caller's to +/// interpret. +fn curl_with_status(url: &str, out: &Path) -> Result> { + let output = Command::new("curl") + .args(["--silent", "--show-error", "--retry", "3"]) + .args(["--output".as_ref(), out.as_os_str()]) + .args(["--write-out", "%{http_code}", url]) + .output() + .map_err(|e| vec![format!("failed to run curl: {e}")])?; + if !output.status.success() { + return Err(vec![format!( + "curl {url} failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )]); + } + String::from_utf8(output.stdout).map_err(|e| vec![format!("curl status not utf-8: {e}")]) +} + +/// crates.io sparse-index path for a crate name (the registry's length rules). +fn index_path(name: &str) -> String { + match name.len() { + 1 => format!("1/{name}"), + 2 => format!("2/{name}"), + 3 => format!("3/{}/{name}", &name[..1]), + _ => format!("{}/{}/{name}", &name[..2], &name[2..4]), + } +} + +/// Whether the sparse-index body lists `version`. Yanked entries still count: +/// a yanked version number can never be republished. +fn index_lists_version(index_body: &str, version: &str) -> bool { + index_body + .lines() + .any(|line| vers_value(line).is_some_and(|v| v == version)) +} + +/// The `"vers"` value of one sparse-index JSON line. Only the top-level +/// version record carries a `"vers":"…"` string pair, so a substring scan is +/// sufficient for the machine-generated index format. +fn vers_value(line: &str) -> Option<&str> { + let start = line.find("\"vers\":\"")? + "\"vers\":\"".len(); + let rest = &line[start..]; + Some(&rest[..rest.find('"')?]) +} + +fn download_published(scratch: &Path, krate: &PackagedCrate) -> Result> { + let out = scratch.join(format!("{}.published.crate", krate.id())); + let url = format!( + "https://static.crates.io/crates/{}/{}-{}.crate", + krate.name, krate.name, krate.version + ); + let status_code = curl_with_status(&url, &out)?; + if status_code != "200" { + return Err(vec![format!( + "download of published {} returned HTTP {status_code} ({url})", + krate.id() + )]); + } + Ok(out) +} + +fn extract(archive: &Path, dest: &Path) -> Result<(), Vec> { + std::fs::create_dir_all(dest) + .map_err(|e| vec![format!("cannot create {}: {e}", dest.display())])?; + let output = Command::new("tar") + .args(["--extract", "--gzip", "--file"]) + .arg(archive) + .args(["--directory".as_ref(), dest.as_os_str()]) + .output() + .map_err(|e| vec![format!("failed to run tar: {e}")])?; + if !output.status.success() { + return Err(vec![format!( + "tar extraction of {} failed: {}", + archive.display(), + String::from_utf8_lossy(&output.stderr).trim() + )]); + } + Ok(()) +} + +/// Recursive unified diff, published → local, excluding the always-different +/// `.cargo_vcs_info.json` (it embeds the packaging commit's SHA). +fn diff_trees(published_dir: &Path, local_dir: &Path) -> Result> { + let output = Command::new("diff") + .args([ + "--recursive", + "--unified", + "--exclude", + ".cargo_vcs_info.json", + ]) + .args([published_dir.as_os_str(), local_dir.as_os_str()]) + .output() + .map_err(|e| vec![format!("failed to run diff: {e}")])?; + match output.status.code() { + Some(0) => Ok(Comparison::Identical), + Some(1) => Ok(Comparison::Differs(truncate_lines( + &String::from_utf8_lossy(&output.stdout), + DIFF_LINE_CAP, + ))), + _ => Err(vec![format!( + "diff failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )]), + } +} + +/// First `cap` lines of `text`, with an elision note when truncated. +fn truncate_lines(text: &str, cap: usize) -> String { + let total = text.lines().count(); + if total <= cap { + return text.trim_end().to_string(); + } + let kept: Vec<&str> = text.lines().take(cap).collect(); + format!( + "{}\n… (diff truncated: {} more lines; run `cargo run --manifest-path \ + tools/workbench/Cargo.toml --bin {PROG} -- --mode advisory` locally for the full diff)", + kept.join("\n"), + total - cap + ) +} + +#[cfg(test)] +mod packaged_crate_line { + use super::packaged_crate_line; + + #[test] + fn parses_cargo_status_lines() { + let parsed = packaged_crate_line(" Packaging zaino-common v0.2.0 (/w/zaino-common)") + .expect("status line parses"); + assert_eq!(parsed.name, "zaino-common"); + assert_eq!(parsed.version, "0.2.0"); + } + + #[test] + fn keeps_prerelease_versions_intact() { + let parsed = packaged_crate_line(" Packaging zainod v0.4.3-ironwood.1 (/w/zainod)") + .expect("prerelease line parses"); + assert_eq!(parsed.version, "0.4.3-ironwood.1"); + } + + #[test] + fn ignores_other_cargo_output() { + assert!(packaged_crate_line(" Updating crates.io index").is_none()); + assert!(packaged_crate_line("warning: crate zaino-proto@0.1.3 already exists").is_none()); + assert!(packaged_crate_line(" Packaged 17 files, 135.4KiB").is_none()); + } +} + +#[cfg(test)] +mod index_path { + use super::index_path; + + #[test] + fn follows_the_registry_length_rules() { + assert_eq!(index_path("a"), "1/a"); + assert_eq!(index_path("ab"), "2/ab"); + assert_eq!(index_path("abc"), "3/a/abc"); + assert_eq!(index_path("zainod"), "za/in/zainod"); + assert_eq!(index_path("zaino-state"), "za/in/zaino-state"); + } +} + +#[cfg(test)] +mod index_lists_version { + use super::index_lists_version; + + const INDEX: &str = concat!( + r#"{"name":"zainod","vers":"0.3.0","deps":[{"name":"vers","req":"^1"}],"yanked":false}"#, + "\n", + r#"{"name":"zainod","vers":"0.3.1","deps":[],"yanked":true}"#, + "\n", + ); + + #[test] + fn finds_exact_versions_only() { + assert!(index_lists_version(INDEX, "0.3.0")); + assert!(!index_lists_version(INDEX, "0.3")); + assert!(!index_lists_version(INDEX, "0.3.2")); + } + + #[test] + fn yanked_versions_still_count_as_published() { + assert!(index_lists_version(INDEX, "0.3.1")); + } +} + +#[cfg(test)] +mod truncate_lines { + use super::truncate_lines; + + #[test] + fn short_diffs_pass_through() { + assert_eq!(truncate_lines("a\nb\n", 3), "a\nb"); + } + + #[test] + fn long_diffs_are_capped_with_an_elision_note() { + let capped = truncate_lines("a\nb\nc\nd\n", 2); + assert!(capped.starts_with("a\nb\n…")); + assert!(capped.contains("2 more lines")); + } +} diff --git a/tools/workbench/src/bin/get-zebra-git-ref.rs b/tools/workbench/src/bin/get-zebra-git-ref.rs new file mode 100644 index 000000000..d14bbb1ee --- /dev/null +++ b/tools/workbench/src/bin/get-zebra-git-ref.rs @@ -0,0 +1,36 @@ +//! Resolve `ZEBRA_VERSION` to the git ref the zebra source build checks out. +//! +//! `ZEBRA_VERSION` (from `.env.testing-artifacts`) is canonically the Docker +//! Hub image tag — bare, e.g. "6.0.0-rc.0" — while zebra's git release tags +//! carry a `v` prefix. Both container build entry points (`build-image.sh`, +//! `build-n-push-ci-image.yaml`) run this to derive the checkout ref they +//! pass as the `ZEBRA_GIT_REF` build-arg, so an unresolvable version fails +//! before the build instead of as a pathspec error after the zebra clone. +//! +//! Usage: `get-zebra-git-ref ` (falls back to the +//! `ZEBRA_VERSION` environment variable). + +use workbench::{git, run, zebra_git_ref, zebra_ref_probes, ZEBRA_REPO_URL}; + +fn main() { + run( + "get-zebra-git-ref", + || { + let version = std::env::args() + .nth(1) + .or_else(|| std::env::var("ZEBRA_VERSION").ok()) + .filter(|v| !v.is_empty()) + .ok_or_else(|| { + vec!["usage: get-zebra-git-ref \ +(or set the ZEBRA_VERSION environment variable)" + .to_string()] + })?; + let probes = zebra_ref_probes(&version); + let mut args = vec!["ls-remote", ZEBRA_REPO_URL]; + args.extend(probes.iter().map(String::as_str)); + let output = git(&args)?; + zebra_git_ref(&version, &output) + }, + |git_ref| println!("{git_ref}"), + ) +} diff --git a/tools/workbench/src/lib.rs b/tools/workbench/src/lib.rs index 6f04ba4a9..eb577aeea 100644 --- a/tools/workbench/src/lib.rs +++ b/tools/workbench/src/lib.rs @@ -81,6 +81,57 @@ pub fn toolchain_channel(root: &Path) -> Result> { Ok(channel) } +/// The zebra repository probed by `get-zebra-git-ref`. +pub const ZEBRA_REPO_URL: &str = "https://github.com/ZcashFoundation/zebra"; + +/// The `git ls-remote` refs to probe for a `ZEBRA_VERSION` value: its +/// `v`-prefixed release tag, a bare tag, and a branch, in that precedence. +pub fn zebra_ref_probes(version: &str) -> [String; 3] { + [ + format!("refs/tags/v{version}"), + format!("refs/tags/{version}"), + format!("refs/heads/{version}"), + ] +} + +/// Resolve `ZEBRA_VERSION` to the git ref the zebra source build checks out, +/// given the `git ls-remote` output for [`zebra_ref_probes`]. +/// +/// `ZEBRA_VERSION` is canonically the Docker Hub image tag (bare, e.g. +/// "6.0.0-rc.0"), but zebra's git release tags carry a `v` prefix, so a +/// release version maps to `v{version}`. A branch or plain-tag pin resolves +/// as-is, and a commit SHA passes through (`ls-remote` matches ref names, not +/// commits). Anything else is an error — reported at resolve time instead of +/// surfacing as a checkout pathspec error after the container build stage has +/// cloned the whole zebra repo. +pub fn zebra_git_ref(version: &str, ls_remote_output: &str) -> Result> { + let v_tag = format!("refs/tags/v{version}"); + let matches_v_tag = ls_remote_output + .lines() + .filter_map(|line| line.split('\t').nth(1)) + .any(|r| r == v_tag); + + let some_ref_matched = ls_remote_output.lines().any(|l| !l.trim().is_empty()); + if matches_v_tag { + Ok(format!("v{version}")) + } else if some_ref_matched || is_commit_sha_shaped(version) { + Ok(version.to_string()) + } else { + Err(vec![format!( + "ZEBRA_VERSION={version} matches no zebra tag (v-prefixed or bare), \ +branch, or commit-SHA shape" + )]) + } +} + +/// 7 to 40 lowercase-hex characters — an abbreviated or full git commit SHA. +fn is_commit_sha_shaped(version: &str) -> bool { + (7..=40).contains(&version.len()) + && version + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + /// Value of a `channel = "..."` line, mirroring `^[[:space:]]*channel[[:space:]]*=`. /// `None` for comments, other keys, or a line without a double-quoted value. fn channel_value(line: &str) -> Option { @@ -115,6 +166,38 @@ mod tests { assert_eq!(channel_value("[toolchain]"), None); } + #[test] + fn zebra_git_ref_prefers_the_v_tag() { + // Annotated tags also list a peeled `^{}` line; the plain ref wins. + let out = "abc123\trefs/tags/v6.0.0-rc.0\nabc456\trefs/tags/v6.0.0-rc.0^{}\n"; + assert_eq!( + zebra_git_ref("6.0.0-rc.0", out).as_deref(), + Ok("v6.0.0-rc.0") + ); + } + + #[test] + fn zebra_git_ref_passes_branches_and_bare_tags_through() { + let out = "abc123\trefs/heads/main\n"; + assert_eq!(zebra_git_ref("main", out).as_deref(), Ok("main")); + } + + #[test] + fn zebra_git_ref_passes_sha_shapes_through_unprobed() { + assert_eq!( + zebra_git_ref("15d578362448fb8c4a5d29a00dcfe8adb5184082", "").as_deref(), + Ok("15d578362448fb8c4a5d29a00dcfe8adb5184082") + ); + assert_eq!(zebra_git_ref("15d5783", "").as_deref(), Ok("15d5783")); + } + + #[test] + fn zebra_git_ref_rejects_unresolvable_values() { + assert!(zebra_git_ref("not-a-real-ref", "").is_err()); + // Short-hex-lookalike below the 7-char floor is rejected too. + assert!(zebra_git_ref("abc", "").is_err()); + } + #[test] fn numeric_validation_matches_x_y_z() { assert!(is_concrete_numeric("1.96.0")); diff --git a/zainod-heights-from-validator-spec.md b/zainod-heights-from-validator-spec.md new file mode 100644 index 000000000..d78f94b0b --- /dev/null +++ b/zainod-heights-from-validator-spec.md @@ -0,0 +1,170 @@ +# Spec: zainod learns regtest activation heights from the Validator + +Solves zingolabs/zaino#1076. Requested by infras (`zingolabs/infrastructure#278`, +`bump_to_NU6.3`) in coordination with zaino's `ironwood_activation` e2e suite +(zingolabs/zaino#1368). Companion spec on the infras side: +`infras/dev/zaino-ironwood-activation-infra-spec.md` (devtool wallets on +arbitrary regtest heights — "Provider heights"). + +Governing invariant, decided 2026-07-06 (recorded as infras ADR 0003): + +> **The Validator's configured activation heights are authoritative. Every +> other component mirrors them by the strongest mechanism its protocol +> allows.** No component may treat compiled-in or config-file regtest +> heights as chain truth. + +Concretely, per component: + +- **Indexer (this spec)**: queries the Validator's `getblockchaininfo` at + startup and adopts the reported schedule. +- **Wallet client (infras side, same release as the guard lift)**: the + light-client protocol doesn't expose the schedule, so the wallet can't + ask the Validator itself — the harness queries `getblockchaininfo` on + the wallet's behalf and writes the *derived* heights into the wallet's + `activation-heights.toml`. The derivation is enforced at compile time: + the wallet config's regtest variant demands an opaque + `ValidatorHeights`, whose only public constructor is + `WalletNetwork::from_validator(&validator)`. There is no + caller-supplied-heights escape hatch; writing a height vector into a + wallet config is unrepresentable. + +Consequence for zaino's e2e suite: once both sides land, your +`ironwood_activation` fixture heights get typed in exactly one place — the +zebrad launch config. The wallet derives them via +`WalletNetwork::from_validator`; zainod adopts them via this spec; no +test-side constant needs to mirror anything. Expect the zcash_local_net +pin bump to be loud: `ZainodConfig.network` narrows to a payload-free +kind (Mainnet / PubTestnet / Regtest), `ZcashDevtoolConfig::faucet()/recipient()` +take a `WalletNetwork` parameter, and any test that assigned heights into +a wallet config stops compiling — the fix is always to launch the +validator first and derive. + +## Problem + +zainod's regtest activation heights are a compiled-in constant, silently +assumed to match whatever zebrad was launched with: + +- The zainod TOML carries only the string `network = "Regtest"`. + Deserialization inflates it to + `Network::Regtest(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS)` + (`packages/zaino-common/src/config/network.rs:66`); the constant + (`network.rs:13-26`) documents itself as "must equal zcash_local_net's + `supported_regtest_activation_heights`" — a cross-repo mirror maintained + by hand across three repos (zebra config, zaino constant, infras canonical + heights). +- That `Network` flows into everything downstream as truth: + `StateService::spawn` passes `config.common.network.to_zebra_network()` + into `init_read_state_with_syncer` (`zaino-state/src/backends/state.rs:213-218`) + and into `NodeBackedChainIndex::new` (`state.rs:267-276`); + `FetchService::spawn` does the equivalent (`backends/fetch.rs:122+`). +- When the running zebrad's heights differ, block commitment verification + computes against the wrong upgrade schedule and the chain-index sync loop + dies with `InvalidData("Block commitment could not be computed")` + (`zaino-state/src/chain_index/types/helpers.rs:183`, + `block.commitment(network)`). + +The mismatch is not hypothetical: infras is lifting its client-side guard so +devtool wallets run on **arbitrary** regtest heights, and zaino's own +`ironwood_activation` fixture (`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`) +puts NU6.3 at height 6 with everything else at 1–2. No compiled constant can +ever match caller-chosen heights. The only correct source is the running +zebrad — and it already publishes its schedule: `getblockchaininfo` returns +an `upgrades` map with per-upgrade `activationheight`, which zaino already +parses (`GetBlockchainInfoResponse.upgrades`, +`zaino-fetch/src/jsonrpsee/response.rs:356-359`, round-trip tested at +`response.rs:530+`). Both backends already call the validator at startup +(`get_info` at `state.rs:198`, `get_blockchain_info` in the tip-wait loop at +`state.rs:226`). The handshake exists; the heights are simply never adopted +from it. + +## Requested change + +1. **Learn heights at first contact.** When the configured network kind is + Regtest, each backend fetches `getblockchaininfo` from the validator + **before** constructing anything that consumes a + `zebra_chain::parameters::Network` — the ReadStateService syncer, the + chain index, the mempool source. Build the runtime regtest `Network` from + the reported `upgrades` map. The RPC client is already constructed first + (`state.rs:190-196`), so the ordering is achievable without restructuring. +2. **Learned heights are the only source.** `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` + ceases to exist as a truth source. Recommended shape: the *config-level* + network type degrades to a kind (Mainnet / PubTestnet / Regtest, no payload), + and only the *runtime* type constructed after the handshake carries + heights — making pre-handshake height reads unrepresentable. A minimal + variant (populate the existing payload from the handshake before first + use) is acceptable if the type split is too invasive, but then nothing may + read the payload before adoption. Behavior is mandated; shape is + implementer's choice. +3. **An upgrade absent from the validator's map is never-activated.** This + matches zebra's own semantics and the devtool's absent-key encoding. + Do not backfill absent upgrades from any default. +4. **Failure is loud, never defaulted.** If the network kind is Regtest and + the upgrades map cannot be fetched or parsed, startup fails with an error + naming the validator endpoint. Under no circumstances fall back to a + compiled or configured height set — a silently wrong schedule is the + exact failure mode this spec removes. + +## Acceptance + +The fixture that must work end-to-end (as zebrad regtest config heights): + +``` +overwinter = 1, sapling = 1, blossom = 1, heartwood = 1, canopy = 1, +nu5 = 2, nu6 = 2, nu6_1 = 2, nu6_2 = 2, nu6_3 = 6 +``` + +1. **Boundary sync**: launch zebrad regtest with the fixture; launch zainod + against it with only `network = "Regtest"` in its TOML. Mine past height + 6. zainod's chain index syncs across the NU6.3 boundary with no + commitment error, and serves compact blocks for both eras (Orchard + coinbase at heights 2–5, Ironwood from 6). +2. **No-recompile proof**: the same zainod binary and config, pointed at a + zebrad running a *different* schedule (the canonical all-at-2 set), also + syncs — demonstrating heights come from the validator, not the build. +3. **Contract test on the upgrades map**: pin the `getblockchaininfo → + heights` mapping with a test against real zebrad output (live regtest or + a recorded golden response) — not hand-written assertions about what the + RPC is believed to return. In particular, establish from the real output + (not from reasoning) which upgrades zebrad includes in the map (e.g. + whether anything pre-Overwinter appears) and encode exactly that. +4. **No residual truth source**: no non-test code path consumes + `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or any other compiled regtest + schedule as chain truth. Test-only fixtures that *are* their own chain + (mockchain sources, proptest block generators) legitimately keep explicit + heights — there, the test is the validator. + +## Known unknowns to check while implementing + +- **StateService init ordering**: `init_read_state_with_syncer` receives the + network at construction and the ReadStateService validates blocks with it. + Confirm nothing else needs a `Network` before the validator RPC is + reachable; if something does, that is a design conflict to surface, not + paper over. +- **zcashd as validator**: the legacy e2e path (`devtool_zcashd.rs`) drives + zcashd, whose `getblockchaininfo` also reports an `upgrades` map. Prefer + making the adoption path validator-generic so zcashd rides along; if its + map shape differs materially, scope this spec to zebrad and record the + divergence in the zcashd test docs. +- **zaino-testutils**: `live-tests/zaino-testutils/src/lib.rs:497` uses the + constant to *launch zebrad* — that is legitimate harness-side + configuration of the truth source, not adoption of it. Keep the height + set, but move/rename it so it can no longer be mistaken for (or imported + as) zainod's own schedule. +- **Reorg/restart**: heights are learned once at startup. A validator + restarted mid-session with different heights invalidates the indexer's + world; document that zainod must be restarted with its validator (regtest + harnesses already do this), rather than attempting live re-adoption. + +## Out of scope + +- Mainnet / The Public Testnet: compiled zebra network parameters are correct there; no + handshake is needed and none should be added. +- The infras-side client guard lift (Provider heights) — lands on + `infrastructure#278` independently. + +## Delivery + +A rev on a zaino branch that `live-tests` can exercise. Downstream effects +when it lands: zaino#1368's `ironwood_activation` tests gain a zainod that +can actually serve their fixture, and infras' nu6_3-at-6 integration test +(currently `#[ignore]`d naming #1076) flips live.