diff --git a/.dockerignore b/.dockerignore index 8f4a1e0f8bb..13ebf9e7c22 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,6 +23,9 @@ !deploy/watchdog/** !tower-* !xtask +!deploy +!deploy/watchdog +!deploy/watchdog/** !zebra-* !zebrad !docker/entrypoint.sh diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index af0d6a1a237..71d7155eab7 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -2,7 +2,7 @@ name: Coverage on: push: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths: - "**/*.rs" - "**/Cargo.toml" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5cd8dd2bdee..0a2ab86b3fe 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,7 @@ name: Lint on: pull_request: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths: - "**/*.rs" - "**/Cargo.toml" @@ -13,7 +13,7 @@ on: - supply-chain/** push: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths: - "**/*.rs" - "**/Cargo.toml" @@ -23,6 +23,11 @@ on: - .github/workflows/lint.yml - supply-chain/** + # Run the full lint suite when a PR is queued so the merge queue validates the + # merged result against the latest `main`. `merge_group` ignores path filters, + # so this runs unconditionally on every queued entry. + merge_group: + # Ensures that only one workflow task will run at a time. Previous builds, if # already in process, will get cancelled. Only the latest commit will be allowed # to run, cancelling any workflows in between diff --git a/.github/workflows/status-checks.patch.yml b/.github/workflows/status-checks.patch.yml index d237c6414d5..021529e55c0 100644 --- a/.github/workflows/status-checks.patch.yml +++ b/.github/workflows/status-checks.patch.yml @@ -7,7 +7,7 @@ name: Status Check Patch on: pull_request: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths-ignore: # This MUST be an exact inverse of paths in lint.yml, tests-unit.yml, and test-crates.yml # to ensure mutual exclusion - only one set of workflows runs @@ -16,6 +16,8 @@ on: - '**/Cargo.lock' - .cargo/config.toml - '**/clippy.toml' + - supply-chain/** + - .config/nextest.toml - .github/workflows/lint.yml - .github/workflows/tests-unit.yml - .github/workflows/test-crates.yml diff --git a/.github/workflows/test-crates.yml b/.github/workflows/test-crates.yml index 74c4cf65a1e..4edbce0b01f 100644 --- a/.github/workflows/test-crates.yml +++ b/.github/workflows/test-crates.yml @@ -2,7 +2,7 @@ name: Test Crate Build on: pull_request: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths: - "**/*.rs" - "**/Cargo.toml" @@ -12,7 +12,7 @@ on: - .github/workflows/test-crates.yml push: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths: - "**/*.rs" - "**/Cargo.toml" @@ -21,6 +21,11 @@ on: - "**/clippy.toml" - .github/workflows/test-crates.yml + # Run the full crate-build suite when a PR is queued so the merge queue + # validates the merged result against the latest `main`. `merge_group` ignores + # path filters, so this runs unconditionally on every queued entry. + merge_group: + workflow_dispatch: # Ensures that only one workflow task will run at a time. Previous builds, if @@ -181,8 +186,10 @@ jobs: - uses: ./.github/actions/setup-zebra-build - name: Build ${{ matrix.crate }} crate with MSRV, all features and all targets - # zebrad can have a higher MSRV than the other crates since it is a binary, so we skip the MSRV build for it. - if: matrix.crate != 'zebrad' + # zebrad and zebra-watchdog can have a higher MSRV than the other crates since they are + # binaries, not libraries, so we skip the MSRV build for them. zebra-watchdog depends on + # sentry, which requires a newer rustc than the workspace library MSRV. + if: matrix.crate != 'zebrad' && matrix.crate != 'zebra-watchdog' run: | cargo clippy --package ${{ matrix.crate }} --all-features --all-targets -- -D warnings cargo build --package ${{ matrix.crate }} --all-features --all-targets diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index e929bb60c71..f14e64efb4c 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -2,7 +2,8 @@ name: Test Docker Config on: pull_request: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] paths: - "**/*.rs" - "**/Cargo.toml" @@ -13,8 +14,14 @@ on: - zebrad/tests/common/configs/** - .github/workflows/test-docker.yml + schedule: + # Keep release-mode Docker coverage off the critical PR path. + - cron: "0 9 * * *" + + workflow_dispatch: + push: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths: - "**/*.rs" - "**/Cargo.toml" @@ -46,6 +53,7 @@ env: jobs: build-docker-image: name: Build Docker Image + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'A-release') permissions: contents: read id-token: write @@ -56,6 +64,13 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 with: persist-credentials: false + # The `tests` target image bakes in the full `target/release` build tree, which + # overflows the ~14 GB free on a standard ubuntu-latest runner. Reclaim space by + # removing preinstalled toolchains we don't use for this build. + - name: Free up runner disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/.ghcup + df -h / - name: Inject slug/short variables uses: rlespinasse/github-slug-action@e6f261660910b273384c5c42b17a0217881b217a #v5.6.0 with: @@ -84,6 +99,7 @@ jobs: test-configurations: name: Test ${{ matrix.name }} needs: build-docker-image + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'A-release') permissions: contents: read actions: read @@ -165,6 +181,13 @@ jobs: with: persist-credentials: false + # `docker load` of the `tests` image needs more space than the ~14 GB free on a + # standard ubuntu-latest runner. Reclaim space by removing unused toolchains. + - name: Free up runner disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/.ghcup + df -h / + - name: Download artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 with: @@ -241,3 +264,4 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe #v1.2.2 with: jobs: ${{ toJSON(needs) }} + allowed-skips: build-docker-image,test-configurations diff --git a/.github/workflows/tests-unit.yml b/.github/workflows/tests-unit.yml index 728ecb1720b..ed42016aa5b 100644 --- a/.github/workflows/tests-unit.yml +++ b/.github/workflows/tests-unit.yml @@ -2,22 +2,38 @@ name: Unit Tests on: pull_request: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] paths: - "**/*.rs" - "**/Cargo.toml" - "**/Cargo.lock" - .config/nextest.toml - .github/workflows/tests-unit.yml + - docker/docker-compose.zakura-regtest-e2e.yml + - docker/zakura-regtest-e2e/** push: - branches: [main] + branches: [main, ironwood-main, evan/ironwood-qol, feat/p2p-v2] paths: - "**/*.rs" - "**/Cargo.toml" - "**/Cargo.lock" - .config/nextest.toml - .github/workflows/tests-unit.yml + - docker/docker-compose.zakura-regtest-e2e.yml + - docker/zakura-regtest-e2e/** + + # Run the full unit-test suite when a PR is queued so the merge queue validates + # the merged result against the latest `main`. `merge_group` ignores path + # filters, so this runs unconditionally on every queued entry. + merge_group: + + schedule: + # Keep release-mode coverage off the critical PR path. + - cron: "0 8 * * *" + + workflow_dispatch: # Ensures that only one workflow task will run at a time. Previous builds, if # already in process, will get cancelled. Only the latest commit will be allowed @@ -49,16 +65,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - rust-version: [stable, beta] + os: [ubuntu-latest] + rust-version: [stable] features: [default-release-binaries] - exclude: - # Exclude macOS beta due to limited runner capacity and slower performance - # Exclude Windows beta to reduce the amount of minutes cost per workflow run - - os: macos-latest - rust-version: beta - - os: windows-latest - rust-version: beta steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 @@ -67,34 +76,22 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 #v1.16.1 with: toolchain: ${{ matrix.rust-version }} - cache-key: unit-tests-${{ matrix.os }}-${{ matrix.rust-version }}-${{ matrix.features }} + cache-key: unit-tests-${{ matrix.os }}-${{ matrix.rust-version }}-${{ matrix.features }}-${{ (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'A-release')) && 'debug' || 'release' }} cache-on-failure: true - uses: taiki-e/install-action@65851e10cd6c377f11a60e600abc07cb08643468 #v2.79.3 with: tool: cargo-nextest - uses: ./.github/actions/setup-zebra-build - # Windows-specific setup - - name: Install LLVM on Windows - if: matrix.os == 'windows-latest' - run: | - choco install llvm -y - echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - - name: Minimise proptest cases on macOS and Windows - # We set cases to 1, because some tests already run 1 case by default. - # We keep maximum shrink iterations at the default value, because it only happens on failure. - # - # Windows compilation and tests are slower than other platforms. - if: matrix.os == 'windows-latest' - run: | - echo "PROPTEST_CASES=1" >> $GITHUB_ENV - echo "PROPTEST_MAX_SHRINK_ITERS=1024" >> $GITHUB_ENV - - name: Run unit tests - run: cargo nextest run --profile all-tests --locked --release --features "${{ matrix.features }}" --run-ignored=all + run: | + if [[ "${RELEASE_TESTS}" == "true" ]]; then + cargo nextest run --profile all-tests --locked --release --features "${{ matrix.features }}" --run-ignored=all + else + cargo nextest run --profile all-tests --locked --features "${{ matrix.features }}" --run-ignored=all + fi env: + RELEASE_TESTS: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'A-release') }} TEST_LARGE_CHECKPOINTS: 1 check-no-git-dependencies: @@ -121,6 +118,99 @@ jobs: - name: Check no git dependencies run: cargo nextest run --profile check-no-git-dependencies --locked --run-ignored=only + zakura-regtest-e2e-pr-gate: + name: Zakura regtest e2e (pr-gate) + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push' + permissions: + contents: read + id-token: write + statuses: write + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 #v1.16.1 + with: + toolchain: stable + cache-key: zakura-regtest-e2e-pr-gate + cache-on-failure: true + - uses: ./.github/actions/setup-zebra-build + - name: Build zebrad (debug) + run: cargo build -p zebrad --bin zebrad + env: + CXXFLAGS: "-include cstdint" + - name: Run Zakura regtest PR gate + run: cargo test -p zebrad --test zakura_regtest_e2e -- --ignored --nocapture + env: + CXXFLAGS: "-include cstdint" + ZAKURA_REGTEST_E2E: "1" + ZAKURA_E2E_MODE: pr-gate + ZAKURA_E2E_TRACE_DIR: ${{ runner.temp }}/zakura-regtest-e2e-traces-pr-gate + ZAKURA_REGTEST_E2E_LABEL: zakura-pr-gate + - name: Upload Zakura e2e traces + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: zakura-regtest-e2e-pr-gate + path: ${{ runner.temp }}/zakura-regtest-e2e-traces-pr-gate + if-no-files-found: ignore + + zakura-regtest-e2e: + name: Zakura regtest e2e (${{ matrix.zakura-mode }}) + # The long four-node docker stack runs only on the nightly schedule and on + # demand. PRs and merge queue use the trimmed pr-gate job above. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + permissions: + contents: read + id-token: write + statuses: write + runs-on: ubuntu-latest + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + zakura-mode: + - checkpoint-long + - no-checkpoint-long + - restart-matrix + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 #v1.16.1 + with: + toolchain: stable + cache-key: zakura-regtest-e2e + cache-on-failure: true + - uses: ./.github/actions/setup-zebra-build + # The e2e drives a host-networked docker-compose stack of four nodes running + # the host-built debug zebrad binary; ubuntu-latest runners ship Docker and + # Compose, so no extra setup is needed. Build the binary up front so a compile + # failure surfaces clearly and separately from the e2e run. + - name: Build zebrad (debug) + run: cargo build -p zebrad --bin zebrad + env: + CXXFLAGS: "-include cstdint" + - name: Run Zakura regtest dual-stack e2e + # The test is `#[ignore]` and additionally returns early in CI unless + # `ZAKURA_REGTEST_E2E=1` is set, so both flags are required to force it. + run: cargo test -p zebrad --test zakura_regtest_e2e -- --ignored --nocapture + env: + CXXFLAGS: "-include cstdint" + ZAKURA_REGTEST_E2E: "1" + ZAKURA_E2E_MODE: ${{ matrix.zakura-mode }} + ZAKURA_E2E_TRACE_DIR: ${{ runner.temp }}/zakura-regtest-e2e-traces-${{ matrix.zakura-mode }} + ZAKURA_REGTEST_E2E_LABEL: zakura-${{ matrix.zakura-mode }} + - name: Upload Zakura e2e traces + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: zakura-regtest-e2e-${{ matrix.zakura-mode }} + path: ${{ runner.temp }}/zakura-regtest-e2e-traces-${{ matrix.zakura-mode }} + if-no-files-found: ignore + test-success: name: test success runs-on: ubuntu-latest @@ -128,10 +218,15 @@ jobs: needs: - unit-tests - check-no-git-dependencies + - zakura-regtest-e2e-pr-gate + - zakura-regtest-e2e timeout-minutes: 30 steps: - name: Decide whether the needed jobs succeeded or failed uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe #v1.2.2 with: jobs: ${{ toJSON(needs) }} - allowed-skips: check-no-git-dependencies + # The release-only dependency check and mode-specific e2e lanes are + # skipped outside their event scopes. Failed, non-skipped jobs still + # fail this aggregate status. + allowed-skips: check-no-git-dependencies, zakura-regtest-e2e-pr-gate, zakura-regtest-e2e diff --git a/Cargo.toml b/Cargo.toml index 84a8431e498..d82e07d3da2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,7 +187,15 @@ allow-branch = ["main"] # Compilation settings [profile.dev] -panic = "abort" +# panic = "unwind" (NOT "abort") is REQUIRED for Zakura peer-pipe panic +# containment: a panicking peer task must be caught and disconnect only that +# peer, never abort the whole node (a trivial remote DoS otherwise). Because we +# now unwind, every shared/consensus-state critical section must stay panic-free +# (no `.await`/`.expect()`/`.unwrap()`/indexing under a lock) — a panic mid- +# mutation would unwind through it instead of aborting. zebra-network enforces +# unwind with a `#[cfg(panic = "abort")] compile_error!`. See +# security_requirements.md SR-1 and SR-2. +panic = "unwind" # Speed up tests by optimizing performance-critical crates. The parameters # values are the ones from the `release` profile @@ -308,7 +316,10 @@ incremental = false codegen-units = 16 [profile.release] -panic = "abort" +# See [profile.dev]: panic = "unwind" is required for peer-pipe panic containment +# (security_requirements.md SR-1); do not set "abort". Enforced by a +# compile_error guard in zebra-network. +panic = "unwind" # Speed up release builds and sync tests using link-time optimization. # Some of Zebra's code is CPU-intensive, and needs extra optimizations for peak performance. diff --git a/deny.toml b/deny.toml index 2061a8c3850..86520e0660c 100644 --- a/deny.toml +++ b/deny.toml @@ -16,6 +16,7 @@ ignore = [ "RUSTSEC-2024-0375", # atty (unmaintained) — transitive via abscissa_core -> structopt "RUSTSEC-2021-0145", # atty (unsound) — transitive via abscissa_core -> structopt "RUSTSEC-2024-0370", # proc-macro-error — transitive via abscissa_core -> structopt + "RUSTSEC-2026-0173", # proc-macro-error2 (unmaintained) — transitive via getset "RUSTSEC-2025-0119", # number_prefix — transitive via indicatif "RUSTSEC-2025-0141", # bincode — direct dependency "RUSTSEC-2026-0118", # hickory-proto — transitive via iroh diff --git a/docker/docker-compose.zakura-regtest-e2e.yml b/docker/docker-compose.zakura-regtest-e2e.yml index e725088741e..4189c9c6c12 100644 --- a/docker/docker-compose.zakura-regtest-e2e.yml +++ b/docker/docker-compose.zakura-regtest-e2e.yml @@ -13,7 +13,7 @@ # docker compose -f docker/docker-compose.zakura-regtest-e2e.yml up -d # # Or just run docker/zakura-regtest-e2e/run.sh, which does all of the above and -# asserts coexistence + propagation. +# asserts legacy compatibility, Zakura propagation, and block sync. # # Host networking is required: Regtest only accepts localhost initial peers # (zebra-network/src/config.rs: initial_peers filters to is_localhost), so the @@ -43,8 +43,8 @@ services: entrypoint: ["/usr/local/bin/zebrad", "--config", "/etc/zebrad/node1.toml", "start"] volumes: - ${ZEBRAD_BIN:-../target/debug/zebrad}:/usr/local/bin/zebrad:ro - - ./zakura-regtest-e2e/node1.toml:/etc/zebrad/node1.toml:ro - - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}/node1:/traces + - ${ZAKURA_NODE1_CONFIG:-./zakura-regtest-e2e/node1.toml}:/etc/zebrad/node1.toml:ro + - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}:/traces zakura-node-2: <<: *zakura-node @@ -54,8 +54,8 @@ services: entrypoint: ["/usr/local/bin/zebrad", "--config", "/etc/zebrad/node2.toml", "start"] volumes: - ${ZEBRAD_BIN:-../target/debug/zebrad}:/usr/local/bin/zebrad:ro - - ./zakura-regtest-e2e/node2.toml:/etc/zebrad/node2.toml:ro - - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}/node2:/traces + - ${ZAKURA_NODE2_CONFIG:-./zakura-regtest-e2e/node2.toml}:/etc/zebrad/node2.toml:ro + - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}:/traces zakura-node-3: <<: *zakura-node @@ -65,8 +65,8 @@ services: entrypoint: ["/usr/local/bin/zebrad", "--config", "/etc/zebrad/node3.toml", "start"] volumes: - ${ZEBRAD_BIN:-../target/debug/zebrad}:/usr/local/bin/zebrad:ro - - ./zakura-regtest-e2e/node3.toml:/etc/zebrad/node3.toml:ro - - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}/node3:/traces + - ${ZAKURA_NODE3_CONFIG:-./zakura-regtest-e2e/node3.toml}:/etc/zebrad/node3.toml:ro + - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}:/traces zakura-node-4: <<: *zakura-node @@ -76,5 +76,5 @@ services: entrypoint: ["/usr/local/bin/zebrad", "--config", "/etc/zebrad/node4.toml", "start"] volumes: - ${ZEBRAD_BIN:-../target/debug/zebrad}:/usr/local/bin/zebrad:ro - - ./zakura-regtest-e2e/node4.toml:/etc/zebrad/node4.toml:ro - - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}/node4:/traces + - ${ZAKURA_NODE4_CONFIG:-./zakura-regtest-e2e/node4.toml}:/etc/zebrad/node4.toml:ro + - ${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}:/traces diff --git a/docker/zakura-regtest-e2e/node2.toml b/docker/zakura-regtest-e2e/node2.toml index 54a708ca588..195148df55c 100644 --- a/docker/zakura-regtest-e2e/node2.toml +++ b/docker/zakura-regtest-e2e/node2.toml @@ -8,6 +8,14 @@ [network] network = "Regtest" +# Persistent Zakura identity (distinct from node1's 0101..01). Production nodes +# keep a stable node id across restarts; the e2e's from-scratch reset must too, +# otherwise a restarted node returns under a *new* identity while reusing the +# same loopback address, colliding with its own stale entry in node1's iroh node +# map. node1 then holds the dead connection until the ~150s app idle reaper, +# stalling the kind-6 catch-up. With a stable identity the restart is a normal +# same-peer reconnect that node1's duplicate-eviction path clears in seconds. +zakura_node_secret_key = "0202020202020202020202020202020202020202020202020202020202020202" # Inert with legacy_p2p = false (no TCP listener is opened), but the field is # required, so keep it on a distinct loopback port for clarity. listen_addr = "127.0.0.1:18333" @@ -31,6 +39,13 @@ trace_dir = "/traces/node2" [state] ephemeral = true +[sync] +# Force the from-scratch genesis-bootstrap path: skip the Regtest direct genesis +# self-seed so this pure-Zakura node must download and verify genesis from node1 +# over Zakura before native header/body sync can advance. This is the exact path +# a Mainnet/Testnet Zakura-only node takes from an empty state. +debug_skip_regtest_genesis_self_seed = true + [rpc] listen_addr = "127.0.0.1:18332" enable_cookie_auth = false diff --git a/docker/zakura-regtest-e2e/run.sh b/docker/zakura-regtest-e2e/run.sh index c6711de4eac..1a8899ee188 100755 --- a/docker/zakura-regtest-e2e/run.sh +++ b/docker/zakura-regtest-e2e/run.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash # -# Drive and assert the Zakura regtest e2e — a short smoke test covering all three -# coexistence modes at once. +# Drive and assert the Zakura regtest e2e. # # Topology (all on 127.0.0.1 via host networking): # node1 dual-stack seed (legacy TCP + Zakura) rpc 18232 metrics 19001 @@ -12,18 +11,25 @@ # node4 dual-stack (legacy TCP + Zakura) rpc 18532 metrics 19004 -> node1 # dials node1 over legacy TCP, then upgrades to Zakura # -# Asserts (kept deliberately small — this is a smoke test): +# Asserts: # 1. all four nodes come up on a fresh Regtest chain, -# 2. legacy TCP coexistence: node3 peers with node1 (getpeerinfo), +# 2. legacy TCP compatibility: node3 peers with node1 (getpeerinfo), # 3. the legacy->Zakura upgrade ran (zakura_p2p_handshake_upgraded on node1/node4), # 4. the pure Zakura-only node2 has zero legacy peers (no legacy stack at all), +# 4b. the pure Zakura-only node2 bootstraps the genesis block over Zakura: with +# the Regtest genesis self-seed disabled (sync.debug_skip_regtest_genesis_self_seed) +# and no legacy stack, node2 can only reach genesis by downloading it from +# node1 over Zakura — the production Mainnet/Testnet bootstrap path, # 5. blocks generated on node1 propagate to the pure-Zakura node2 AND the # legacy-only node3 — so node2, which has no legacy stack, proves -# pure-Zakura propagation. -# 6. the upgraded dual-stack node4 propagation path is checked and reported, -# but is non-gating by default while the P2 upgrade-lifetime regression is -# tracked in stako/p2p-services/P2_E2E_KNOWN_ISSUES.md. Set -# ZAKURA_REGTEST_E2E_STRICT_UPGRADE=1 to make node4 propagation fatal. +# pure-Zakura propagation, +# 6. Zakura v2 nodes reach sync.block.verified_tip.height == +# sync.block.best_header_tip.height after gossip propagation, +# 7. after a from-scratch reset of the pure-Zakura node2 (empty state while +# node1 sits idle at the tip), node2 re-downloads the whole chain over +# kind-6 block sync — gossip cannot help because node1 re-advertises +# nothing, so this exercises the production Mainnet-from-0 / catch-up path, +# 8. a non-finalized reorg converges with no block-sync byte-budget leak. # # No image is built: each container runs the HOST-built zebrad binary # bind-mounted into debian:trixie-slim. If the binary is missing it is built @@ -36,22 +42,103 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_FILE="${SCRIPT_DIR}/../docker-compose.zakura-regtest-e2e.yml" REPO_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" -# Generate at least two blocks. Zebra's syncer intentionally discards locator +log() { printf '\n=== %s ===\n' "$*"; } +fail() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } + +ZAKURA_E2E_MODE="${ZAKURA_E2E_MODE:-smoke}" +ZAKURA_E2E_LONG_BLOCKS="${ZAKURA_E2E_LONG_BLOCKS:-4000}" + +case "${ZAKURA_E2E_MODE}" in + smoke) + DEFAULT_GENERATE_BLOCKS=3 + DEFAULT_CATCHUP_BLOCKS=200 + DEFAULT_CHECKPOINT_INTERVAL=100 + DEFAULT_PROPAGATE_TIMEOUT=150 + DEFAULT_CATCHUP_TIMEOUT=300 + ZAKURA_E2E_DISABLE_CHECKPOINTS=0 + ZAKURA_E2E_RESTART_MATRIX=0 + ZAKURA_E2E_REQUIRE_HANDOFF=0 + ;; + pr-gate) + DEFAULT_GENERATE_BLOCKS=3 + DEFAULT_CATCHUP_BLOCKS=160 + DEFAULT_CHECKPOINT_INTERVAL=80 + DEFAULT_PROPAGATE_TIMEOUT=180 + DEFAULT_CATCHUP_TIMEOUT=450 + ZAKURA_E2E_DISABLE_CHECKPOINTS=0 + ZAKURA_E2E_RESTART_MATRIX=0 + ZAKURA_E2E_REQUIRE_HANDOFF=1 + ;; + checkpoint-long) + DEFAULT_GENERATE_BLOCKS=3 + DEFAULT_CATCHUP_BLOCKS=$(( ZAKURA_E2E_LONG_BLOCKS - DEFAULT_GENERATE_BLOCKS )) + DEFAULT_CHECKPOINT_INTERVAL=400 + DEFAULT_PROPAGATE_TIMEOUT=300 + DEFAULT_CATCHUP_TIMEOUT=1200 + ZAKURA_E2E_DISABLE_CHECKPOINTS=0 + ZAKURA_E2E_RESTART_MATRIX=0 + ZAKURA_E2E_REQUIRE_HANDOFF=1 + ;; + no-checkpoint-long) + DEFAULT_GENERATE_BLOCKS=3 + DEFAULT_CATCHUP_BLOCKS=$(( ZAKURA_E2E_LONG_BLOCKS - DEFAULT_GENERATE_BLOCKS )) + DEFAULT_CHECKPOINT_INTERVAL=0 + DEFAULT_PROPAGATE_TIMEOUT=300 + DEFAULT_CATCHUP_TIMEOUT=1800 + ZAKURA_E2E_DISABLE_CHECKPOINTS=1 + ZAKURA_E2E_RESTART_MATRIX=0 + ZAKURA_E2E_REQUIRE_HANDOFF=0 + ;; + restart-matrix) + DEFAULT_GENERATE_BLOCKS=3 + DEFAULT_CATCHUP_BLOCKS=$(( ZAKURA_E2E_LONG_BLOCKS - DEFAULT_GENERATE_BLOCKS )) + DEFAULT_CHECKPOINT_INTERVAL=400 + DEFAULT_PROPAGATE_TIMEOUT=300 + DEFAULT_CATCHUP_TIMEOUT=1800 + ZAKURA_E2E_DISABLE_CHECKPOINTS=0 + ZAKURA_E2E_RESTART_MATRIX=1 + ZAKURA_E2E_REQUIRE_HANDOFF=1 + ;; + *) + fail "unknown ZAKURA_E2E_MODE='${ZAKURA_E2E_MODE}' (expected smoke, pr-gate, checkpoint-long, no-checkpoint-long, restart-matrix)" + ;; +esac + +(( DEFAULT_CATCHUP_BLOCKS >= 0 )) || fail "ZAKURA_E2E_LONG_BLOCKS must be >= ${DEFAULT_GENERATE_BLOCKS}" + +# Generate at least three blocks. Zebra's syncer intentionally discards locator # responses that extend only one block, so a one-block run can fail even when -# the Zakura request path is working. -GENERATE_BLOCKS="${GENERATE_BLOCKS:-2}" +# the Zakura request path is working. Three blocks also avoids leaving a +# one-block remainder if the tiny regtest topology drops an early response. +GENERATE_BLOCKS="${GENERATE_BLOCKS:-${DEFAULT_GENERATE_BLOCKS}}" +# Extra blocks mined on node1 after the propagation assertions and before the +# from-scratch reset, so the kind-6 catch-up re-downloads a real burst of bodies +# rather than a handful. Hundreds of blocks is what fills the inbound wire queue +# and exercises the body-flood path that wedged in production; a 3-block catch-up +# never gets near it. Set to 0 to skip the deepening and keep the legacy 3-block +# catch-up. +CATCHUP_BLOCKS="${CATCHUP_BLOCKS:-${DEFAULT_CATCHUP_BLOCKS}}" READY_TIMEOUT="${READY_TIMEOUT:-120}" # Propagation to the Zakura peer can take a little while: the dual-stack tries # the (empty) legacy peer set first, and the legacy->Zakura upgrade re-dials a # few times before the connection settles. The loop exits as soon as the block # arrives, so a generous ceiling only matters on failure. -PROPAGATE_TIMEOUT="${PROPAGATE_TIMEOUT:-150}" - -log() { printf '\n=== %s ===\n' "$*"; } -fail() { printf '\nFAILED: %s\n' "$*" >&2; exit 1; } +PROPAGATE_TIMEOUT="${PROPAGATE_TIMEOUT:-${DEFAULT_PROPAGATE_TIMEOUT}}" +# The from-scratch reset catch-up is given its own, larger ceiling. After a +# restart the reconnecting node returns under the same identity but reuses its +# stable loopback endpoint, so the seed briefly holds the dead pre-reset +# connection; re-establishing the kind-6 block-sync peer can take until the +# Zakura app idle reaper releases that stale connection (bounded by the ~150s +# idle window, but variable). Recovery is reliable but not fast, so the 120s +# READY_TIMEOUT used for the (fast) startup assertions is too tight here. This +# ceiling only matters on failure: the waits exit as soon as catch-up starts. +CATCHUP_TIMEOUT="${CATCHUP_TIMEOUT:-${DEFAULT_CATCHUP_TIMEOUT}}" +CHECKPOINT_INTERVAL="${CHECKPOINT_INTERVAL:-${DEFAULT_CHECKPOINT_INTERVAL}}" +RUN_LABEL="${ZAKURA_REGTEST_E2E_LABEL:-zakura-${ZAKURA_E2E_MODE}}" command -v docker >/dev/null || fail "docker is required" command -v jq >/dev/null || fail "jq is required to parse RPC responses" +command -v python3 >/dev/null || fail "python3 is required to run the trace oracle" # Ensure a host-built zebrad binary exists; build it (debug) if not. ZEBRAD_BIN="${ZEBRAD_BIN:-${REPO_DIR}/target/debug/zebrad}" @@ -61,28 +148,117 @@ if [[ ! -x "${ZEBRAD_BIN}" ]]; then fi [[ -x "${ZEBRAD_BIN}" ]] || fail "zebrad binary not found at ${ZEBRAD_BIN}" export ZEBRAD_BIN -ZAKURA_E2E_TRACE_DIR="${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces}" +ZAKURA_E2E_TRACE_DIR="${ZAKURA_E2E_TRACE_DIR:-/tmp/zakura-regtest-e2e-traces-${RUN_LABEL}}" export ZAKURA_E2E_TRACE_DIR mkdir -p \ "${ZAKURA_E2E_TRACE_DIR}/node1" \ "${ZAKURA_E2E_TRACE_DIR}/node2" \ "${ZAKURA_E2E_TRACE_DIR}/node3" \ "${ZAKURA_E2E_TRACE_DIR}/node4" +TIMELINE_FILE="${ZAKURA_E2E_TRACE_DIR}/timeline.jsonl" +CONFIG_DIR="$(mktemp -d "${TMPDIR:-/tmp}/zakura-regtest-e2e-configs.XXXXXX")" +for node in 1 2 3 4; do + cp "${SCRIPT_DIR}/node${node}.toml" "${CONFIG_DIR}/node${node}.toml" +done +if [[ "${ZAKURA_E2E_DISABLE_CHECKPOINTS}" == "1" ]]; then + sed -i \ + 's|^network = "Regtest"$|network = { params = { checkpoints = false } }|' \ + "${CONFIG_DIR}/node2.toml" + grep -q '^network = { params = { checkpoints = false } }' "${CONFIG_DIR}/node2.toml" \ + || fail "failed to disable node2 Regtest checkpoints" +fi +if [[ "${ZAKURA_E2E_RESTART_MATRIX}" == "1" ]]; then + sed -i \ + 's|^ephemeral = true$|cache_dir = "/tmp/zakura-node2-state"\nephemeral = false|' \ + "${CONFIG_DIR}/node2.toml" + grep -q '^ephemeral = false$' "${CONFIG_DIR}/node2.toml" \ + || fail "failed to make node2 state persistent for restart-matrix" +fi +# node2 runs with the default header-sync config (accept_new_blocks = true). We no +# longer try to force a body gap by suppressing tip-flood acceptance — that never +# worked, because node2 still fills bodies via the inbound advertisement -> download +# path regardless of that flag. kind-6 block sync is instead exercised by the +# from-scratch reset phase below, which removes gossip as a source entirely. +export ZAKURA_NODE1_CONFIG="${CONFIG_DIR}/node1.toml" +export ZAKURA_NODE2_CONFIG="${CONFIG_DIR}/node2.toml" +export ZAKURA_NODE3_CONFIG="${CONFIG_DIR}/node3.toml" +export ZAKURA_NODE4_CONFIG="${CONFIG_DIR}/node4.toml" + +log "run mode: ${ZAKURA_E2E_MODE} (${RUN_LABEL})" log "using zebrad binary: ${ZEBRAD_BIN}" log "writing Zakura traces under: ${ZAKURA_E2E_TRACE_DIR}" +ORACLE_RAN=0 + +trace_dir_has_jsonl() { + [[ -d "${ZAKURA_E2E_TRACE_DIR}" ]] \ + && find "${ZAKURA_E2E_TRACE_DIR}"/node* -maxdepth 1 -type f -name '*.jsonl' -print -quit 2>/dev/null | grep -q . +} + +assert_trace_layout() { + local missing=0 + for file in \ + node1/commit_state.jsonl \ + node1/block_sync.jsonl \ + node1/header_sync.jsonl \ + node2/commit_state.jsonl \ + node2/block_sync.jsonl \ + node2/header_sync.jsonl \ + node4/commit_state.jsonl \ + node4/block_sync.jsonl \ + node4/header_sync.jsonl + do + if [[ ! -s "${ZAKURA_E2E_TRACE_DIR}/${file}" ]]; then + printf ' missing expected trace file: %s\n' "${ZAKURA_E2E_TRACE_DIR}/${file}" >&2 + missing=1 + fi + done + if [[ "${missing}" != "0" ]]; then + log "trace directory contents" + find "${ZAKURA_E2E_TRACE_DIR}" -maxdepth 3 -type f -print | sort >&2 || true + fail "Zakura traces were not written to the expected node*/ layout" + fi +} + +run_trace_oracle() { + trace_dir_has_jsonl || return 0 + ORACLE_RAN=1 + log "running Zakura trace oracle" + local oracle_args=( + "--commit-elapsed-ms" "${ZAKURA_E2E_ORACLE_COMMIT_ELAPSED_MS:-1800000}" + "--persistent-lag-seconds" "${ZAKURA_E2E_ORACLE_PERSISTENT_LAG_SECONDS:-180}" + "--handoff-stall-seconds" "${ZAKURA_E2E_ORACLE_HANDOFF_STALL_SECONDS:-180}" + ) + if [[ "${ZAKURA_E2E_REQUIRE_HANDOFF}" == "1" ]]; then + oracle_args+=("--require-handoff-boundary") + fi + python3 "${SCRIPT_DIR}/trace_oracle.py" "${oracle_args[@]}" "${ZAKURA_E2E_TRACE_DIR}" +} + cleanup() { + local status=$? + set +e + snapshot_timeline "cleanup" || true + if [[ "${ORACLE_RAN}" != "1" ]]; then + run_trace_oracle + fi log "node logs (tail)" docker compose -f "${COMPOSE_FILE}" logs --tail=30 || true + if [[ -d "${ZAKURA_E2E_TRACE_DIR}" ]]; then + docker compose -f "${COMPOSE_FILE}" logs --no-color --timestamps \ + > "${ZAKURA_E2E_TRACE_DIR}/docker-compose.log" 2>&1 || true + fi log "tearing down" docker compose -f "${COMPOSE_FILE}" down --remove-orphans --timeout 5 || true + rm -rf "${CONFIG_DIR}" + return "${status}" } trap cleanup EXIT -# rpc [json-params] -> raw JSON-RPC response +# rpc [json-params] [max-time-seconds] -> raw JSON-RPC response rpc() { - local port="$1" method="$2" params="${3:-[]}" - curl -s --max-time 10 -H 'content-type: application/json' \ + local port="$1" method="$2" params="${3:-[]}" max_time="${4:-10}" + curl -s --max-time "${max_time}" -H 'content-type: application/json' \ --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"${method}\",\"params\":${params}}" \ "http://127.0.0.1:${port}/" } @@ -94,9 +270,86 @@ metric() { | awk -v n="${name}" '$1==n {v=$2} END {print (v==""?0:v)}' } -wait_metric_at_least() { - local port="$1" name="$2" want="$3" label="$4" deadline=$((SECONDS + READY_TIMEOUT)) +timeline_node_snapshot() { + local phase="$1" node="$2" rpc_port="$3" metrics_port="$4" + local rpc_height best_header verified zakura_peers legacy_peers + local requests bodies_received bodies_served budget reorder applying outstanding + + rpc_height=$(block_count "${rpc_port}" 2>/dev/null || printf '0') + best_header=$(metric "${metrics_port}" sync_block_best_header_tip_height) + verified=$(metric "${metrics_port}" sync_block_verified_tip_height) + zakura_peers=$(metric "${metrics_port}" zakura_p2p_conn_active) + legacy_peers=$(peer_count "${rpc_port}" 2>/dev/null || printf '0') + requests=$(metric "${metrics_port}" sync_block_request_sent) + bodies_received=$(metric "${metrics_port}" sync_block_body_received) + bodies_served=$(metric "${metrics_port}" sync_block_body_served) + budget=$(metric "${metrics_port}" sync_block_budget_reserved_bytes) + reorder=$(metric "${metrics_port}" sync_block_reorder_buffered_bytes) + applying=$(metric "${metrics_port}" sync_block_applying) + outstanding=$(metric "${metrics_port}" sync_block_outstanding) + + jq -n -c \ + --arg phase "${phase}" \ + --arg mode "${ZAKURA_E2E_MODE}" \ + --arg node "${node}" \ + --argjson seconds "${SECONDS}" \ + --argjson rpc_height "${rpc_height}" \ + --argjson best_header_tip "${best_header}" \ + --argjson verified_body_tip "${verified}" \ + --argjson active_zakura_peers "${zakura_peers}" \ + --argjson legacy_peer_count "${legacy_peers}" \ + --argjson block_sync_request_sent "${requests}" \ + --argjson block_sync_body_received "${bodies_received}" \ + --argjson block_sync_body_served "${bodies_served}" \ + --argjson budget_reserved_bytes "${budget}" \ + --argjson reorder_buffered_bytes "${reorder}" \ + --argjson applying "${applying}" \ + --argjson outstanding "${outstanding}" \ + '{ + phase: $phase, + mode: $mode, + seconds: $seconds, + node: $node, + rpc_height: $rpc_height, + best_header_tip: $best_header_tip, + verified_body_tip: $verified_body_tip, + active_zakura_peers: $active_zakura_peers, + legacy_peer_count: $legacy_peer_count, + block_sync_request_sent: $block_sync_request_sent, + block_sync_body_received: $block_sync_body_received, + block_sync_body_served: $block_sync_body_served, + budget_reserved_bytes: $budget_reserved_bytes, + reorder_buffered_bytes: $reorder_buffered_bytes, + applying: $applying, + outstanding: $outstanding + }' >> "${TIMELINE_FILE}" +} + +snapshot_timeline() { + local phase="$1" + timeline_node_snapshot "${phase}" node1 18232 19001 || true + timeline_node_snapshot "${phase}" node2 18332 19002 || true + timeline_node_snapshot "${phase}" node3 18432 19003 || true + timeline_node_snapshot "${phase}" node4 18532 19004 || true +} + +wait_metric_zero() { + local port="$1" name="$2" label="$3" deadline=$((SECONDS + READY_TIMEOUT)) local value + while (( SECONDS < deadline )); do + value=$(metric "${port}" "${name}") + printf ' %s %s=%s (want 0)\n' "${label}" "${name}" "${value}" + if awk "BEGIN{exit !(${value} == 0)}"; then + return 0 + fi + sleep 3 + done + fail "${label} ${name} did not return to zero within ${READY_TIMEOUT}s" +} + +wait_metric_at_least() { + local port="$1" name="$2" want="$3" label="$4" timeout="${5:-${READY_TIMEOUT}}" + local deadline=$((SECONDS + timeout)) value while (( SECONDS < deadline )); do value=$(metric "${port}" "${name}") printf ' %s %s=%s (want >= %s)\n' "${label}" "${name}" "${value}" "${want}" @@ -105,7 +358,7 @@ wait_metric_at_least() { fi sleep 3 done - fail "${label} ${name} stayed below ${want} within ${READY_TIMEOUT}s" + fail "${label} ${name} stayed below ${want} within ${timeout}s" } wait_ready() { @@ -119,8 +372,182 @@ wait_ready() { fail "${name} RPC did not become ready within ${READY_TIMEOUT}s" } +reset_node2_from_scratch() { + local label="$1" + docker compose -f "${COMPOSE_FILE}" stop zakura-node-2 \ + || fail "could not stop node2 for ${label}" + if [[ "${ZAKURA_E2E_RESTART_MATRIX}" == "1" ]]; then + docker compose -f "${COMPOSE_FILE}" rm -f zakura-node-2 \ + || fail "could not remove node2 container for ${label}" + docker compose -f "${COMPOSE_FILE}" up -d zakura-node-2 \ + || fail "could not recreate node2 for ${label}" + else + docker compose -f "${COMPOSE_FILE}" start zakura-node-2 \ + || fail "could not restart node2 for ${label}" + fi + wait_ready 18332 "node2 (${label})" +} + +restart_node2_preserving_state() { + local label="$1" + docker compose -f "${COMPOSE_FILE}" stop zakura-node-2 \ + || fail "could not stop node2 for ${label}" + docker compose -f "${COMPOSE_FILE}" start zakura-node-2 \ + || fail "could not restart node2 for ${label}" + wait_ready 18332 "node2 (${label})" +} + peer_count() { rpc "$1" getpeerinfo | jq -r 'if .result then (.result | length) else 0 end'; } block_count() { rpc "$1" getblockcount | jq -r '.result // 0'; } +block_hash() { rpc "$1" getblockhash "[$2]" | jq -r '.result // empty'; } +strict_upgrade() { [[ "${ZAKURA_REGTEST_E2E_STRICT_UPGRADE:-0}" == "1" ]]; } + +invalidate_block_if_present() { + local port="$1" height="$2" hash="$3" label="$4" + local current_height current_hash + + current_height=$(block_count "${port}") + if [[ "${current_height}" -lt "${height}" ]]; then + printf ' %s skipping invalidate: height=%s below %s\n' \ + "${label}" "${current_height}" "${height}" + return 0 + fi + + current_hash=$(block_hash "${port}" "${height}") + if [[ "${current_hash}" != "${hash}" ]]; then + printf ' %s skipping invalidate: hash mismatch at height %s (have %s, want %s)\n' \ + "${label}" "${height}" "${current_hash}" "${hash}" + return 0 + fi + + rpc "${port}" invalidateblock "[\"${hash}\"]" | jq -e '.error == null' >/dev/null \ + || fail "invalidateblock failed on RPC port ${port}" +} + +wait_block_count_at_least() { + local port="$1" want="$2" label="$3" timeout="${4:-${PROPAGATE_TIMEOUT}}" + local deadline=$((SECONDS + timeout)) height + while (( SECONDS < deadline )); do + height=$(block_count "${port}") + printf ' %s height=%s (want >= %s)\n' "${label}" "${height}" "${want}" + [[ "${height}" -ge "${want}" ]] && return 0 + sleep 3 + done + fail "${label} did not reach height ${want}" +} + +wait_block_count_equal() { + local port="$1" want="$2" label="$3" deadline=$((SECONDS + READY_TIMEOUT)) + local height + while (( SECONDS < deadline )); do + height=$(block_count "${port}") + printf ' %s height=%s (want %s)\n' "${label}" "${height}" "${want}" + [[ "${height}" -eq "${want}" ]] && return 0 + sleep 3 + done + fail "${label} did not settle at height ${want}" +} + +wait_zakura_body_frontier_at_tip() { + local metrics_port="$1" rpc_port="$2" target="$3" label="$4" timeout="${5:-${PROPAGATE_TIMEOUT}}" + local deadline=$((SECONDS + timeout)) header body rpc_height + while (( SECONDS < deadline )); do + header=$(metric "${metrics_port}" sync_block_best_header_tip_height) + body=$(metric "${metrics_port}" sync_block_verified_tip_height) + rpc_height=$(block_count "${rpc_port}") + printf ' %s body_tip=%s header_tip=%s rpc_height=%s (target %s)\n' \ + "${label}" "${body}" "${header}" "${rpc_height}" "${target}" + if awk "BEGIN{exit !(${header} >= ${target} && ${body} == ${header} && ${rpc_height} >= ${target})}"; then + return 0 + fi + sleep 3 + done + fail "${label} did not reach body frontier == header tip at target ${target}" +} + +wait_zakura_body_frontiers_at_tip() { + local target="$1" phase="$2" + wait_zakura_body_frontier_at_tip 19001 18232 "${target}" "node1 ${phase}" + wait_zakura_body_frontier_at_tip 19002 18332 "${target}" "node2 ${phase}" + if strict_upgrade; then + wait_zakura_body_frontier_at_tip 19004 18532 "${target}" "node4 ${phase}" + else + h4=$(block_count 18532) + header4=$(metric 19004 sync_block_best_header_tip_height) + body4=$(metric 19004 sync_block_verified_tip_height) + printf ' node4 %s optional body_tip=%s header_tip=%s rpc_height=%s (target %s)\n' \ + "${phase}" "${body4}" "${header4}" "${h4}" "${target}" + fi +} + +assert_block_sync_budget_empty() { + local phase="$1" + wait_metric_zero 19001 sync_block_budget_reserved_bytes "node1 ${phase} budget" + wait_metric_zero 19002 sync_block_budget_reserved_bytes "node2 ${phase} budget" + wait_metric_zero 19001 sync_block_reorder_buffered_bytes "node1 ${phase} reorder" + wait_metric_zero 19002 sync_block_reorder_buffered_bytes "node2 ${phase} reorder" + wait_metric_zero 19001 sync_block_applying "node1 ${phase} applying" + wait_metric_zero 19002 sync_block_applying "node2 ${phase} applying" + wait_metric_zero 19001 sync_block_outstanding "node1 ${phase} outstanding" + wait_metric_zero 19002 sync_block_outstanding "node2 ${phase} outstanding" + if strict_upgrade; then + wait_metric_zero 19004 sync_block_budget_reserved_bytes "node4 ${phase} budget" + wait_metric_zero 19004 sync_block_reorder_buffered_bytes "node4 ${phase} reorder" + wait_metric_zero 19004 sync_block_applying "node4 ${phase} applying" + wait_metric_zero 19004 sync_block_outstanding "node4 ${phase} outstanding" + else + printf ' node4 %s optional budget=%s reorder=%s applying=%s outstanding=%s\n' \ + "${phase}" \ + "$(metric 19004 sync_block_budget_reserved_bytes)" \ + "$(metric 19004 sync_block_reorder_buffered_bytes)" \ + "$(metric 19004 sync_block_applying)" \ + "$(metric 19004 sync_block_outstanding)" + fi + snapshot_timeline "${phase}-cleanup" +} + +restart_node2_at_height_then_catch_up() { + local restart_height="$1" label="$2" target="$3" + log "restart-matrix: resetting node2, restarting around height ${restart_height} (${label})" + reset_node2_from_scratch "restart-matrix ${label} reset" + if (( restart_height > 0 )); then + wait_block_count_at_least 18332 "${restart_height}" "node2 restart-matrix ${label} pre-restart" "${CATCHUP_TIMEOUT}" + else + local n2_genesis="" + local deadline=$((SECONDS + READY_TIMEOUT)) + while (( SECONDS < deadline )); do + n2_genesis=$(block_hash 18332 0) + printf ' node2 restart-matrix %s genesis=%s\n' "${label}" "${n2_genesis:-}" + [[ -n "${n2_genesis}" ]] && break + sleep 3 + done + [[ -n "${n2_genesis}" ]] || fail "node2 did not bootstrap genesis before ${label} restart" + fi + + restart_node2_preserving_state "restart-matrix ${label}" + if (( $(block_count 18332) < target )); then + wait_metric_at_least 19002 sync_block_request_sent 1 "node2 restart-matrix ${label}" "${CATCHUP_TIMEOUT}" + fi + wait_block_count_at_least 18332 "${target}" "node2 restart-matrix ${label} catch-up" "${CATCHUP_TIMEOUT}" + wait_zakura_body_frontier_at_tip 19002 18332 "${target}" "node2 restart-matrix ${label}" "${CATCHUP_TIMEOUT}" + assert_block_sync_budget_empty "restart-matrix ${label}" +} + +run_restart_matrix() { + local target="$1" + local around_399=$(( target > 399 ? 399 : target )) + local around_400=$(( target > 400 ? 400 : target )) + local around_401=$(( target > 401 ? 401 : target )) + local around_mid=$(( target > 2000 ? 2000 : target / 2 )) + local near_tip_gap_height=$(( target > 1000 ? target - 1000 : 0 )) + + restart_node2_at_height_then_catch_up 0 "height-0" "${target}" + restart_node2_at_height_then_catch_up "${around_399}" "height-399" "${target}" + restart_node2_at_height_then_catch_up "${around_400}" "height-400" "${target}" + restart_node2_at_height_then_catch_up "${around_401}" "height-401" "${target}" + restart_node2_at_height_then_catch_up "${around_mid}" "height-2000" "${target}" + restart_node2_at_height_then_catch_up "${near_tip_gap_height}" "near-tip-1000-gap" "${target}" +} log "starting cluster (host-built binary, no image build)" docker compose -f "${COMPOSE_FILE}" up -d @@ -130,6 +557,7 @@ wait_ready 18232 node1 wait_ready 18332 node2 wait_ready 18432 node3 wait_ready 18532 node4 +snapshot_timeline "rpc-ready" log "asserting legacy TCP backwards-compat (legacy-only node3 peers with node1)" # node3 speaks only the legacy protocol, so it stays a legacy TCP peer of node1. @@ -144,6 +572,7 @@ while (( SECONDS < deadline )); do sleep 3 done [[ "${n3}" -ge 1 ]] || fail "legacy-only node3 never peered with node1 over legacy TCP" +snapshot_timeline "legacy-peers-ready" log "asserting pure-Zakura node2 has no legacy peers (legacy_p2p = false)" # node2 has no legacy stack at all, so it must never have a legacy TCP peer; its @@ -165,18 +594,56 @@ while (( SECONDS < deadline )); do done [[ "${upgraded}" -eq 1 ]] || fail \ "node1 and node4 never upgraded their legacy connection to Zakura" +snapshot_timeline "zakura-upgrade-ready" log "asserting live Zakura peer readiness" # The upgrade metric above is a historical counter. Wait for the live peer gauge # before mining so propagation assertions exercise an active Zakura path. wait_metric_at_least 19002 zakura_p2p_conn_active 1 node2 -if [[ "${ZAKURA_REGTEST_E2E_STRICT_UPGRADE:-0}" == "1" ]]; then +if strict_upgrade; then wait_metric_at_least 19004 zakura_p2p_conn_active 1 node4 fi +# Prove the pure-Zakura node bootstrapped genesis over Zakura. +# +# node2 sets sync.debug_skip_regtest_genesis_self_seed, so it does NOT commit the +# Regtest genesis locally. With no legacy stack, its only way to obtain genesis is +# to download and verify it from node1 over Zakura (the bootstrap_genesis_then_pause +# path). Until that succeeds, native header sync cannot anchor at genesis and the +# node stays stuck at height 0 -- the exact Mainnet Zakura-only bug this guards +# against. This assertion runs before any block is mined, so node1 is still at the +# genesis-only tip and node2 must reach height 0 purely by fetching genesis. +log "asserting pure-Zakura node2 bootstrapped genesis over Zakura (self-seed disabled)" +genesis_hash=$(block_hash 18232 0) +[[ -n "${genesis_hash}" ]] || fail "could not read Regtest genesis hash from node1" +deadline=$((SECONDS + READY_TIMEOUT)); n2_genesis="" +while (( SECONDS < deadline )); do + n2_genesis=$(block_hash 18332 0) + printf ' node2 genesis(height 0)=%s (want %s)\n' "${n2_genesis:-}" "${genesis_hash}" + [[ "${n2_genesis}" == "${genesis_hash}" ]] && break + sleep 3 +done +[[ "${n2_genesis}" == "${genesis_hash}" ]] || fail \ + "pure-Zakura node2 never committed genesis over Zakura (self-seed disabled); genesis bootstrap is broken" +# Confirm genesis arrived via a real download+verify, not a self-seed that silently +# ignored the flag. `request_genesis` only logs the download-start line when genesis +# is ABSENT at startup, so this line cannot appear on a self-seeding node -- it +# proves node2 fetched genesis from a peer over Zakura. +deadline=$((SECONDS + READY_TIMEOUT)); n2_bootstrapped=0 +while (( SECONDS < deadline )); do + if docker compose -f "${COMPOSE_FILE}" logs zakura-node-2 2>/dev/null \ + | grep -q "starting genesis block download and verify"; then + n2_bootstrapped=1; break + fi + sleep 2 +done +[[ "${n2_bootstrapped}" -eq 1 ]] || fail \ + "node2 committed genesis but never logged a genesis download; the self-seed shortcut may have run instead of a real over-Zakura fetch" +snapshot_timeline "genesis-bootstrap" + log "generating ${GENERATE_BLOCKS} block(s) on node1" for ((i = 1; i <= GENERATE_BLOCKS; i++)); do - if [[ "${ZAKURA_REGTEST_E2E_STRICT_UPGRADE:-0}" == "1" ]]; then + if strict_upgrade; then wait_metric_at_least 19004 zakura_p2p_conn_active 1 "node4 before block ${i}" fi @@ -190,7 +657,7 @@ for ((i = 1; i <= GENERATE_BLOCKS; i++)); do # advertisements. Mine them one at a time so a node with one in-flight # download from a peer does not intentionally ignore the next advertisement # from that same peer before it has accepted the first block. - if (( i < GENERATE_BLOCKS )) && [[ "${ZAKURA_REGTEST_E2E_STRICT_UPGRADE:-0}" == "1" ]]; then + if (( i < GENERATE_BLOCKS )) && strict_upgrade; then deadline=$((SECONDS + PROPAGATE_TIMEOUT)) while (( SECONDS < deadline )); do h4=$(block_count 18532) @@ -201,7 +668,7 @@ for ((i = 1; i <= GENERATE_BLOCKS; i++)); do done if [[ "${h4}" -lt "${target}" ]]; then - if [[ "${ZAKURA_REGTEST_E2E_STRICT_UPGRADE:-0}" == "1" ]]; then + if strict_upgrade; then fail "upgraded dual-stack node4 did not ingest generated block ${i} before the next block (got ${h4}, want ${target})" fi @@ -210,6 +677,7 @@ for ((i = 1; i <= GENERATE_BLOCKS; i++)); do fi fi done +snapshot_timeline "post-generate-mining" log "asserting block propagation to node2 (pure Zakura), node3 (legacy TCP), and checking node4 (known upgraded-Zakura issue)" deadline=$((SECONDS + PROPAGATE_TIMEOUT)) @@ -217,7 +685,11 @@ while (( SECONDS < deadline )); do h2=$(block_count 18332); h3=$(block_count 18432); h4=$(block_count 18532) printf ' node2 height=%s node3 height=%s node4 height=%s (target %s)\n' \ "${h2}" "${h3}" "${h4}" "${target}" - [[ "${h2}" -ge "${target}" && "${h3}" -ge "${target}" && "${h4}" -ge "${target}" ]] && break + if strict_upgrade; then + [[ "${h2}" -ge "${target}" && "${h3}" -ge "${target}" && "${h4}" -ge "${target}" ]] && break + else + [[ "${h2}" -ge "${target}" && "${h3}" -ge "${target}" ]] && break + fi sleep 3 done [[ "${h2}" -ge "${target}" ]] || fail \ @@ -225,7 +697,7 @@ done [[ "${h3}" -ge "${target}" ]] || fail \ "block did not propagate to legacy-only node3 over TCP (got ${h3}, want ${target})" if [[ "${h4}" -lt "${target}" ]]; then - if [[ "${ZAKURA_REGTEST_E2E_STRICT_UPGRADE:-0}" == "1" ]]; then + if strict_upgrade; then fail "block did not propagate to upgraded dual-stack node4 over the Zakura adapter (got ${h4}, want ${target})" fi @@ -235,4 +707,177 @@ else printf ' node4 upgraded-Zakura propagation reached height=%s\n' "${h4}" fi -log "PASS: legacy coexistence + Zakura upgrade handshake + pure-Zakura and legacy block propagation verified" +log "asserting Zakura body frontier reached the header tip after gossip propagation" +wait_zakura_body_frontiers_at_tip "${target}" "post-generate" +assert_block_sync_budget_empty "post-generate" +snapshot_timeline "post-generate" + +# --------------------------------------------------------------------------- +# Deepen the chain before the from-scratch reset so the kind-6 catch-up below +# re-downloads many bodies in a burst. Crossing hundreds of blocks is what fills +# the inbound block-sync wire queue and exercises the body-flood path that +# wedged in production (a full queue silently dropping solicited bodies, then a +# checkpoint-range commit waiting forever on the gap). A 3-block catch-up never +# fills that queue, so the earlier topology could not reproduce the stall. node2 +# is still connected and tracks these via gossip, but it is wiped by the reset +# below, forcing a real kind-6 re-download of the whole deepened chain. +if (( CATCHUP_BLOCKS > 0 )); then + log "deepening node1 chain by ${CATCHUP_BLOCKS} block(s) before the reset catch-up" + remaining=${CATCHUP_BLOCKS} + while (( remaining > 0 )); do + batch=$(( remaining < 50 ? remaining : 50 )) + # `generate` mines sequentially (~0.25s/block on a debug build), so a 50-block batch can + # exceed the default 10s RPC deadline — give it a generous, batch-scaled timeout. + rpc 18232 generate "[${batch}]" "$(( batch * 4 + 30 ))" \ + | jq -e ".result | length == ${batch}" >/dev/null \ + || fail "bulk generate of ${batch} block(s) failed on node1" + remaining=$(( remaining - batch )) + printf ' mined batch of %s; node1 height=%s (%s remaining)\n' \ + "${batch}" "$(block_count 18232)" "${remaining}" + done + # Make the deepened tip the working target so the from-scratch catch-up spans + # the whole chain and the trailing reorg stays a cheap one-block tip reorg + # rather than unwinding everything mined here. + target=$(block_count 18232) + log "waiting for node1 body frontier to settle at the deepened tip ${target}" + wait_zakura_body_frontier_at_tip 19001 18232 "${target}" "node1 deepened" + snapshot_timeline "deepened" +fi + +# --------------------------------------------------------------------------- +# Exercise kind-6 block sync via a from-scratch reset of the pure-Zakura node. +# +# Above, node2 reached the tip while connected — but it got there through inbound +# gossip (advertisement -> download), not kind-6 block sync: in this tiny topology +# gossip delivers every body the instant it is mined, so no body gap ever forms. +# The only way to force kind-6 is to remove gossip as a source. node2 has ephemeral +# state, so stopping its container discards its chain; node1 keeps the chain and +# mines nothing more. On reconnect node2 has a real, gossip-unfillable gap (node1 +# re-advertises nothing), so it must re-download every existing block over the +# dedicated block-sync stream. This is the production Mainnet-from-0 / +# restart-catch-up path. +log "resetting pure-Zakura node2 to force a from-scratch kind-6 catch-up" +catchup_target=$(block_count 18232) +[[ "${catchup_target}" -ge 1 ]] || fail \ + "node1 has no chain for node2 to catch up to (height ${catchup_target})" +before_node1_served=$(metric 19001 sync_block_body_served) +snapshot_timeline "pre-reset-catch-up" + +# --------------------------------------------------------------------------- +# Derive a Regtest checkpoint list from node1's chain and rewrite node2's config before the +# restart, so the from-scratch catch-up below verifies through real checkpoint ranges +# (batch-commit). That is the production path Regtest's genesis-only checkpoint list cannot +# exercise, and the exact shape of the "drop-through" wedge: a body missing inside a +# checkpoint range stalls that range's indefinite-wait commit. Hashes are only known once +# mined, so this runs after the deepening. Only `checkpoints` is overridden — Regtest +# genesis/magic/PoW are preserved (see build_regtest_params in zebra-network), so node2 still +# peers with node1; checkpoint_sync defaults to true, so node2 verifies the whole range. The +# config shape is locked by zebra-network's +# `configured_regtest_checkpoints_preserve_regtest_identity` unit test. +# Keep the highest checkpoint strictly below the tip so the trailing tip reorg later is never +# blocked by a final (immutable) checkpoint. +checkpoint_ceiling=$(( catchup_target - 2 )) +if [[ "${ZAKURA_E2E_DISABLE_CHECKPOINTS}" == "1" ]]; then + log "node2 Regtest checkpoints are disabled; catch-up verifies through the full verifier after genesis" +elif (( CHECKPOINT_INTERVAL > 0 && checkpoint_ceiling >= CHECKPOINT_INTERVAL )); then + # block::Hash deserializes as a 32-byte array in internal (display-reversed) order, so + # convert each getblockhash hex into a reversed decimal byte array, e.g. + # "029f..e327" -> [39, 227, ..., 2]. + hash_to_internal_bytes() { + local hex="$1" out="" i + (( ${#hex} == 64 )) || fail "unexpected block hash length for '${hex}'" + for (( i = 62; i >= 0; i -= 2 )); do + out+="$(( 16#${hex:i:2} )), " + done + printf '[%s]' "${out%, }" + } + + cp_entries="" + cp_count=0 + h=0 + while (( h <= checkpoint_ceiling )); do + cp_hash=$(block_hash 18232 "${h}") + [[ -n "${cp_hash}" ]] || fail "could not read node1 block hash at checkpoint height ${h}" + [[ -n "${cp_entries}" ]] && cp_entries+=", " + cp_entries+="[${h}, $(hash_to_internal_bytes "${cp_hash}")]" + cp_count=$(( cp_count + 1 )) + h=$(( h + CHECKPOINT_INTERVAL )) + done + + # Replace node2's plain `network = "Regtest"` with a ConfiguredRegtest inline table carrying + # the derived checkpoints (the deserializer matches ConfiguredRegtest by the `params` key). + sed -i \ + "s|^network = \"Regtest\"\$|network = { params = { checkpoints = [${cp_entries}] } }|" \ + "${CONFIG_DIR}/node2.toml" + grep -q '^network = { params = ' "${CONFIG_DIR}/node2.toml" \ + || fail "failed to inject derived checkpoints into node2 config" + log "node2 will checkpoint-verify its kind-6 catch-up against ${cp_count} derived checkpoint(s) up to height ${checkpoint_ceiling}" +else + log "chain too short (tip ${catchup_target}, interval ${CHECKPOINT_INTERVAL}); node2 catches up with the genesis-only checkpoint list" +fi + +reset_node2_from_scratch "post-reset catch-up" +snapshot_timeline "post-reset-catch-up-started" + +# node2's own counters restart from zero, so assert absolute kind-6 activity, not a +# delta. With node1 idle at the tip, reaching catchup_target from an empty state is +# only possible by downloading every body over block sync. +wait_metric_at_least 19002 sync_block_request_sent 1 "node2 catch-up" "${CATCHUP_TIMEOUT}" +wait_metric_at_least 19002 sync_block_body_received 1 "node2 catch-up" "${CATCHUP_TIMEOUT}" +wait_block_count_at_least 18332 "${catchup_target}" "node2 catch-up" "${CATCHUP_TIMEOUT}" +wait_zakura_body_frontier_at_tip 19002 18332 "${catchup_target}" "node2 catch-up" "${CATCHUP_TIMEOUT}" + +# node2 dials only node1, so node1 must have served the catch-up bodies over kind-6. +after_node1_served=$(metric 19001 sync_block_body_served) +printf ' node1 kind-6 bodies served=%s (before reset %s)\n' \ + "${after_node1_served}" "${before_node1_served}" +if ! awk "BEGIN{exit !(${after_node1_served} > ${before_node1_served})}"; then + fail "node2 caught up from scratch but node1's kind-6 body-served metric did not increase — bodies did not flow over block sync" +fi +assert_block_sync_budget_empty "post-catch-up" +log "node2 re-downloaded ${catchup_target} block(s) from scratch over kind-6 block sync" +snapshot_timeline "post-catch-up" + +if [[ "${ZAKURA_E2E_RESTART_MATRIX}" == "1" ]]; then + run_restart_matrix "${catchup_target}" +fi + +log "asserting non-finalized reorg survival with no block-sync budget leak" +old_tip_hash=$(block_hash 18232 "${target}") +[[ -n "${old_tip_hash}" ]] || fail "could not read old tip hash at height ${target}" +invalidate_block_if_present 18232 "${target}" "${old_tip_hash}" node1 +invalidate_block_if_present 18332 "${target}" "${old_tip_hash}" node2 +invalidate_block_if_present 18432 "${target}" "${old_tip_hash}" node3 +invalidate_block_if_present 18532 "${target}" "${old_tip_hash}" node4 +reorg_base=$((target - 1)) +wait_block_count_equal 18232 "${reorg_base}" node1 +wait_block_count_equal 18332 "${reorg_base}" node2 +if strict_upgrade; then + wait_block_count_equal 18532 "${reorg_base}" node4 +fi +rpc 18232 generate "[2]" | jq -e '.result | length == 2' >/dev/null \ + || fail "generate RPC failed on node1 after invalidating old tip" +target=$(block_count 18232) +wait_block_count_at_least 18332 "${target}" node2 +wait_block_count_at_least 18432 "${target}" node3 +if strict_upgrade; then + wait_block_count_at_least 18532 "${target}" node4 +fi +wait_zakura_body_frontiers_at_tip "${target}" "post-reorg" +wait_metric_at_least 19001 sync_block_reorg_reset 1 node1 +wait_metric_at_least 19002 sync_block_reorg_reset 1 node2 +assert_block_sync_budget_empty "post-reorg" +snapshot_timeline "post-reorg" + +if [[ "${ZAKURA_E2E_RESTART_MATRIX}" == "1" ]]; then + log "restart-matrix: restarting node2 after the non-finalized reorg" + restart_node2_preserving_state "restart-matrix post-reorg" + wait_block_count_at_least 18332 "${target}" "node2 restart-matrix post-reorg" "${CATCHUP_TIMEOUT}" + wait_zakura_body_frontier_at_tip 19002 18332 "${target}" "node2 restart-matrix post-reorg" "${CATCHUP_TIMEOUT}" + assert_block_sync_budget_empty "restart-matrix post-reorg" +fi + +assert_trace_layout +run_trace_oracle + +log "PASS (${RUN_LABEL}): Zakura body frontier, peer serving, reorg survival, and legacy compatibility verified" diff --git a/docker/zakura-regtest-e2e/trace_oracle.py b/docker/zakura-regtest-e2e/trace_oracle.py new file mode 100644 index 00000000000..f1cb3e46144 --- /dev/null +++ b/docker/zakura-regtest-e2e/trace_oracle.py @@ -0,0 +1,1372 @@ +#!/usr/bin/env python3 +"""Zakura regtest trace oracle. + +The oracle reads the per-node JSONL trace layout produced by the Docker e2e: + + node*/commit_state.jsonl + node*/block_sync.jsonl + node*/header_sync.jsonl + +It checks high-signal sync invariants and prints compact diagnostics for the +first offending row in each node. It intentionally depends only on Python's +standard library so it can run from CI failure traps. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + + +COMMIT_START = "commit_start" +COMMIT_STALLED = "commit_stalled" +COMMIT_FINISH = "commit_finish" +SYNC_FRONTIER_TRANSITION = "sync_frontier_transition" + +BLOCK_GET_BLOCKS_SENT = "block_get_blocks_sent" +BLOCK_BODY_RECEIVED = "block_body_received" +BLOCK_APPLY_FINISHED = "block_apply_finished" +BLOCK_FRONTIERS_CHANGED = "block_frontiers_changed" +BLOCK_CHAIN_TIP_RESET = "block_chain_tip_reset" +BLOCK_SYNC_STATE = "block_sync_state" + +HEADER_FRONTIER_ADVANCED = "header_frontier_advanced" +HEADER_FRONTIER_REANCHORED = "header_frontier_reanchored" +HEADER_GET_HEADERS_SENT = "header_get_headers_sent" +HEADER_RANGE_COMMITTED = "header_range_committed" +HEADER_RANGE_REJECTED = "header_range_rejected" +HEADER_PEER_VIOLATION = "header_peer_violation" + +QUERY_NEEDED_BLOCKS = "query_needed_blocks" + +RESET_CAUSES = {"verified_reset"} +BODY_PROGRESS_EVENTS = { + BLOCK_GET_BLOCKS_SENT, + BLOCK_BODY_RECEIVED, + BLOCK_APPLY_FINISHED, + BLOCK_FRONTIERS_CHANGED, + BLOCK_CHAIN_TIP_RESET, +} +RECENT_ACTIVITY_EVENTS = 2_000 +RECENT_ACTIVITY_MICROS = 180 * 1_000_000 +COMMIT_EVENT_WINDOW = 200_000 +COMMIT_MICROS_WINDOW = 30 * 60 * 1_000_000 +COMMIT_TREND_MIN_MS = 300_000 +COMMIT_TREND_FACTOR = 4.0 +APPLY_CLASS_CHECKPOINT = "checkpoint" +APPLY_CLASS_FULL = "full" + + +@dataclass(frozen=True) +class TraceRow: + node: str + table: str + index: int + row: dict[str, Any] + + @property + def event(self) -> str: + return str(self.row.get("event", "")) + + @property + def ts(self) -> int | None: + return int_field(self.row, "ts") + + +@dataclass +class Failure: + node: str + invariant: str + row: TraceRow | None + detail: dict[str, Any] + + +@dataclass(frozen=True) +class OracleOptions: + commit_event_window: int = COMMIT_EVENT_WINDOW + commit_elapsed_micros: int = COMMIT_MICROS_WINDOW + persistent_lag_micros: int = RECENT_ACTIVITY_MICROS + require_handoff_boundary: bool = False + handoff_stall_micros: int = RECENT_ACTIVITY_MICROS + commit_trend_min_ms: int = COMMIT_TREND_MIN_MS + commit_trend_factor: float = COMMIT_TREND_FACTOR + + +class NodeTrace: + def __init__(self, node: str, tables: dict[str, list[TraceRow]]) -> None: + self.node = node + self.tables = tables + self.rows = sorted( + [row for rows in tables.values() for row in rows], + key=lambda row: (row.ts if row.ts is not None else -1, row.table, row.index), + ) + + def table(self, name: str) -> list[TraceRow]: + return self.tables.get(name, []) + + def events(self, table: str, event: str) -> list[TraceRow]: + return [row for row in self.table(table) if row.event == event] + + def latest(self, table: str, event: str | None = None) -> TraceRow | None: + rows = self.table(table) + if event is not None: + rows = [row for row in rows if row.event == event] + return rows[-1] if rows else None + + +def int_field(row: dict[str, Any], field: str) -> int | None: + value = row.get(field) + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None + + +def row_key(row: TraceRow) -> tuple[str, Any] | None: + token = int_field(row.row, "apply_token") + if token is not None: + return ("token", token) + + height = int_field(row.row, "height") + block_hash = row.row.get("hash") + if height is not None and block_hash is not None: + return ("height_hash", (height, block_hash)) + + action = row.row.get("action") + range_start = int_field(row.row, "range_start") + range_count = int_field(row.row, "range_count") + if action is not None and range_start is not None and range_count is not None: + return ("action_range", (action, range_start, range_count)) + + return None + + +def compact_row(row: TraceRow | None) -> dict[str, Any] | None: + if row is None: + return None + + return { + "table": row.table, + "index": row.index, + "event": row.event, + "ts": row.ts, + "row": row.row, + } + + +def read_jsonl(path: Path, node: str, table: str) -> list[TraceRow]: + if not path.exists(): + return [] + + rows: list[TraceRow] = [] + with path.open("r", encoding="utf-8") as handle: + for index, line in enumerate(handle, start=1): + line = line.strip() + if not line: + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + rows.append( + TraceRow( + node, + table, + index, + { + "event": "json_decode_error", + "path": str(path), + "line": line[:400], + "error": str(error), + }, + ) + ) + continue + if isinstance(value, dict): + rows.append(TraceRow(node, table, index, value)) + return rows + + +def load_traces(root: Path) -> list[NodeTrace]: + nodes: list[NodeTrace] = [] + node_dirs = sorted(path for path in root.iterdir() if path.is_dir() and path.name.startswith("node")) + if not node_dirs and any(root.joinpath(name).exists() for name in trace_files()): + node_dirs = [root] + + for node_dir in node_dirs: + candidates = [node_dir] + if not any(node_dir.joinpath(name).exists() for name in trace_files()): + candidates = [ + child + for child in sorted(path for path in node_dir.iterdir() if path.is_dir()) + if any(child.joinpath(name).exists() for name in trace_files()) + ] + + for trace_dir in candidates: + node = node_dir.name if trace_dir == node_dir else f"{node_dir.name}/{trace_dir.name}" + tables = { + "commit_state": read_jsonl(trace_dir / "commit_state.jsonl", node, "commit_state"), + "block_sync": read_jsonl(trace_dir / "block_sync.jsonl", node, "block_sync"), + "header_sync": read_jsonl(trace_dir / "header_sync.jsonl", node, "header_sync"), + } + if any(tables.values()): + nodes.append(NodeTrace(node, tables)) + + return nodes + + +def trace_files() -> Iterable[str]: + yield "commit_state.jsonl" + yield "block_sync.jsonl" + yield "header_sync.jsonl" + + +def check_commit_pairs(node: NodeTrace, options: OracleOptions) -> list[Failure]: + failures: list[Failure] = [] + finishes: dict[tuple[str, Any], list[TraceRow]] = {} + for finish in node.events("commit_state", COMMIT_FINISH): + key = row_key(finish) + if key is not None: + finishes.setdefault(key, []).append(finish) + + for start in node.events("commit_state", COMMIT_START): + key = row_key(start) + if key is None: + failures.append( + failure(node, "commit_start_has_match_key", start, {"reason": "missing apply_token and height/hash"}) + ) + continue + + candidates = [ + finish + for finish in finishes.get(key, []) + if finish.index > start.index or later_ts(finish, start) + ] + if not candidates: + failures.append(failure(node, "commit_start_has_finish", start, {"key": key})) + continue + + finish = candidates[0] + if finish.index - start.index > options.commit_event_window: + failures.append( + failure( + node, + "commit_finish_within_event_window", + start, + {"key": key, "finish": compact_row(finish), "event_delta": finish.index - start.index}, + ) + ) + elif start.ts is not None and finish.ts is not None and finish.ts - start.ts > options.commit_elapsed_micros: + failures.append( + failure( + node, + "commit_finish_within_elapsed_window", + start, + {"key": key, "finish": compact_row(finish), "elapsed_us": finish.ts - start.ts}, + ) + ) + else: + elapsed_ms = int_field(finish.row, "elapsed_ms") + if elapsed_ms is not None and elapsed_ms * 1_000 > options.commit_elapsed_micros: + failures.append( + failure( + node, + "commit_finish_within_elapsed_window", + start, + {"key": key, "finish": compact_row(finish), "elapsed_ms": elapsed_ms}, + ) + ) + + for stalled in node.events("commit_state", COMMIT_STALLED): + key = row_key(stalled) + later_finish = [ + finish + for finish in finishes.get(key, []) + if key is not None and (finish.index > stalled.index or later_ts(finish, stalled)) + ] + if not later_finish: + failures.append(failure(node, "commit_stalled_not_terminal", stalled, {"key": key})) + + failures.extend(check_commit_latency_trend(node, options)) + return failures + + +def check_commit_latency_trend(node: NodeTrace, options: OracleOptions) -> list[Failure]: + failures: list[Failure] = [] + elapsed_by_class: dict[str, list[tuple[TraceRow, int]]] = {} + + for row in node.events("commit_state", COMMIT_FINISH): + elapsed_ms = int_field(row.row, "elapsed_ms") + if elapsed_ms is None: + continue + + commit_class = str(row.row.get("apply_class") or row.row.get("action") or "unknown") + elapsed_by_class.setdefault(commit_class, []).append((row, elapsed_ms)) + + for commit_class, samples in elapsed_by_class.items(): + if len(samples) < 3: + continue + + for previous, current in zip(samples, samples[1:]): + previous_row, previous_ms = previous + current_row, current_ms = current + if previous_ms <= 0: + continue + if current_ms < options.commit_trend_min_ms: + continue + if current_ms >= int(previous_ms * options.commit_trend_factor): + failures.append( + failure( + node, + "commit_latency_trend_bounded", + current_row, + { + "commit_class": commit_class, + "previous": compact_row(previous_row), + "previous_elapsed_ms": previous_ms, + "current_elapsed_ms": current_ms, + "trend_factor": options.commit_trend_factor, + }, + ) + ) + break + + first_row, first_ms = samples[0] + last_row, last_ms = samples[-1] + if ( + first_ms > 0 + and last_ms >= options.commit_trend_min_ms + and last_ms >= int(first_ms * options.commit_trend_factor) + and nondecreasing([elapsed_ms for _, elapsed_ms in samples]) + and not any( + f.invariant == "commit_latency_trend_bounded" + and f.detail.get("commit_class") == commit_class + for f in failures + ) + ): + failures.append( + failure( + node, + "commit_latency_trend_bounded", + last_row, + { + "commit_class": commit_class, + "first": compact_row(first_row), + "first_elapsed_ms": first_ms, + "last_elapsed_ms": last_ms, + "trend_factor": options.commit_trend_factor, + }, + ) + ) + + return failures + + +def nondecreasing(values: list[int]) -> bool: + return all(right >= left for left, right in zip(values, values[1:])) + + +def later_ts(candidate: TraceRow, previous: TraceRow) -> bool: + return ( + candidate.ts is not None + and previous.ts is not None + and candidate.ts >= previous.ts + and candidate is not previous + ) + + +def check_frontiers(node: NodeTrace) -> list[Failure]: + failures: list[Failure] = [] + for row in node.events("commit_state", SYNC_FRONTIER_TRANSITION): + old_finalized = int_field(row.row, "old_finalized_height") + new_finalized = int_field(row.row, "new_finalized_height") + if old_finalized is not None and new_finalized is not None and new_finalized < old_finalized: + failures.append( + failure( + node, + "finalized_height_monotonic", + row, + {"old_finalized_height": old_finalized, "new_finalized_height": new_finalized}, + ) + ) + + old_verified = int_field(row.row, "old_verified_body_height") + new_verified = int_field(row.row, "new_verified_body_height") + cause = str(row.row.get("cause", "")) + if ( + old_verified is not None + and new_verified is not None + and new_verified < old_verified + and cause not in RESET_CAUSES + ): + failures.append( + failure( + node, + "verified_body_height_monotonic_except_reset", + row, + { + "cause": cause, + "old_verified_body_height": old_verified, + "new_verified_body_height": new_verified, + }, + ) + ) + if cause == "header_reanchored": + old_header = int_field(row.row, "old_best_header_height") + new_header = int_field(row.row, "new_best_header_height") + if old_verified is not None and new_verified is not None and new_verified < old_verified: + failures.append(failure(node, "header_reanchor_did_not_lower_verified_body", row, {})) + if old_finalized is not None and new_finalized is not None and new_finalized < old_finalized: + failures.append(failure(node, "header_reanchor_did_not_lower_finalized", row, {})) + if old_header is not None and new_header is not None and new_header > old_header: + failures.append( + failure( + node, + "header_reanchor_only_lowers_best_header", + row, + {"old_best_header_height": old_header, "new_best_header_height": new_header}, + ) + ) + + return failures + + +def check_block_sync_activity(node: NodeTrace, options: OracleOptions) -> list[Failure]: + failures: list[Failure] = [] + states = node.events("block_sync", BLOCK_SYNC_STATE) + real_activity = sorted( + [row for row in node.table("block_sync") if row.event in BODY_PROGRESS_EVENTS] + + [row for row in node.table("commit_state") if commit_state_row_is_body_progress(row)], + key=sort_key, + ) + query_rows = [ + row + for row in node.table("commit_state") + if row.row.get("action") == QUERY_NEEDED_BLOCKS + and row.event in {"state_read_start", "state_read_success", "state_read_error", "state_read_timeout"} + ] + + for state in states: + best = int_field(state.row, "best_header_tip") + verified = int_field(state.row, "verified_block_tip") + if best is None or verified is None or best <= verified: + continue + + has_real_activity = has_near_activity(state, real_activity, options) + if not has_real_activity: + recent_query_count = len(rows_near(state, query_rows, options)) + invariant = ( + "lagging_body_sync_not_query_spin" + if recent_query_count > 0 + else "lagging_body_sync_has_real_activity" + ) + failures.append( + failure( + node, + invariant, + state, + { + "best_header_tip": best, + "verified_block_tip": verified, + "recent_query_needed_blocks": recent_query_count, + "nearby_real_activity": [compact_row(row) for row in rows_near(state, real_activity, options)[-5:]], + }, + ) + ) + break + + if body_sync_has_pinned_queue(state) and not has_forward_activity(state, real_activity, options): + failures.append( + failure( + node, + "block_sync_queue_lag_makes_progress", + state, + { + "best_header_tip": best, + "verified_block_tip": verified, + "needed_count": int_field(state.row, "needed_count"), + "queue_len": int_field(state.row, "queue_len"), + "assigned_len": int_field(state.row, "assigned_len"), + "outstanding": int_field(state.row, "outstanding"), + }, + ) + ) + break + + final_state = states[-1] if states else None + if final_state is not None: + leaked = { + name: int_field(final_state.row, name) + for name in ("budget_reserved", "reorder", "applying", "outstanding") + } + bad = {name: value for name, value in leaked.items() if value is not None and value != 0} + if bad: + failures.append(failure(node, "final_block_sync_state_has_no_leaks", final_state, {"leaked": bad})) + + return failures + + +def sort_key(row: TraceRow) -> tuple[int, str, int]: + return (row.ts if row.ts is not None else -1, row.table, row.index) + + +def commit_state_row_is_body_progress(row: TraceRow) -> bool: + if row.event == SYNC_FRONTIER_TRANSITION: + old_verified = int_field(row.row, "old_verified_body_height") + new_verified = int_field(row.row, "new_verified_body_height") + cause = str(row.row.get("cause", "")) + return ( + old_verified is not None + and new_verified is not None + and (new_verified > old_verified or cause in RESET_CAUSES) + ) + if row.event == COMMIT_FINISH and row.row.get("source") == "block_sync_driver": + return True + return False + + +def rows_near(state: TraceRow, activity: list[TraceRow], options: OracleOptions) -> list[TraceRow]: + state_ts = state.ts + if state_ts is None: + lower = max(0, state.index - RECENT_ACTIVITY_EVENTS) + upper = state.index + RECENT_ACTIVITY_EVENTS + return [row for row in activity if row.table != state.table or lower <= row.index <= upper] + + return [ + row + for row in activity + if row.ts is not None and abs(row.ts - state_ts) <= options.persistent_lag_micros + ] + + +def has_near_activity(state: TraceRow, activity: list[TraceRow], options: OracleOptions) -> bool: + return bool(rows_near(state, activity, options)) + + +def has_forward_activity(state: TraceRow, activity: list[TraceRow], options: OracleOptions) -> bool: + state_ts = state.ts + if state_ts is None: + return any( + row.table != state.table or state.index <= row.index <= state.index + RECENT_ACTIVITY_EVENTS + for row in activity + ) + return any( + row.ts is not None + and state_ts <= row.ts <= state_ts + options.persistent_lag_micros + for row in activity + ) + + +def body_sync_has_pinned_queue(state: TraceRow) -> bool: + return any( + (int_field(state.row, field) or 0) > 0 + for field in ("needed_count", "queue_len", "assigned_len", "outstanding") + ) + + +def check_header_recovery(node: NodeTrace, options: OracleOptions) -> list[Failure]: + failures: list[Failure] = [] + recoveries = [ + row + for row in node.table("header_sync") + if row.event in {HEADER_FRONTIER_REANCHORED, HEADER_FRONTIER_ADVANCED, HEADER_RANGE_COMMITTED, HEADER_GET_HEADERS_SENT} + ] + + for rejected in node.events("header_sync", HEADER_RANGE_REJECTED): + stage = str(rejected.row.get("validation_stage", "")) + error = str(rejected.row.get("error_kind", "")) + if stage != "link" and error != "first_header_does_not_link": + continue + + if header_reject_has_recovery(rejected, recoveries, options): + continue + if not trace_extends_past(node, rejected, options.persistent_lag_micros): + continue + + failures.append( + failure( + node, + "header_link_reject_recovers", + rejected, + { + "validation_stage": stage, + "error_kind": error, + "range_start": int_field(rejected.row, "range_start"), + "anchor_hash": rejected.row.get("anchor_hash"), + }, + ) + ) + break + + return failures + + +def header_reject_has_recovery(rejected: TraceRow, recoveries: list[TraceRow], options: OracleOptions) -> bool: + start = int_field(rejected.row, "range_start") + rejected_ts = rejected.ts + + for row in recoveries: + if rejected_ts is not None and row.ts is not None: + if not (rejected_ts <= row.ts <= rejected_ts + options.persistent_lag_micros): + continue + elif row.index <= rejected.index or row.index > rejected.index + RECENT_ACTIVITY_EVENTS: + continue + + if row.event == HEADER_FRONTIER_REANCHORED: + return True + if row.event == HEADER_RANGE_COMMITTED: + return True + if row.event == HEADER_FRONTIER_ADVANCED: + height = int_field(row.row, "height") + if start is None or height is None or height >= start: + return True + if row.event == HEADER_GET_HEADERS_SENT: + next_start = int_field(row.row, "range_start") + if start is None or next_start is None or next_start >= start: + return True + + return False + + +def trace_extends_past(node: NodeTrace, row: TraceRow, micros: int) -> bool: + if row.ts is None: + return True + latest_ts = max((candidate.ts for candidate in node.rows if candidate.ts is not None), default=row.ts) + return latest_ts - row.ts >= micros + + +def check_checkpoint_to_full_handoff(nodes: list[NodeTrace], options: OracleOptions) -> list[Failure]: + observations = [checkpoint_to_full_handoff(node, options) for node in nodes] + if any(observation[0] is None for observation in observations): + return [] + + concrete_failures = [ + failure + for failure, _ in observations + if failure is not None and failure.invariant != "checkpoint_to_full_handoff_observed" + ] + if concrete_failures: + return concrete_failures[:1] + + diagnostics_by_node = {node.node: diagnostics(node) for node in nodes} + best_detail = next((detail for _, detail in observations if detail), {}) + return [ + Failure( + "", + "checkpoint_to_full_handoff_observed", + None, + { + **best_detail, + "diagnostics": diagnostics_by_node, + }, + ) + ] + + +def checkpoint_to_full_handoff( + node: NodeTrace, options: OracleOptions +) -> tuple[Failure | None, dict[str, Any]]: + checkpoint_finish: TraceRow | None = None + checkpoint_height: int | None = None + + for row in node.events("commit_state", COMMIT_FINISH): + apply_class = row.row.get("apply_class") + height = int_field(row.row, "height") + if apply_class == APPLY_CLASS_CHECKPOINT and height is not None: + checkpoint_finish = row + checkpoint_height = height + elif ( + apply_class == APPLY_CLASS_FULL + and checkpoint_finish is not None + and height is not None + and checkpoint_height is not None + and height > checkpoint_height + and (row.index > checkpoint_finish.index or later_ts(row, checkpoint_finish)) + ): + if checkpoint_finish.ts is not None and row.ts is not None: + elapsed = row.ts - checkpoint_finish.ts + if elapsed > options.handoff_stall_micros: + return ( + failure( + node, + "checkpoint_to_full_handoff_within_window", + row, + { + "checkpoint": compact_row(checkpoint_finish), + "full": compact_row(row), + "elapsed_us": elapsed, + }, + ), + {}, + ) + return (None, {}) + + return ( + failure( + node, + "checkpoint_to_full_handoff_observed", + checkpoint_finish, + {"latest_checkpoint_height": checkpoint_height}, + ), + {"latest_checkpoint_height": checkpoint_height}, + ) + + +def failure(node: NodeTrace, invariant: str, row: TraceRow | None, detail: dict[str, Any]) -> Failure: + return Failure( + node=node.node, + invariant=invariant, + row=row, + detail={ + **detail, + "diagnostics": diagnostics(node), + }, + ) + + +def diagnostics(node: NodeTrace) -> dict[str, Any]: + starts = node.events("commit_state", COMMIT_START) + finishes_by_key = {row_key(row) for row in node.events("commit_state", COMMIT_FINISH)} + unmatched = [] + for start in starts: + key = row_key(start) + if key is not None and key not in finishes_by_key: + unmatched.append({"key": key, "row": compact_row(start)}) + if len(unmatched) >= 8: + break + + needed_rows = [ + row + for row in node.table("commit_state") + if row.row.get("action") in {QUERY_NEEDED_BLOCKS, "needed_blocks"} + ] + + return { + "last_frontier_transition": compact_row(node.latest("commit_state", SYNC_FRONTIER_TRANSITION)), + "unmatched_commit_starts": unmatched, + "latest_needed_blocks": compact_row(needed_rows[-1] if needed_rows else None), + "latest_block_sync_state": compact_row(node.latest("block_sync", BLOCK_SYNC_STATE)), + "recent_peer_violations": [ + compact_row(row) for row in node.events("header_sync", HEADER_PEER_VIOLATION)[-5:] + ], + "latest_header_reanchor": compact_row(node.latest("header_sync", HEADER_FRONTIER_REANCHORED)), + } + + +def run_oracle(root: Path, options: OracleOptions = OracleOptions()) -> list[Failure]: + nodes = load_traces(root) + if not nodes: + return [Failure("", "trace_files_exist", None, {"root": str(root)})] + + failures: list[Failure] = [] + for node in nodes: + for row in node.rows: + if row.event == "json_decode_error": + failures.append(failure(node, "trace_jsonl_is_valid", row, {})) + failures.extend(check_commit_pairs(node, options)) + failures.extend(check_frontiers(node)) + failures.extend(check_block_sync_activity(node, options)) + failures.extend(check_header_recovery(node, options)) + + if options.require_handoff_boundary: + failures.extend(check_checkpoint_to_full_handoff(nodes, options)) + + return failures + + +def print_failures(failures: list[Failure]) -> None: + for item in failures: + payload = { + "node": item.node, + "invariant": item.invariant, + "first_offending_row": compact_row(item.row), + **item.detail, + } + print("TRACE ORACLE FAILURE", file=sys.stderr) + print(json.dumps(payload, indent=2, sort_keys=True), file=sys.stderr) + + +def write_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, sort_keys=True)) + handle.write("\n") + + +def run_self_test() -> None: + with tempfile.TemporaryDirectory(prefix="zakura-trace-oracle.") as tmp: + root = Path(tmp) + + good = root / "good" / "node1" + write_jsonl( + good / "commit_state.jsonl", + [ + {"ts": 1, "event": COMMIT_START, "apply_token": 1, "height": 1, "hash": "aa"}, + {"ts": 2, "event": COMMIT_FINISH, "apply_token": 1, "height": 1, "hash": "aa"}, + { + "ts": 3, + "event": SYNC_FRONTIER_TRANSITION, + "cause": "verified_grow", + "old_finalized_height": 0, + "new_finalized_height": 0, + "old_verified_body_height": 0, + "new_verified_body_height": 1, + "old_best_header_height": 1, + "new_best_header_height": 1, + }, + ], + ) + write_jsonl( + good / "block_sync.jsonl", + [ + { + "ts": 4, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 1, + "best_header_tip": 1, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + } + ], + ) + assert not run_oracle(root / "good"), "good trace should pass" + + handoff = root / "handoff" / "node2" + write_jsonl( + handoff / "commit_state.jsonl", + [ + { + "ts": 1, + "event": COMMIT_START, + "apply_token": 10, + "apply_class": APPLY_CLASS_CHECKPOINT, + "height": 80, + "hash": "cc", + }, + { + "ts": 2, + "event": COMMIT_FINISH, + "apply_token": 10, + "apply_class": APPLY_CLASS_CHECKPOINT, + "height": 80, + "hash": "cc", + "elapsed_ms": 1, + }, + { + "ts": 3, + "event": COMMIT_START, + "apply_token": 11, + "apply_class": APPLY_CLASS_FULL, + "height": 81, + "hash": "dd", + }, + { + "ts": 4, + "event": COMMIT_FINISH, + "apply_token": 11, + "apply_class": APPLY_CLASS_FULL, + "height": 81, + "hash": "dd", + "elapsed_ms": 1, + }, + ], + ) + assert not run_oracle( + root / "handoff", + OracleOptions(require_handoff_boundary=True), + ), "checkpoint-to-full handoff trace should pass" + + no_handoff = root / "no_handoff" / "node2" + write_jsonl( + no_handoff / "commit_state.jsonl", + [ + { + "ts": 1, + "event": COMMIT_START, + "apply_token": 12, + "apply_class": APPLY_CLASS_CHECKPOINT, + "height": 80, + "hash": "ee", + }, + { + "ts": 2, + "event": COMMIT_FINISH, + "apply_token": 12, + "apply_class": APPLY_CLASS_CHECKPOINT, + "height": 80, + "hash": "ee", + }, + ], + ) + assert any( + f.invariant == "checkpoint_to_full_handoff_observed" + for f in run_oracle(root / "no_handoff", OracleOptions(require_handoff_boundary=True)) + ) + + slow_handoff = root / "slow_handoff" / "node2" + write_jsonl( + slow_handoff / "commit_state.jsonl", + [ + { + "ts": 1, + "event": COMMIT_FINISH, + "apply_token": 20, + "apply_class": APPLY_CLASS_CHECKPOINT, + "height": 80, + "hash": "aa", + }, + { + "ts": 10_000_000, + "event": COMMIT_FINISH, + "apply_token": 21, + "apply_class": APPLY_CLASS_FULL, + "height": 81, + "hash": "bb", + }, + ], + ) + assert any( + f.invariant == "checkpoint_to_full_handoff_within_window" + for f in run_oracle( + root / "slow_handoff", + OracleOptions(require_handoff_boundary=True, handoff_stall_micros=1_000_000), + ) + ) + + bad_commit = root / "bad_commit" / "node1" + write_jsonl( + bad_commit / "commit_state.jsonl", + [{"ts": 1, "event": COMMIT_START, "apply_token": 9, "height": 9, "hash": "bb"}], + ) + assert any(f.invariant == "commit_start_has_finish" for f in run_oracle(root / "bad_commit")) + + terminal_stalled = root / "terminal_stalled" / "node1" + write_jsonl( + terminal_stalled / "commit_state.jsonl", + [ + { + "ts": 1, + "event": COMMIT_STALLED, + "apply_token": 15, + "height": 15, + "hash": "ff", + } + ], + ) + assert any(f.invariant == "commit_stalled_not_terminal" for f in run_oracle(root / "terminal_stalled")) + + bad_frontier = root / "bad_frontier" / "node1" + write_jsonl( + bad_frontier / "commit_state.jsonl", + [ + { + "ts": 1, + "event": SYNC_FRONTIER_TRANSITION, + "cause": "verified_grow", + "old_finalized_height": 10, + "new_finalized_height": 9, + "old_verified_body_height": 10, + "new_verified_body_height": 9, + } + ], + ) + failures = run_oracle(root / "bad_frontier") + assert any(f.invariant == "finalized_height_monotonic" for f in failures) + assert any(f.invariant == "verified_body_height_monotonic_except_reset" for f in failures) + + valid_reset = root / "valid_reset" / "node1" + write_jsonl( + valid_reset / "commit_state.jsonl", + [ + { + "ts": 1, + "event": SYNC_FRONTIER_TRANSITION, + "cause": "verified_reset", + "old_finalized_height": 10, + "new_finalized_height": 10, + "old_verified_body_height": 10, + "new_verified_body_height": 9, + } + ], + ) + assert not run_oracle(root / "valid_reset"), "valid reorg/reset should pass" + + persistent_lag = root / "persistent_lag" / "node1" + write_jsonl( + persistent_lag / "block_sync.jsonl", + [ + { + "ts": 1, + "event": BLOCK_GET_BLOCKS_SENT, + "range_start": 1, + "range_count": 1, + }, + { + "ts": 500_000_000, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 1, + "best_header_tip": 10, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + }, + ], + ) + assert any( + f.invariant == "lagging_body_sync_has_real_activity" + for f in run_oracle(root / "persistent_lag", OracleOptions(persistent_lag_micros=1_000_000)) + ) + + query_spin = root / "query_spin" / "node1" + write_jsonl( + query_spin / "commit_state.jsonl", + [ + {"ts": 1_000_000, "event": "state_read_start", "action": QUERY_NEEDED_BLOCKS}, + { + "ts": 2_000_000, + "event": "state_read_success", + "action": QUERY_NEEDED_BLOCKS, + "range_count": 0, + }, + {"ts": 3_000_000, "event": "state_read_start", "action": QUERY_NEEDED_BLOCKS}, + { + "ts": 4_000_000, + "event": "state_read_success", + "action": QUERY_NEEDED_BLOCKS, + "range_count": 0, + }, + ], + ) + write_jsonl( + query_spin / "block_sync.jsonl", + [ + { + "ts": 5_000_000, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 10, + "best_header_tip": 20, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + "needed_count": 0, + "queue_len": 0, + "assigned_len": 0, + } + ], + ) + assert any( + f.invariant == "lagging_body_sync_not_query_spin" + for f in run_oracle(root / "query_spin", OracleOptions(persistent_lag_micros=10_000_000)) + ) + + self_healed_wedge = root / "self_healed_wedge" / "node1" + write_jsonl( + self_healed_wedge / "block_sync.jsonl", + [ + { + "ts": 1, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 10, + "best_header_tip": 30, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + "needed_count": 0, + "queue_len": 0, + "assigned_len": 0, + }, + { + "ts": 200_000_000, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 10, + "best_header_tip": 30, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + "needed_count": 0, + "queue_len": 0, + "assigned_len": 0, + }, + {"ts": 400_000_000, "event": BLOCK_GET_BLOCKS_SENT, "range_start": 11, "range_count": 20}, + { + "ts": 401_000_000, + "event": BLOCK_FRONTIERS_CHANGED, + "verified_block_tip": 30, + "best_header_tip": 30, + }, + { + "ts": 402_000_000, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 30, + "best_header_tip": 30, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + }, + ], + ) + assert any( + f.invariant == "lagging_body_sync_has_real_activity" + for f in run_oracle(root / "self_healed_wedge", OracleOptions(persistent_lag_micros=30_000_000)) + ) + + lag_with_progress = root / "lag_with_progress" / "node1" + write_jsonl( + lag_with_progress / "block_sync.jsonl", + [ + {"ts": 10, "event": BLOCK_GET_BLOCKS_SENT, "range_start": 2, "range_count": 4}, + { + "ts": 20, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 1, + "best_header_tip": 5, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + "needed_count": 4, + "queue_len": 0, + "assigned_len": 0, + }, + {"ts": 30, "event": BLOCK_BODY_RECEIVED, "height": 2}, + { + "ts": 40, + "event": BLOCK_FRONTIERS_CHANGED, + "verified_block_tip": 2, + "best_header_tip": 5, + }, + { + "ts": 50, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 5, + "best_header_tip": 5, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 0, + }, + ], + ) + assert not run_oracle(root / "lag_with_progress", OracleOptions(persistent_lag_micros=1_000_000)) + + queued_lag = root / "queued_lag" / "node1" + write_jsonl( + queued_lag / "block_sync.jsonl", + [ + {"ts": 9_500_000, "event": BLOCK_GET_BLOCKS_SENT, "range_start": 11, "range_count": 1}, + { + "ts": 10_000_000, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 10, + "best_header_tip": 20, + "budget_reserved": 0, + "reorder": 0, + "applying": 0, + "outstanding": 1, + "needed_count": 10, + "queue_len": 1, + "assigned_len": 1, + }, + ], + ) + assert any( + f.invariant == "block_sync_queue_lag_makes_progress" + for f in run_oracle(root / "queued_lag", OracleOptions(persistent_lag_micros=1_000_000)) + ) + + latency_trend = root / "latency_trend" / "node1" + write_jsonl( + latency_trend / "commit_state.jsonl", + [ + { + "ts": 1, + "event": COMMIT_FINISH, + "apply_token": 1, + "apply_class": APPLY_CLASS_FULL, + "height": 1, + "hash": "aa", + "elapsed_ms": 440_000, + }, + { + "ts": 2, + "event": COMMIT_FINISH, + "apply_token": 2, + "apply_class": APPLY_CLASS_FULL, + "height": 2, + "hash": "bb", + "elapsed_ms": 2_020_000, + }, + { + "ts": 3, + "event": COMMIT_FINISH, + "apply_token": 3, + "apply_class": APPLY_CLASS_FULL, + "height": 3, + "hash": "cc", + "elapsed_ms": 2_100_000, + }, + ], + ) + assert any( + f.invariant == "commit_latency_trend_bounded" + for f in run_oracle( + root / "latency_trend", + OracleOptions(commit_elapsed_micros=3_000_000_000, commit_trend_min_ms=300_000), + ) + ) + + stranded_anchor = root / "stranded_anchor" / "node1" + write_jsonl( + stranded_anchor / "header_sync.jsonl", + [ + { + "ts": 1, + "event": HEADER_RANGE_REJECTED, + "range_start": 100, + "range_count": 1, + "anchor_hash": "aa", + "validation_stage": "link", + "error_kind": "first_header_does_not_link", + }, + {"ts": 2_000_000, "event": HEADER_PEER_VIOLATION, "reason": "invalid_range"}, + ], + ) + assert any( + f.invariant == "header_link_reject_recovers" + for f in run_oracle(root / "stranded_anchor", OracleOptions(persistent_lag_micros=1_000_000)) + ) + + recovered_anchor = root / "recovered_anchor" / "node1" + write_jsonl( + recovered_anchor / "header_sync.jsonl", + [ + { + "ts": 1, + "event": HEADER_RANGE_REJECTED, + "range_start": 100, + "range_count": 1, + "anchor_hash": "aa", + "validation_stage": "link", + "error_kind": "first_header_does_not_link", + }, + {"ts": 2, "event": HEADER_FRONTIER_REANCHORED, "height": 99}, + ], + ) + assert not run_oracle(root / "recovered_anchor", OracleOptions(persistent_lag_micros=1_000_000)) + + bad_leak = root / "bad_leak" / "node1" + write_jsonl( + bad_leak / "block_sync.jsonl", + [ + { + "ts": 1, + "event": BLOCK_SYNC_STATE, + "verified_block_tip": 5, + "best_header_tip": 5, + "budget_reserved": 42, + "reorder": 0, + "applying": 1, + "outstanding": 0, + } + ], + ) + assert any(f.invariant == "final_block_sync_state_has_no_leaks" for f in run_oracle(root / "bad_leak")) + + print("trace_oracle self-test: PASS") + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("trace_dir", nargs="?", type=Path, help="root trace directory containing node* subdirectories") + parser.add_argument("--self-test", action="store_true", help="run fixture-based oracle tests") + parser.add_argument( + "--commit-elapsed-ms", + type=int, + default=COMMIT_MICROS_WINDOW // 1_000, + help="maximum allowed elapsed time between commit_start and commit_finish", + ) + parser.add_argument( + "--persistent-lag-seconds", + type=int, + default=RECENT_ACTIVITY_MICROS // 1_000_000, + help="body/header lag must have real get-blocks or body progress within this window", + ) + parser.add_argument( + "--handoff-stall-seconds", + type=int, + default=RECENT_ACTIVITY_MICROS // 1_000_000, + help="maximum allowed gap between a checkpoint commit and the first higher full commit", + ) + parser.add_argument( + "--commit-trend-min-ms", + type=int, + default=COMMIT_TREND_MIN_MS, + help="minimum commit latency before trend degradation is actionable", + ) + parser.add_argument( + "--commit-trend-factor", + type=float, + default=COMMIT_TREND_FACTOR, + help="maximum allowed step-up or monotonic first-to-last commit latency factor", + ) + parser.add_argument( + "--require-handoff-boundary", + action="store_true", + help="require at least one checkpoint commit followed by a higher full-verifier commit", + ) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + if args.self_test: + run_self_test() + return 0 + + if args.trace_dir is None: + print("trace_oracle.py: trace_dir is required unless --self-test is used", file=sys.stderr) + return 2 + + failures = run_oracle( + args.trace_dir, + OracleOptions( + commit_elapsed_micros=args.commit_elapsed_ms * 1_000, + persistent_lag_micros=args.persistent_lag_seconds * 1_000_000, + require_handoff_boundary=args.require_handoff_boundary, + handoff_stall_micros=args.handoff_stall_seconds * 1_000_000, + commit_trend_min_ms=args.commit_trend_min_ms, + commit_trend_factor=args.commit_trend_factor, + ), + ) + if failures: + print_failures(failures) + return 1 + + print(f"trace_oracle: PASS ({args.trace_dir})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index c6a5017a2d9..6c528a631cf 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -663,6 +663,10 @@ criteria = "safe-to-deploy" version = "0.13.1" criteria = "safe-to-deploy" +[[exemptions.filetime]] +version = "0.2.29" +criteria = "safe-to-deploy" + [[exemptions.find-msvc-tools]] version = "0.1.9" criteria = "safe-to-deploy" @@ -798,7 +802,7 @@ version = "2.7.1" criteria = "safe-to-run" [[exemptions.halo2_gadgets]] -version = "0.4.0" +version = "0.5.0" criteria = "safe-to-deploy" [[exemptions.halo2_legacy_pdqsort]] @@ -1400,7 +1404,7 @@ version = "0.4.0" criteria = "safe-to-deploy" [[exemptions.orchard]] -version = "0.13.1" +version = "0.14.0" criteria = "safe-to-deploy" [[exemptions.ordered-map]] @@ -2107,6 +2111,10 @@ criteria = "safe-to-deploy" version = "0.2.0" criteria = "safe-to-deploy" +[[exemptions.tar]] +version = "0.4.45" +criteria = "safe-to-deploy" + [[exemptions.tempfile]] version = "3.27.0" criteria = "safe-to-deploy" @@ -2596,7 +2604,7 @@ version = "1.3.0" criteria = "safe-to-deploy" [[exemptions.zcash_address]] -version = "0.11.0" +version = "0.12.0" criteria = "safe-to-deploy" [[exemptions.zcash_encoding]] @@ -2608,23 +2616,23 @@ version = "0.4.0" criteria = "safe-to-deploy" [[exemptions.zcash_keys]] -version = "0.13.0" +version = "0.14.0" criteria = "safe-to-deploy" [[exemptions.zcash_primitives]] -version = "0.27.0" +version = "0.28.0" criteria = "safe-to-deploy" [[exemptions.zcash_proofs]] -version = "0.27.0" +version = "0.28.0" criteria = "safe-to-deploy" [[exemptions.zcash_protocol]] -version = "0.8.0" +version = "0.9.0" criteria = "safe-to-deploy" [[exemptions.zcash_script]] -version = "0.4.4" +version = "0.4.5" criteria = "safe-to-deploy" [[exemptions.zcash_spec]] @@ -2632,7 +2640,7 @@ version = "0.2.1" criteria = "safe-to-deploy" [[exemptions.zcash_transparent]] -version = "0.7.0" +version = "0.8.0" criteria = "safe-to-deploy" [[exemptions.zerocopy]] diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index b55fc606ea0..60c94230790 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -782,6 +782,12 @@ who = "Chris Fallin " criteria = "safe-to-deploy" version = "0.4.18" +[[audits.bytecode-alliance.audits.tar]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.45 -> 0.4.46" +notes = "Nothing awry, no changes to unsafe or such." + [[audits.bytecode-alliance.audits.textwrap]] who = "Chris Fallin " criteria = "safe-to-deploy" @@ -883,6 +889,24 @@ criteria = "safe-to-deploy" delta = "0.243.0 -> 0.244.0" notes = "The Bytecode Alliance is the author of this crate" +[[audits.bytecode-alliance.audits.xattr]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +version = "1.2.0" +notes = "This crate contains `unsafe` calls to libc `extattr_*` functions as one would expect from the crate's purpose." + +[[audits.bytecode-alliance.audits.xattr]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +delta = "1.2.0 -> 1.3.1" +notes = "Minor changes to MacOS-specific code." + +[[audits.bytecode-alliance.audits.xattr]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "1.3.1 -> 1.6.1" +notes = "Refactorings and minor updates, nothing out of place." + [[audits.bytecode-alliance.audits.zeroize]] who = "Pat Hickey " criteria = "safe-to-deploy" diff --git a/zebra-chain/src/local_genesis.rs b/zebra-chain/src/local_genesis.rs index b7fb32b9ada..3df45a36766 100644 --- a/zebra-chain/src/local_genesis.rs +++ b/zebra-chain/src/local_genesis.rs @@ -520,9 +520,12 @@ mod tests { NetworkUpgrade::Nu7.activation_height(network), Some(activation_height) ); + // NU7 inherits the post-Blossom consensus target spacing (75s); this is the + // protocol block spacing and is independent of the harness `target_spacing_secs` + // option used to space the generated seeded-block timestamps. assert_eq!( NetworkUpgrade::target_spacing_for_height(network, activation_height).num_seconds(), - 25 + i64::from(crate::parameters::POST_BLOSSOM_POW_TARGET_SPACING) ); } diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index 13c8de91462..1a1217d44c9 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -106,6 +106,14 @@ impl VerifyBlockError { } } + /// Returns the state location for duplicate commit requests. + pub fn duplicate_location(&self) -> Option<&zs::KnownBlock> { + match self { + VerifyBlockError::Commit(commit_err) => commit_err.duplicate_location(), + _ => None, + } + } + /// Returns a suggested misbehaviour score increment for a certain error. pub fn misbehavior_score(&self) -> u32 { use VerifyBlockError::*; diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs index 6c3387fa57e..23ccc26a385 100644 --- a/zebra-consensus/src/checkpoint.rs +++ b/zebra-consensus/src/checkpoint.rs @@ -230,8 +230,11 @@ where /// details. /// /// This function is designed for use in tests. + /// + /// Only reachable from test/`proptest-impl` builds: the `checkpoint` module is + /// private and [`CheckpointVerifier`] is re-exported only under those cfgs. #[allow(dead_code)] - pub(crate) fn from_list( + pub fn from_list( list: impl IntoIterator, network: &Network, initial_tip: Option<(block::Height, block::Hash)>, @@ -1040,6 +1043,17 @@ impl VerifyCheckpointError { } } + /// Returns the state location for duplicate commit requests. + pub fn duplicate_location(&self) -> Option<&zs::KnownBlock> { + match self { + VerifyCheckpointError::CommitCheckpointVerified(source) => source + .downcast_ref::() + .and_then(|error| error.duplicate_location()), + VerifyCheckpointError::VerifyBlock(block_error) => block_error.duplicate_location(), + _ => None, + } + } + /// Returns a suggested misbehaviour score increment for a certain error. pub fn misbehavior_score(&self) -> u32 { // TODO: Adjust these values based on zcashd (#9258). diff --git a/zebra-consensus/src/lib.rs b/zebra-consensus/src/lib.rs index 0c2f3e1f9b1..2b0d1d55ad0 100644 --- a/zebra-consensus/src/lib.rs +++ b/zebra-consensus/src/lib.rs @@ -46,6 +46,8 @@ pub mod transaction; #[cfg(any(test, feature = "proptest-impl"))] pub use block::check::difficulty_is_valid; +#[cfg(any(test, feature = "proptest-impl"))] +pub use checkpoint::CheckpointVerifier; pub use block::{subsidy::funding_stream_address, Request, VerifyBlockError, MAX_BLOCK_SIGOPS}; pub use checkpoint::{VerifyCheckpointError, MAX_CHECKPOINT_BYTE_COUNT, MAX_CHECKPOINT_HEIGHT_GAP}; diff --git a/zebra-consensus/src/router.rs b/zebra-consensus/src/router.rs index 904fa5bcdc5..91bfa46ee12 100644 --- a/zebra-consensus/src/router.rs +++ b/zebra-consensus/src/router.rs @@ -143,6 +143,14 @@ impl RouterError { } } + /// Returns the state location for duplicate commit requests. + pub fn duplicate_location(&self) -> Option<&zs::KnownBlock> { + match self { + RouterError::Checkpoint { source, .. } => source.duplicate_location(), + RouterError::Block { source, .. } => source.duplicate_location(), + } + } + /// Returns a suggested misbehaviour score increment for a certain error. pub fn misbehavior_score(&self) -> u32 { // TODO: Adjust these values based on zcashd (#9258). diff --git a/zebra-network/src/config.rs b/zebra-network/src/config.rs index 5f2c80d7543..44d4a39c95e 100644 --- a/zebra-network/src/config.rs +++ b/zebra-network/src/config.rs @@ -5,11 +5,15 @@ use std::{ fmt, io::{self, ErrorKind}, net::{IpAddr, SocketAddr}, + path::Path, + str::FromStr, sync::Arc, time::Duration, }; use indexmap::IndexSet; +use iroh::SecretKey; +use rand::rngs::OsRng; use serde::{de, Deserialize, Deserializer, Serializer}; use tokio::fs; @@ -74,6 +78,14 @@ impl serde::Serialize for ZakuraNodeSecretKey { } } +/// An error that can occur while resolving the Zakura node secret key. +#[derive(Clone, Debug, thiserror::Error)] +pub enum ZakuraSecretKeyError { + /// The configured `zakura_node_secret_key` is not a valid iroh secret key. + #[error("configured zakura_node_secret_key is not a valid iroh secret key")] + InvalidConfigured, +} + /// The number of times Zebra will retry each initial peer's DNS resolution, /// before checking if any other initial peers have returned addresses. /// @@ -606,6 +618,111 @@ impl Config { Err(error) => Err(error.error), } } + + /// Resolves the Zakura native iroh [`SecretKey`] for this node, persisting a + /// freshly generated key on first use so the node keeps a stable + /// [`NodeId`](iroh::NodeId) across restarts. + /// + /// Resolution order: + /// 1. If [`zakura_node_secret_key`](Self::zakura_node_secret_key) is configured, + /// it is parsed and used verbatim. An unparseable value is a hard error. + /// 2. Otherwise, if [`cache_dir`](Self::cache_dir) is enabled, the persisted key + /// file is loaded; when it is missing or unreadable a fresh key is generated + /// and written atomically with owner-only (`0o600`) permissions, so every + /// later startup reuses the same identity. + /// 3. If the cache dir is disabled, an ephemeral key is generated for this run. + /// + /// # Security + /// + /// The persisted key file is the node's long-term private identity. It is + /// written beside the peer cache and restricted to owner read/write on Unix. + pub fn zakura_secret_key(&self) -> Result { + if let Some(secret) = &self.zakura_node_secret_key { + return SecretKey::from_str(secret.expose_secret()) + .map_err(|_| ZakuraSecretKeyError::InvalidConfigured); + } + + match self + .cache_dir + .zakura_node_secret_key_file_path(&self.network) + { + Some(key_file) => Ok(load_or_generate_zakura_secret_key(&key_file)), + // The cache dir is disabled, so there is nowhere to persist a stable + // key: fall back to an ephemeral identity for this run. + None => Ok(SecretKey::generate(OsRng)), + } + } +} + +/// Loads a persisted Zakura secret key from `key_file`, or generates, persists, and +/// returns a fresh key when the file is absent or cannot be parsed. +/// +/// I/O failures are logged and downgraded to an ephemeral key for this run, so a +/// read-only or full cache directory never prevents the node from starting. +fn load_or_generate_zakura_secret_key(key_file: &Path) -> SecretKey { + match std::fs::read_to_string(key_file) { + Ok(contents) => match SecretKey::from_str(contents.trim()) { + Ok(secret_key) => return secret_key, + Err(_) => warn!( + ?key_file, + "ignoring unparseable Zakura node secret key file; regenerating a new identity" + ), + }, + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => warn!( + ?error, + ?key_file, + "could not read Zakura node secret key file; regenerating a new identity" + ), + } + + let secret_key = SecretKey::generate(OsRng); + persist_zakura_secret_key(key_file, &secret_key); + secret_key +} + +/// Atomically writes `secret_key` to `key_file` as lowercase hex and restricts the +/// file to owner-only access. Persistence failures are logged but not fatal. +fn persist_zakura_secret_key(key_file: &Path, secret_key: &SecretKey) { + let encoded = hex::encode(secret_key.to_bytes()); + + match atomic_write(key_file.to_path_buf(), encoded.as_bytes()) { + Ok(Ok(path)) => { + if let Err(error) = restrict_secret_key_file_permissions(&path) { + warn!( + ?error, + ?path, + "persisted Zakura node secret key but could not restrict its permissions" + ); + } else { + info!(?path, "persisted a new Zakura node secret key"); + } + } + Ok(Err(error)) => warn!( + ?error, + ?key_file, + "could not persist Zakura node secret key; using an ephemeral identity this run" + ), + Err(error) => warn!( + ?error, + ?key_file, + "could not persist Zakura node secret key; using an ephemeral identity this run" + ), + } +} + +/// Restricts the persisted secret key file to owner read/write (`0o600`) on Unix. +#[cfg(unix)] +fn restrict_secret_key_file_permissions(path: &Path) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) +} + +/// File permissions are not restricted on non-Unix platforms. +#[cfg(not(unix))] +fn restrict_secret_key_file_permissions(_path: &Path) -> io::Result<()> { + Ok(()) } impl Default for Config { diff --git a/zebra-network/src/config/tests/vectors.rs b/zebra-network/src/config/tests/vectors.rs index 144efefdae6..397def3b05a 100644 --- a/zebra-network/src/config/tests/vectors.rs +++ b/zebra-network/src/config/tests/vectors.rs @@ -13,8 +13,8 @@ use zebra_chain::{ use crate::{ constants::{INBOUND_PEER_LIMIT_MULTIPLIER, OUTBOUND_PEER_LIMIT_MULTIPLIER}, - zakura::{DEFAULT_HS_MAX_INFLIGHT, DEFAULT_HS_RANGE}, - Config, + zakura::{DEFAULT_HS_MAX_INFLIGHT, DEFAULT_HS_RANGE, DEFAULT_ZAKURA_LISTEN_ADDR}, + CacheDir, Config, }; #[test] @@ -214,6 +214,18 @@ fn p2p_v2_unknown_future_config_fields_are_rejected() { header_sync.to_string().contains("unknown field"), "unexpected error for unknown header-sync field: {header_sync}", ); + + let block_sync = toml::from_str::( + r#" + [zakura.block_sync] + future_field = true + "#, + ) + .expect_err("deny_unknown_fields rejects unknown block-sync fields"); + assert!( + block_sync.to_string().contains("unknown field"), + "unexpected error for unknown block-sync field: {block_sync}", + ); } #[test] @@ -237,6 +249,11 @@ fn p2p_v2_config_roundtrip_keeps_dconfig_zakura_fields() { max_headers_per_response = 333 max_inflight_requests = 9 status_refresh_interval = "45s" + + [zakura.block_sync] + replace_legacy_syncer = true + max_blocks_per_response = 5 + status_refresh_interval = "12s" "#, ) .unwrap(); @@ -252,7 +269,69 @@ fn p2p_v2_config_roundtrip_keeps_dconfig_zakura_fields() { assert!(serialized.contains("max_headers_per_response = 333")); assert!(serialized.contains("max_inflight_requests = 9")); assert!(serialized.contains("status_refresh_interval = \"45s\"")); + assert!(serialized.contains("[zakura.block_sync]")); + assert!(!serialized.contains("replace_legacy_syncer")); + assert!(serialized.contains("max_blocks_per_response = 5")); + assert!(serialized.contains("status_refresh_interval = \"12s\"")); assert_eq!(toml::from_str::(&serialized).unwrap(), config); + assert!( + !config.zakura.block_sync.replace_legacy_syncer, + "deprecated replace_legacy_syncer config is accepted but ignored" + ); +} + +#[test] +fn configured_regtest_checkpoints_preserve_regtest_identity() { + let _init_guard = zebra_test::init(); + + // Mirrors the per-node config the zakura-regtest-e2e harness writes for the from-scratch + // catch-up node: a Regtest node that overrides only the checkpoint list (derived at + // runtime from the miner's chain). Regtest identity — genesis hash and network magic — + // must be preserved so the node still peers with a plain-Regtest miner; only checkpoint + // verification is added. + let genesis = Network::new_regtest(Default::default()).genesis_hash(); + let checkpoint = zebra_chain::block::Hash([7; 32]); + + // The exact minimal `[network.params]` table the harness writes: it overrides only the + // checkpoint list and lets every other Regtest parameter default. `block::Hash` + // serializes as a 32-byte array in internal (display-reversed) order, so the harness must + // emit byte arrays, not hex — this asserts that exact form parses. + let bytes_csv = |hash: zebra_chain::block::Hash| { + hash.0 + .iter() + .map(|byte| byte.to_string()) + .collect::>() + .join(", ") + }; + // The harness rewrites node2's `network = "Regtest"` line in place with this inline table + // (a single-line `sed` replacement), so verify exactly that form. + let inline = format!( + "network = {{ params = {{ checkpoints = [[0, [{}]], [10, [{}]]] }} }}\n", + bytes_csv(genesis), + bytes_csv(checkpoint), + ); + + let config: Config = toml::from_str(&inline) + .expect("the harness's inline ConfiguredRegtest checkpoint TOML deserializes"); + + assert!( + config.network.is_regtest(), + "a checkpoint-only override must stay Regtest", + ); + assert_eq!( + config.network.genesis_hash(), + genesis, + "Regtest genesis hash is preserved, so the node still peers with a plain-Regtest miner", + ); + + let checkpoints = config.network.checkpoint_list(); + assert_eq!( + checkpoints.max_height(), + Height(10), + "the derived checkpoint list replaces the genesis-only Regtest default", + ); + assert_eq!(checkpoints.hash(Height(0)), Some(genesis)); + assert_eq!(checkpoints.hash(Height(10)), Some(checkpoint)); } #[test] @@ -289,6 +368,7 @@ fn default_config_uses_ipv6() { assert_eq!(config.listen_addr.to_string(), "[::]:8233"); assert!(config.listen_addr.is_ipv6()); + assert_eq!(config.zakura.listen_addr, Some(DEFAULT_ZAKURA_LISTEN_ADDR)); } #[test] @@ -347,3 +427,142 @@ fn temporary_orchard_disabling_soft_fork_height_serialization_roundtrip() { Some(soft_fork_height), ); } + +/// With `v2_p2p` enabled, default config (no `zakura_node_secret_key`), and a +/// writable cache dir, the generated Zakura iroh identity must be persisted on +/// first use and reused on every later startup, so the node's `NodeId` is stable +/// across restarts. +/// +/// This is the regression test for `claude-ephemeral-node-secret-on-restart`: +/// before the fix, `Config::zakura_secret_key` generated a fresh ephemeral key on +/// every call and never wrote the reserved cache-dir key file, so two startups +/// produced different `NodeId`s and no key file existed. +#[test] +fn zakura_secret_key_is_persisted_and_stable_across_restarts() { + let _init_guard = zebra_test::init(); + + let cache_dir = tempfile::tempdir().expect("failed to create temp cache dir"); + + let config = Config { + cache_dir: CacheDir::custom_path(cache_dir.path()), + zakura_node_secret_key: None, + v2_p2p: true, + ..Config::default() + }; + + let key_file = config + .cache_dir + .zakura_node_secret_key_file_path(&config.network) + .expect("an enabled cache dir must yield a key file path"); + + // The key file must not exist before first use. + assert!( + !key_file.exists(), + "key file should not exist before the first startup", + ); + + // First startup: generate and persist a fresh key. + let first = config + .zakura_secret_key() + .expect("default config should resolve a secret key"); + + // The reserved cache-dir key file must now exist (atomic create+persist). + assert!( + key_file.exists(), + "first startup must persist the generated key to the cache-dir key file", + ); + + // On Unix, the long-term private identity file must be owner-only (0o600). + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&key_file) + .expect("key file metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600, "persisted secret key file must be owner-only"); + } + + // Second startup with a fresh `Config` reading the same cache dir (simulating + // a process restart) must reuse the persisted key, yielding the same `NodeId`. + let restart_config = Config { + cache_dir: CacheDir::custom_path(cache_dir.path()), + zakura_node_secret_key: None, + v2_p2p: true, + ..Config::default() + }; + let after_restart = restart_config + .zakura_secret_key() + .expect("restart should resolve the persisted secret key"); + + assert_eq!( + first.public(), + after_restart.public(), + "node identity must be stable across restarts when persisted to the cache dir", + ); + + // Calling again on the same config must also be stable. + let again = config + .zakura_secret_key() + .expect("repeat resolution should succeed"); + assert_eq!( + first.public(), + again.public(), + "repeat resolution must reuse the persisted key", + ); +} + +/// A configured `zakura_node_secret_key` must always win and is never overwritten +/// by the cache-dir persistence path; a disabled cache dir falls back to an +/// ephemeral key without writing any file. +#[test] +fn zakura_secret_key_honors_configured_key_and_disabled_cache() { + let _init_guard = zebra_test::init(); + + let cache_dir = tempfile::tempdir().expect("failed to create temp cache dir"); + + // Persist a key first so a key file exists in the cache dir. + let persisting = Config { + cache_dir: CacheDir::custom_path(cache_dir.path()), + zakura_node_secret_key: None, + v2_p2p: true, + ..Config::default() + }; + let persisted = persisting.zakura_secret_key().expect("persist a key"); + + // A configured key (64-char lowercase hex of the all-ones secret) must override + // the persisted cache-dir key. + let configured = "01".repeat(32); + let mut with_key: Config = toml::from_str(&format!("zakura_node_secret_key = '{configured}'")) + .expect("valid configured key parses"); + with_key.cache_dir = CacheDir::custom_path(cache_dir.path()); + let from_config = with_key + .zakura_secret_key() + .expect("configured key should resolve"); + assert_ne!( + from_config.public(), + persisted.public(), + "configured key must override the persisted cache-dir key", + ); + + // A disabled cache dir cannot persist, so it yields an ephemeral key and writes + // no file. + let disabled = Config { + cache_dir: CacheDir::disabled(), + zakura_node_secret_key: None, + v2_p2p: true, + ..Config::default() + }; + assert!( + disabled + .cache_dir + .zakura_node_secret_key_file_path(&disabled.network) + .is_none(), + "disabled cache dir must not yield a key file path", + ); + // Resolving still succeeds (ephemeral), and successive calls may differ. + disabled + .zakura_secret_key() + .expect("disabled cache dir should still resolve an ephemeral key"); +} diff --git a/zebra-network/src/peer/error.rs b/zebra-network/src/peer/error.rs index fb737e63124..5cddd745792 100644 --- a/zebra-network/src/peer/error.rs +++ b/zebra-network/src/peer/error.rs @@ -7,7 +7,10 @@ use thiserror::Error; use tracing_error::TracedError; use zebra_chain::serialization::SerializationError; -use crate::{protocol::external::InventoryHash, zakura::ZakuraUpgradeError}; +use crate::{ + protocol::external::InventoryHash, + zakura::{ZakuraProtocolError, ZakuraUpgradeError}, +}; /// A wrapper around `Arc` that implements `Error`. #[derive(Error, Debug, Clone)] @@ -268,6 +271,17 @@ pub enum HandshakeError { /// The Zakura upgrade hook returned an error. #[error("Zakura P2P v2 upgrade failed: {0}")] ZakuraUpgrade(#[from] ZakuraUpgradeError), + /// A mutually P2P-v2-capable peer framed a Zakura upgrade prelude whose + /// payload failed to decode. + /// + /// The peer advertised `NODE_P2P_V2` and sent a `p2pv2up` message, but its + /// bytes were malformed (oversized, trailing, truncated, or a bad + /// discriminator). This is a protocol violation, so we disconnect the peer + /// on the first offense (SR-7 fail-closed) instead of silently falling back + /// to legacy, which a peer could otherwise use to force a downgrade. Unlike + /// the neutral upgrade outcomes, this is a real peer failure. + #[error("Malformed Zakura P2P v2 upgrade prelude: {0}")] + ZakuraUpgradePreludeMalformed(#[source] ZakuraProtocolError), } impl HandshakeError { diff --git a/zebra-network/src/peer/handshake.rs b/zebra-network/src/peer/handshake.rs index ac25c4f8eae..e8dd976d07d 100644 --- a/zebra-network/src/peer/handshake.rs +++ b/zebra-network/src/peer/handshake.rs @@ -966,6 +966,7 @@ where let outcome = if connected_addr.is_inbound() { run_responder_upgrade( peer_conn, + &connector, &config, nonces, local_node_id, @@ -1089,8 +1090,10 @@ where peer_conn.send(Message::P2pV2Upgrade(init_bytes)).await?; let Some(P2pV2Upgrade::Accept(accept)) = read_upgrade_prelude(peer_conn).await? else { - // A reject, an unexpected variant, a malformed payload, or a closed - // upgrade window: keep the legacy connection. + // A well-formed reject, an unexpected variant, or no prelude within the + // skip window: keep the legacy connection. A malformed prelude is not + // reached here; `read_upgrade_prelude` disconnects on the first + // malformed upgrade message instead of falling back. return Ok(neutral_upgrade_fallback()); }; @@ -1126,6 +1129,7 @@ where /// can dial us. Our iroh router accepts that inbound dial separately. async fn run_responder_upgrade( peer_conn: &mut Framed, + connector: &ZakuraHandshakeConnector, config: &ZakuraHandshakeConfig, nonces: ZakuraLegacyNonces, local_node_id: Vec, @@ -1134,6 +1138,10 @@ async fn run_responder_upgrade( where PeerTransport: AsyncRead + AsyncWrite + Unpin + Send + 'static, { + // A well-formed unexpected variant or no prelude within the skip window: send + // a neutral reject and keep legacy. A malformed prelude is not reached here; + // `read_upgrade_prelude` disconnects on the first malformed upgrade message + // instead of replying with a neutral reject and falling back. let Some(P2pV2Upgrade::Init(init)) = read_upgrade_prelude(peer_conn).await? else { send_upgrade_reject(peer_conn, config).await?; return Ok(neutral_upgrade_fallback()); @@ -1177,6 +1185,18 @@ where return Ok(neutral_upgrade_fallback()); }; + // The peer dials our advertised Zakura endpoint over QUIC after receiving + // `Accept`, and our iroh router registers that inbound connection + // separately. Wait for that native registration before reporting the + // upgrade: the outer handshake drops the legacy TCP connection on + // `Upgraded`, so without this wait an inbound peer that sends a valid `Init` + // and then never completes the native dial would make us discard a working + // legacy connection with no Zakura replacement. This mirrors the initiator's + // `spawn_zakura_dial_to_hints_and_wait` hand-off wait. + if !connector.wait_for_zakura_registration(&peer_id).await { + return Ok(neutral_upgrade_fallback()); + } + Ok(ZakuraUpgradeOutcome::Upgraded { peer_id }) } @@ -1204,8 +1224,18 @@ where /// /// Skips a small bounded number of unrelated messages (the overall handshake /// timeout also applies), so a peer cannot stall the upgrade by streaming other -/// messages. Returns `Ok(None)` if the prelude is malformed or no prelude -/// arrives within the skip bound, so the caller falls back to legacy. +/// messages. +/// +/// Returns `Ok(None)` only when no prelude arrives within the skip bound: a peer +/// that advertised `NODE_P2P_V2` but never frames a `p2pv2up` message is treated +/// as a neutral legacy fallback (compatibility). +/// +/// In contrast, a peer that *does* frame a `p2pv2up` message whose payload fails +/// to decode has violated the upgrade protocol, so this returns +/// [`HandshakeError::ZakuraUpgradePreludeMalformed`] rather than erasing the +/// decode error to `None`. Erasing it would let a peer force a downgrade to +/// legacy by sending malformed upgrade bytes (SR-7 fail-closed); surfacing it +/// disconnects the peer on the first malformed upgrade message. async fn read_upgrade_prelude( peer_conn: &mut Framed, ) -> Result, HandshakeError> @@ -1221,7 +1251,13 @@ where .await .ok_or(HandshakeError::ConnectionClosed)??; if let Message::P2pV2Upgrade(payload) = message { - return Ok(P2pV2Upgrade::decode(&payload).ok()); + return match P2pV2Upgrade::decode(&payload) { + Ok(prelude) => Ok(Some(prelude)), + Err(error) => { + metrics::counter!("zakura.p2p.upgrade.prelude.malformed").increment(1); + Err(HandshakeError::ZakuraUpgradePreludeMalformed(error)) + } + }; } } @@ -1347,7 +1383,8 @@ where HandshakeError::ObsoleteVersion(_) => "obsolete_version", HandshakeError::Timeout => "timeout", HandshakeError::ZakuraUpgradeSelected - | HandshakeError::ZakuraUpgrade(_) => { + | HandshakeError::ZakuraUpgrade(_) + | HandshakeError::ZakuraUpgradePreludeMalformed(_) => { unreachable!("negotiate_version returns before Zakura upgrade routing") } }; diff --git a/zebra-network/src/peer/handshake/tests.rs b/zebra-network/src/peer/handshake/tests.rs index 1cf97bb8dab..0dfb890be5c 100644 --- a/zebra-network/src/peer/handshake/tests.rs +++ b/zebra-network/src/peer/handshake/tests.rs @@ -199,8 +199,19 @@ impl ZakuraService for DropSink { } /// Starts a real Zakura endpoint over loopback QUIC for an upgrade test. +/// +/// The cache dir is disabled so each call resolves a fresh ephemeral iroh +/// identity via `Config::zakura_secret_key`. With the default (enabled) cache +/// dir, every endpoint would load the *same* persisted key for the default +/// network and so share one `NodeId`; iroh then refuses the upgrade dial as a +/// self-connect. Disabling the cache also keeps these tests from writing a +/// secret-key file into the real user cache directory. async fn start_test_zakura_endpoint() -> crate::zakura::ZakuraEndpoint { - crate::zakura::spawn_zakura_endpoint(&test_config(true), |_supervisor, _trace| { + let config = Config { + cache_dir: crate::config::CacheDir::disabled(), + ..test_config(true) + }; + crate::zakura::spawn_zakura_endpoint(&config, |_supervisor, _trace| { Arc::new(DropSink) as Arc }) .await @@ -281,6 +292,274 @@ async fn mutual_p2p_v2_legacy_upgrade_forms_zakura_connection() { remote_endpoint.shutdown().await; } +/// An inbound legacy peer that sends a valid upgrade `Init`, receives our +/// `Accept`, and then never completes the native QUIC dial must not make the +/// responder report `Upgraded` and drop the working legacy TCP connection. +/// +/// Regression test for `claude-legacy-upgrade-premature-upgraded-no-native`: +/// the responder previously returned `Upgraded` immediately after sending +/// `Accept`, so the outer handshake dropped legacy TCP with no registered +/// Zakura replacement. The responder must instead wait for the inbound native +/// registration (mirroring the initiator's hand-off wait) and otherwise fall +/// back to a neutral `Rejected`, keeping the legacy connection. +#[tokio::test] +async fn responder_upgrade_keeps_legacy_when_native_dial_never_registers() { + let _init_guard = zebra_test::init(); + + // A real responder endpoint with a live Zakura supervisor. + let responder_endpoint = start_test_zakura_endpoint().await; + let connector = responder_endpoint.connector(); + + let network = test_config(true).network; + let config = ZakuraHandshakeConfig::for_network(&network); + + // The legacy `version` nonces the responder observed for this peer. + let nonces = ZakuraLegacyNonces { + local_zebra_nonce: Nonce(0x1111_1111_1111_1111), + remote_zebra_nonce: Nonce(0x2222_2222_2222_2222), + }; + + // The responder advertises its own live Zakura hints in the `Accept`. + let (local_node_id, local_direct_addresses) = connector + .local_iroh_hints() + .await + .expect("a live Zakura endpoint exposes local upgrade hints"); + + let (responder_stream, peer_stream) = duplex(16 * 1024); + let mut responder_conn = Framed::new( + responder_stream, + Codec::builder().for_network(&network).finish(), + ); + let mut peer_conn = Framed::new(peer_stream, Codec::builder().for_network(&network).finish()); + + // A valid `Init` that passes the responder's static, nonce, and protocol + // validation, claiming a real 32-byte iroh identity the attacker never + // brings online over native QUIC. + let init = P2pV2UpgradeInit { + magic: PRELUDE_MAGIC, + prelude_version: config.prelude_version, + zakura_protocol_min: config.zakura_protocol_min, + zakura_protocol_max: config.zakura_protocol_max, + network_id: config.network_id, + chain_id: config.chain_id, + capabilities: config.supported_capabilities, + // The peer's nonce labels are the mirror image of the responder's. + local_zebra_nonce: nonces.remote_zebra_nonce, + remote_zebra_nonce: nonces.local_zebra_nonce, + upgrade_nonce: [9u8; 32], + iroh_node_id: vec![7u8; 32], + iroh_direct_addresses: vec![b"192.0.2.1:1".to_vec()], + iroh_relay_hint: None, + max_control_frame_bytes: config.max_control_frame_bytes, + max_open_streams: config.max_open_streams, + }; + + // The malicious initiator: send the `Init`, read the `Accept`, then go + // silent (never dial the responder's native Zakura endpoint), holding the + // legacy stream open for the rest of the test. + let attacker = async move { + let init_bytes = P2pV2Upgrade::Init(init) + .encode() + .expect("a valid init encodes"); + peer_conn + .send(Message::P2pV2Upgrade(init_bytes)) + .await + .expect("the initiator sends its upgrade init"); + + let reply = peer_conn + .next() + .await + .expect("the responder replies before closing the legacy stream") + .expect("the reply frame decodes"); + let Message::P2pV2Upgrade(payload) = reply else { + panic!("the responder must reply with a p2pv2 upgrade message"); + }; + assert!( + matches!(P2pV2Upgrade::decode(&payload), Ok(P2pV2Upgrade::Accept(_))), + "the responder must accept a valid init before waiting for native registration", + ); + + peer_conn + }; + + // The responder must not finalize the upgrade until a native registration + // appears; with no dial it falls back after the appear timeout. The bound + // makes a regression (immediate `Upgraded`) or a hang fail loudly. + let responder = run_responder_upgrade( + &mut responder_conn, + &connector, + &config, + nonces, + local_node_id, + local_direct_addresses, + ); + + let (outcome, _held_peer_conn) = tokio::join!( + tokio::time::timeout(std::time::Duration::from_secs(45), responder), + attacker, + ); + + let outcome = outcome + .expect("responder upgrade resolves within the time bound") + .expect("responder upgrade returns an outcome, not a handshake error"); + + assert!( + matches!(outcome, ZakuraUpgradeOutcome::Rejected { .. }), + "the responder reported {outcome:?}; it must keep the legacy connection \ + (a neutral Rejected fallback) when the inbound peer sends a valid Init but \ + never registers a native Zakura connection", + ); + + responder_endpoint.shutdown().await; +} + +/// An inbound legacy peer that advertised `NODE_P2P_V2` and frames a `p2pv2up` +/// message whose payload fails to decode must be disconnected on the first +/// malformed upgrade message, not silently kept on the legacy connection. +/// +/// Regression test for `claude-legacy-upgrade-malformed-fallback-fail-open` +/// (responder facet): `read_upgrade_prelude` previously erased +/// `P2pV2Upgrade::decode` errors to `None` via `.ok()`, so the responder mapped +/// malformed bytes to a neutral reject plus legacy fallback. That let a peer +/// force a downgrade to legacy by sending garbage upgrade bytes (SR-7 +/// fail-open). The malformed prelude must instead surface as a non-neutral +/// `ZakuraUpgradePreludeMalformed` disconnect. +#[tokio::test] +async fn responder_upgrade_disconnects_on_malformed_prelude() { + let _init_guard = zebra_test::init(); + + let network = test_config(true).network; + let config = ZakuraHandshakeConfig::for_network(&network); + let nonces = ZakuraLegacyNonces { + local_zebra_nonce: Nonce(0x1111_1111_1111_1111), + remote_zebra_nonce: Nonce(0x2222_2222_2222_2222), + }; + // The malformed-prelude branch returns before any endpoint use, so a + // connector without a live endpoint is enough to exercise the responder + // path. + let connector = crate::zakura::ZakuraHandshakeConnector::unavailable(); + + let (responder_stream, peer_stream) = duplex(16 * 1024); + let mut responder_conn = Framed::new( + responder_stream, + Codec::builder().for_network(&network).finish(), + ); + let mut peer_conn = Framed::new(peer_stream, Codec::builder().for_network(&network).finish()); + + // A framed `p2pv2up` message whose payload has an unknown discriminator, so + // `P2pV2Upgrade::decode` fails. Oversized/trailing/truncated payloads share + // this same decode-error path. + let attacker = async move { + peer_conn + .send(Message::P2pV2Upgrade(vec![0xFF; 16])) + .await + .expect("the malformed initiator frames its bogus upgrade prelude"); + peer_conn + }; + + let responder = run_responder_upgrade( + &mut responder_conn, + &connector, + &config, + nonces, + vec![1u8; 32], + vec![b"127.0.0.1:1".to_vec()], + ); + + let (outcome, _held_peer_conn) = tokio::join!( + tokio::time::timeout(std::time::Duration::from_secs(10), responder), + attacker, + ); + + let error = outcome + .expect("the responder upgrade resolves within the time bound") + .expect_err( + "a malformed upgrade prelude must disconnect the peer, not fall back to legacy", + ); + assert!( + matches!(error, HandshakeError::ZakuraUpgradePreludeMalformed(_)), + "the responder returned {error:?}; a malformed p2pv2up prelude must be a \ + first-offense disconnect", + ); + assert!( + !error.is_neutral_disconnect(), + "a malformed upgrade prelude must be a penalized peer failure, not a neutral \ + legacy fallback", + ); +} + +/// The TCP initiator side of the same regression: a peer that advertised +/// `NODE_P2P_V2`, receives our `Init`, and replies with a `p2pv2up` message +/// whose payload fails to decode must be disconnected, not kept on legacy. +/// +/// Regression test for `claude-legacy-upgrade-malformed-fallback-fail-open` +/// (initiator facet): the initiator previously mapped a malformed `Accept` to a +/// neutral legacy fallback. It must instead surface a non-neutral +/// `ZakuraUpgradePreludeMalformed` disconnect. +#[tokio::test] +async fn initiator_upgrade_disconnects_on_malformed_prelude() { + let _init_guard = zebra_test::init(); + + let network = test_config(true).network; + let config = ZakuraHandshakeConfig::for_network(&network); + let nonces = ZakuraLegacyNonces { + local_zebra_nonce: Nonce(0x3333_3333_3333_3333), + remote_zebra_nonce: Nonce(0x4444_4444_4444_4444), + }; + let connector = crate::zakura::ZakuraHandshakeConnector::unavailable(); + + let (initiator_stream, peer_stream) = duplex(16 * 1024); + let mut initiator_conn = Framed::new( + initiator_stream, + Codec::builder().for_network(&network).finish(), + ); + let mut peer_conn = Framed::new(peer_stream, Codec::builder().for_network(&network).finish()); + + // The malicious responder reads our `Init`, then replies with a framed + // `p2pv2up` message whose payload fails to decode (an unknown discriminator) + // instead of a well-formed `Accept`. + let attacker = async move { + let init = peer_conn + .next() + .await + .expect("the initiator frames its upgrade init") + .expect("the init frame decodes at the codec layer"); + assert!( + matches!(init, Message::P2pV2Upgrade(_)), + "the initiator must send an upgrade init first", + ); + peer_conn + .send(Message::P2pV2Upgrade(vec![0xFF; 16])) + .await + .expect("the malformed responder frames its bogus accept"); + peer_conn + }; + + let initiator = run_initiator_upgrade( + &mut initiator_conn, + &connector, + &config, + nonces, + vec![1u8; 32], + vec![b"127.0.0.1:1".to_vec()], + ); + + let (outcome, _held_peer_conn) = tokio::join!( + tokio::time::timeout(std::time::Duration::from_secs(10), initiator), + attacker, + ); + + let error = outcome + .expect("the initiator upgrade resolves within the time bound") + .expect_err("a malformed upgrade accept must disconnect the peer, not fall back to legacy"); + assert!( + matches!(error, HandshakeError::ZakuraUpgradePreludeMalformed(_)), + "the initiator returned {error:?}; a malformed p2pv2up accept must be a \ + first-offense disconnect", + ); + assert!(!error.is_neutral_disconnect()); +} + #[tokio::test] async fn p2p_v2_service_bit_advertisement_follows_config() { let _init_guard = zebra_test::init(); @@ -333,6 +612,12 @@ fn zakura_upgrade_errors_are_neutral_disconnects() { .is_neutral_disconnect() ); assert!(!HandshakeError::Timeout.is_neutral_disconnect()); + // A malformed upgrade prelude is a real peer failure: it must be demoted + // (reported failed), not treated as a neutral legacy fallback. + assert!(!HandshakeError::ZakuraUpgradePreludeMalformed( + crate::zakura::ZakuraProtocolError::InvalidDiscriminator(0xFF) + ) + .is_neutral_disconnect()); } #[test] diff --git a/zebra-network/src/zakura.rs b/zebra-network/src/zakura.rs index 0307f62e51b..ac4f9b1adc1 100644 --- a/zebra-network/src/zakura.rs +++ b/zebra-network/src/zakura.rs @@ -15,7 +15,9 @@ use crate::{ PeerSocketAddr, }; +mod block_sync; mod discovery; +mod exchange; mod handler; mod handshake; mod header_sync; @@ -25,14 +27,17 @@ pub mod testkit; mod trace; pub mod transport; +pub use block_sync::*; pub use discovery::*; +pub use exchange::*; pub use handler::*; pub use handshake::*; pub use header_sync::*; pub use legacy_gossip::*; pub use trace::{ - peer_label as zakura_trace_peer_label, reject_reason_label as zakura_trace_reject_reason_label, - ZakuraTrace, ZakuraTraceEvent, CONN_TABLE, HANDSHAKE_TABLE, HEADER_SYNC_TABLE, + commit_state_trace, peer_label as zakura_trace_peer_label, + reject_reason_label as zakura_trace_reject_reason_label, ZakuraTrace, ZakuraTraceEvent, + BLOCK_SYNC_TABLE, COMMIT_STATE_TABLE, CONN_TABLE, HANDSHAKE_TABLE, HEADER_SYNC_TABLE, LEGACY_REQUEST_TABLE, RATELIMIT_TABLE, STREAM_TABLE, }; pub use transport::*; @@ -289,6 +294,41 @@ impl ZakuraHandshakeConnector { if !endpoint.ensure_upgrade_native_dial(node_addr) { return false; } + if wait_for_zakura_peer(&mut registered, peer_id, ZAKURA_LIVENESS_APPEAR_TIMEOUT).await { + return true; + } + + // The hand-off did not complete within the wait window. The dial spawned + // by `ensure_upgrade_native_dial` uses `RedialPolicy::maintain`, so it + // would keep redialing this peer-supplied address forever and retain its + // `upgrade_dials` entry. Unless the peer registered in the meantime + // (keep its maintained dial as the recovery path), cancel the dial and + // drop the entry so a malicious legacy responder cannot leak unbounded + // maintained dials and outbound QUIC traffic by repeating failed + // upgrades with distinct node ids. + if !registered.borrow().iter().any(|id| id == peer_id) { + endpoint.cancel_upgrade_native_dial(peer_id); + } + false + } + + /// Wait until the upgraded peer's inbound native QUIC connection registers + /// with the local supervisor. + /// + /// Used by the inbound legacy responder, which does not dial: after sending + /// `Accept` the remote peer is expected to dial our advertised Zakura + /// endpoint, and our iroh router registers that connection separately. The + /// outer handshake drops the legacy TCP connection once the upgrade is + /// reported, so the responder must confirm a usable Zakura replacement + /// exists first. Returns `false` (keep legacy) if the peer never registers + /// within [`ZAKURA_LIVENESS_APPEAR_TIMEOUT`] or this node has no live + /// endpoint, so a peer that sends a valid `Init` and then never completes + /// the native dial cannot make us silently drop a working legacy peer. + pub(crate) async fn wait_for_zakura_registration(&self, peer_id: &ZakuraPeerId) -> bool { + let Some(endpoint) = self.endpoint.as_ref() else { + return false; + }; + let mut registered = endpoint.supervisor().subscribe(); wait_for_zakura_peer(&mut registered, peer_id, ZAKURA_LIVENESS_APPEAR_TIMEOUT).await } diff --git a/zebra-network/src/zakura/block_sync/config.rs b/zebra-network/src/zakura/block_sync/config.rs new file mode 100644 index 00000000000..b37f4dd5296 --- /dev/null +++ b/zebra-network/src/zakura/block_sync/config.rs @@ -0,0 +1,227 @@ +use super::{error::*, wire::*, *}; + +/// Default number of blocks advertised per response. +/// +/// Set to the wire ceiling so block-body requests batch as many blocks as the +/// per-response byte cap ([`MAX_BS_RESPONSE_BYTES`]) allows. Small early-chain +/// blocks ride in large counts; large near-tip blocks are byte-capped to fewer. +pub const DEFAULT_BS_BLOCKS_PER_RESPONSE: u32 = 128; +/// Default number of in-flight block requests advertised per peer. +/// +/// Combined with the global byte budget ([`DEFAULT_BS_MAX_INFLIGHT_BLOCK_BYTES`]) +/// this lets the budget — not a small fixed slot count — pace downloads. +pub const DEFAULT_BS_MAX_INFLIGHT: u16 = 16; +/// Default expected serving-peer fanout used to derive the per-peer in-flight +/// byte cap (`max_inflight_block_bytes / expected_peers`). +/// +/// Bounds how much of the global byte budget a single peer can reserve so one +/// fast peer cannot starve the others once the slot cap stops binding. `0` +/// disables per-peer byte fairness. +pub const DEFAULT_BS_EXPECTED_PEERS: usize = 8; +/// Default total response byte target advertised per range response. +pub const DEFAULT_BS_MAX_RESPONSE_BYTES: u32 = 32 * 1024 * 1024; +/// Default global byte budget reserved for later block-download scheduling. +pub const DEFAULT_BS_MAX_INFLIGHT_BLOCK_BYTES: u64 = 4 * 1024 * 1024 * 1024; +/// Default block-sync request timeout reserved for later scheduling. +pub const DEFAULT_BS_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// Default block-sync status refresh interval reserved for later advertisement. +pub const DEFAULT_BS_STATUS_REFRESH_INTERVAL: Duration = Duration::from_secs(30); +/// Default tolerated size-hint deviation percentage reserved for later soft scoring. +pub const DEFAULT_BS_SIZE_DEVIATION_TOLERANCE: u32 = 200; +/// Default block-sync peer fanout reserved for later range scheduling. +pub const DEFAULT_BS_FANOUT: usize = 3; +/// Default body lag where block sync pauses new downloads and lets block propagation finish. +pub const DEFAULT_BS_NEAR_TIP_BODY_DOWNLOAD_PAUSE_BLOCKS: u32 = 2; +/// Maximum peer-advertised response byte target accepted from stream-6 status. +pub const MAX_BS_RESPONSE_BYTES: u32 = { + // This cast is safe: MAX_BS_MESSAGE_BYTES is asserted below 4 MiB. + MAX_BS_MESSAGE_BYTES as u32 +}; + +/// Block-sync peer status advertisement. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct BlockSyncStatus { + /// Earliest block body this peer can serve. + pub servable_low: block::Height, + /// Highest contiguous verified block body this peer can serve. + pub servable_high: block::Height, + /// Hash of `servable_high`. + pub tip_hash: block::Hash, + /// Maximum blocks the sender will serve per requested range. + pub max_blocks_per_response: u32, + /// Maximum concurrent `GetBlocks` requests the sender will service. + pub max_inflight_requests: u16, + /// Maximum total response bytes the sender targets per requested range. + pub max_response_bytes: u32, +} + +impl BlockSyncStatus { + pub(super) fn encode_to(&self, writer: &mut W) -> Result<(), BlockSyncWireError> { + write_height(writer, self.servable_low)?; + write_height(writer, self.servable_high)?; + self.tip_hash.zcash_serialize(&mut *writer)?; + writer.write_u32::(clamp_advertised_blocks(self.max_blocks_per_response))?; + writer.write_u16::(self.max_inflight_requests)?; + writer.write_u32::(self.max_response_bytes.max(1))?; + Ok(()) + } + + pub(super) fn decode_from(reader: &mut R) -> Result { + Ok(Self { + servable_low: read_height(reader)?, + servable_high: read_height(reader)?, + tip_hash: block::Hash::zcash_deserialize(&mut *reader)?, + max_blocks_per_response: clamp_advertised_blocks(reader.read_u32::()?), + max_inflight_requests: clamp_advertised_inflight(reader.read_u16::()?), + max_response_bytes: clamp_advertised_response_bytes(reader.read_u32::()?), + }) + } +} + +impl Default for BlockSyncStatus { + fn default() -> Self { + Self { + servable_low: block::Height::MIN, + servable_high: block::Height::MIN, + tip_hash: block::Hash([0; 32]), + max_blocks_per_response: DEFAULT_BS_BLOCKS_PER_RESPONSE, + max_inflight_requests: DEFAULT_BS_MAX_INFLIGHT, + max_response_bytes: DEFAULT_BS_MAX_RESPONSE_BYTES, + } + } +} + +/// Block-sync configuration nested under the Zakura P2P-v2 config. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct ZakuraBlockSyncConfig { + /// Deprecated compatibility key for older rollout configs. + /// + /// Zakura block sync is now selected by the top-level `v2_p2p` flag. This + /// field is accepted but ignored so older configs keep parsing. + #[doc(hidden)] + #[serde( + default, + skip_serializing, + deserialize_with = "deserialize_ignored_replace_legacy_syncer" + )] + pub replace_legacy_syncer: bool, + /// Maximum blocks this node advertises per `GetBlocks` response. + pub max_blocks_per_response: u32, + /// Maximum concurrent `GetBlocks` requests this node advertises per peer. + pub max_inflight_requests: u16, + /// Maximum total response bytes this node advertises per `GetBlocks` response. + pub max_response_bytes: u32, + /// Maximum estimated bytes reserved for in-flight and buffered block bodies. + pub max_inflight_block_bytes: u64, + /// Expected serving-peer fanout used to derive the per-peer in-flight byte cap + /// (`max_inflight_block_bytes / expected_peers`). + /// + /// Prevents one fast peer from reserving the whole byte budget once the + /// per-peer slot cap stops binding. `0` disables per-peer byte fairness. + pub expected_peers: usize, + /// Timeout for an outstanding block-body range request. + #[serde(with = "humantime_serde")] + pub request_timeout: Duration, + /// How often this node sends unsolicited status refreshes after local frontier changes. + #[serde(with = "humantime_serde")] + pub status_refresh_interval: Duration, + /// Percentage deviation from advertised body-size hints tolerated before soft scoring. + pub size_deviation_tolerance: u32, + /// Number of peers later range scheduling may fan out to for the same body gap. + pub fanout: usize, + /// Body lag at or below which block sync pauses new downloads near the header tip. + pub near_tip_body_download_pause_blocks: u32, + /// Block-sync peer caps and queue limits owned by this service. + pub peer_limits: ServicePeerLimits, +} + +fn deserialize_ignored_replace_legacy_syncer<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let _ = bool::deserialize(deserializer)?; + Ok(false) +} + +impl Default for ZakuraBlockSyncConfig { + fn default() -> Self { + Self { + replace_legacy_syncer: false, + max_blocks_per_response: DEFAULT_BS_BLOCKS_PER_RESPONSE, + max_inflight_requests: DEFAULT_BS_MAX_INFLIGHT, + max_response_bytes: DEFAULT_BS_MAX_RESPONSE_BYTES, + max_inflight_block_bytes: DEFAULT_BS_MAX_INFLIGHT_BLOCK_BYTES, + expected_peers: DEFAULT_BS_EXPECTED_PEERS, + request_timeout: DEFAULT_BS_REQUEST_TIMEOUT, + status_refresh_interval: DEFAULT_BS_STATUS_REFRESH_INTERVAL, + size_deviation_tolerance: DEFAULT_BS_SIZE_DEVIATION_TOLERANCE, + fanout: DEFAULT_BS_FANOUT, + near_tip_body_download_pause_blocks: DEFAULT_BS_NEAR_TIP_BODY_DOWNLOAD_PAUSE_BLOCKS, + peer_limits: ServicePeerLimits::default(), + } + } +} + +impl ZakuraBlockSyncConfig { + /// Return the clamped block-count advertisement for wire status messages. + pub fn advertised_max_blocks_per_response(&self) -> u32 { + clamp_advertised_blocks(self.max_blocks_per_response) + } + + /// Return the locally capped in-flight advertisement for status messages. + pub fn advertised_max_inflight_requests(&self) -> u16 { + clamp_advertised_inflight(self.max_inflight_requests) + } + + /// Return the non-zero response byte advertisement for status messages. + pub fn advertised_max_response_bytes(&self) -> u32 { + clamp_advertised_response_bytes(self.max_response_bytes) + } + + /// Per-peer in-flight byte cap derived from the global budget and expected fanout. + /// + /// Floored at one advertised response so a peer can always reserve at least a + /// full response (otherwise a tiny budget divided by `expected_peers` would + /// starve every peer); the fair-share cap only binds above that. Returns + /// `u64::MAX` (no per-peer limit) when `expected_peers` is `0`. + pub fn per_peer_byte_cap(&self) -> u64 { + match self.expected_peers { + 0 => u64::MAX, + peers => (self.max_inflight_block_bytes / peers as u64) + .max(u64::from(self.advertised_max_response_bytes())), + } + } + + /// Build the inert local status used before the block-sync reactor is wired. + pub fn initial_status(&self) -> BlockSyncStatus { + BlockSyncStatus { + max_blocks_per_response: self.advertised_max_blocks_per_response(), + max_inflight_requests: self.advertised_max_inflight_requests(), + max_response_bytes: self.advertised_max_response_bytes(), + ..BlockSyncStatus::default() + } + } +} + +/// Clamp an advertised block count to the hard stream-6 request cap. +pub fn clamp_advertised_blocks(count: u32) -> u32 { + count.clamp(1, MAX_BS_BLOCKS_PER_REQUEST) +} + +/// Clamp an advertised in-flight request count to the local status ceiling. +pub fn clamp_advertised_inflight(count: u16) -> u16 { + count.clamp(1, DEFAULT_BS_MAX_INFLIGHT) +} + +/// Clamp an advertised response byte target to the largest stream-6 message. +pub fn clamp_advertised_response_bytes(bytes: u32) -> u32 { + bytes.clamp(1, MAX_BS_RESPONSE_BYTES) +} + +/// Maximum inbound `GetBlocks.count` this node will serve before looking at body sizes. +pub fn inbound_get_blocks_count_limit(config: &ZakuraBlockSyncConfig) -> u32 { + config + .advertised_max_blocks_per_response() + .clamp(1, MAX_BS_BLOCKS_PER_REQUEST) +} diff --git a/zebra-network/src/zakura/block_sync/error.rs b/zebra-network/src/zakura/block_sync/error.rs new file mode 100644 index 00000000000..841babe027d --- /dev/null +++ b/zebra-network/src/zakura/block_sync/error.rs @@ -0,0 +1,77 @@ +use super::*; + +/// Structured wire and stateless-validation errors for stream 6. +#[derive(Debug, Error)] +pub enum BlockSyncWireError { + /// A payload or peer-controlled count exceeded its cap. + #[error("Zakura block-sync payload length {actual} exceeds cap {max}")] + OversizedPayload { + /// Actual payload length. + actual: usize, + /// Maximum allowed payload length. + max: usize, + }, + + /// A decoded request or response block count exceeded its contract. + #[error("Zakura block-sync block count {actual} exceeds cap {max}")] + BlockCountLimit { + /// Actual count. + actual: u32, + /// Maximum allowed count. + max: u32, + }, + + /// A `GetBlocks`, `BlocksDone`, or `RangeUnavailable` count was zero. + #[error("Zakura block-sync count must be non-zero")] + ZeroBlockCount, + + /// A decoded block body exceeded the consensus block-size limit. + #[error("Zakura block-sync block length {actual} exceeds consensus cap {max}")] + OversizedBlock { + /// Actual serialized block length. + actual: usize, + /// Maximum allowed serialized block length. + max: usize, + }, + + /// A height exceeded Zebra's supported height range. + #[error("Zakura block-sync height {0} exceeds supported range")] + HeightOutOfRange(u32), + + /// A payload used an unknown stream-6 message discriminator. + #[error("unknown Zakura block-sync message type {0}")] + UnknownMessageType(u8), + + /// A frame used a message type that does not fit stream-6's u8 discriminator. + #[error("unknown Zakura block-sync frame message type {0}")] + UnknownFrameMessageType(u16), + + /// Frame flags are reserved in stream 6. + #[error("unsupported Zakura block-sync frame flags {0}")] + UnsupportedFlags(u16), + + /// Frame and payload message types disagreed. + #[error("Zakura block-sync frame type {frame} disagrees with payload type {payload}")] + MismatchedFrameMessageType { + /// Outer frame message type. + frame: u16, + /// Inner payload message type. + payload: u8, + }, + + /// A decoded payload had trailing bytes. + #[error("trailing bytes in Zakura block-sync payload")] + TrailingBytes, + + /// A numeric conversion failed while handling bounded data. + #[error("numeric overflow while encoding Zakura block-sync {0}")] + NumericOverflow(&'static str), + + /// An I/O error while encoding or decoding. + #[error("Zakura block-sync wire I/O error: {0}")] + Io(#[from] io::Error), + + /// Zcash serialization failed. + #[error("Zakura block-sync Zcash serialization error: {0}")] + Serialization(#[from] SerializationError), +} diff --git a/zebra-network/src/zakura/block_sync/events.rs b/zebra-network/src/zakura/block_sync/events.rs new file mode 100644 index 00000000000..8ae2a6e847a --- /dev/null +++ b/zebra-network/src/zakura/block_sync/events.rs @@ -0,0 +1,173 @@ +use super::{scheduler::*, state::*, wire::*, *}; + +/// Committed header metadata used by block sync to schedule and validate a body. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct BlockSyncBlockMeta { + /// Header-known block height whose body is missing. + pub height: block::Height, + /// Committed header hash expected from the downloaded body. + pub hash: block::Hash, + /// Advisory or confirmed body-size estimate for scheduling. + pub size: BlockSizeEstimate, +} + +/// Facts accepted by the block-sync scaffold and later reactor. +#[derive(Clone, Debug)] +pub enum BlockSyncEvent { + /// A peer became available for stream-6 block sync. + PeerConnected(BlockSyncPeerSession), + /// A peer disconnected; all of its outstanding work is dropped. + PeerDisconnected(ZakuraPeerId), + /// Inbound stream-6 message from `peer`. + WireMessage { + /// Serving peer. + peer: ZakuraPeerId, + /// Decoded stream-6 message. + msg: BlockSyncMessage, + }, + /// Stream-6 frame decoding failed after handler admission. + WireDecodeFailed { + /// Peer that sent the malformed frame. + peer: ZakuraPeerId, + /// Decode/validation error. + error: Arc, + }, + /// Header sync advanced the committed header target. + HeaderTipChanged { + /// Current best header height. + height: block::Height, + /// Current best header hash. + hash: block::Hash, + }, + /// Locally observed finalized or verified-body frontiers changed. + StateFrontiersChanged(BlockSyncFrontiers), + /// State grew the verified body chain tip. + ChainTipGrow(BlockSyncFrontiers), + /// State reset the verified body chain tip after a rollback, best-chain switch, + /// activation boundary, or coalesced multi-block tip update. + ChainTipReset(BlockSyncFrontiers), + /// Driver returned the current body-missing, header-known heights with committed hashes. + NeededBlocks(Vec), + /// Node wiring finished applying a submitted block body. + BlockApplyFinished { + /// Submission token from the matching [`BlockSyncAction::SubmitBlock`]. + token: BlockApplyToken, + /// Submitted block height. + height: block::Height, + /// Submitted block hash. + hash: block::Hash, + /// Apply result from the verifier driver. + result: BlockApplyResult, + /// Locally observed chain frontier after the apply attempt completed. + local_frontier: Option, + }, + /// Node wiring finished or abandoned a `Block` response to an inbound `GetBlocks`. + BlockRangeResponseFinished { + /// Peer whose served-response slot can be released. + peer: ZakuraPeerId, + /// First requested height. + start_height: block::Height, + /// Requested block count. + requested_count: u32, + /// Number of blocks read from state and sent in the response. + returned_count: u32, + }, + /// State returned committed bodies requested by a peer and the reactor should send them. + BlockRangeResponseReady { + /// Peer whose inbound request is being served. + peer: ZakuraPeerId, + /// First requested height. + start_height: block::Height, + /// Requested block count. + requested_count: u32, + /// Bounded committed blocks returned by state. + blocks: Vec<(block::Height, Arc, usize)>, + }, +} + +/// Result of applying a block-sync body through the verifier driver. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum BlockApplyResult { + /// The block was verified and committed. + Committed, + /// The verifier reported the block was already committed. + Duplicate, + /// The verifier rejected the block. + Rejected, + /// The verifier did not answer before the driver timeout. + TimedOut, +} + +/// Monotonic token assigned by the reactor to each verifier submission. +/// +/// The verifier can return stale duplicate completions after a reset and +/// resubmission of the same height/hash. Echoing this token lets the reactor +/// ignore those stale completions instead of releasing a newer in-flight body. +pub type BlockApplyToken = u64; + +/// Actions emitted by the future block-sync reactor for the service seam. +#[derive(Clone, Debug)] +pub enum BlockSyncAction { + /// Queue a typed stream-6 message to a peer. + SendMessage { + /// Destination peer. + peer: ZakuraPeerId, + /// Message that should be written to the peer's stream. + msg: BlockSyncMessage, + }, + /// Ask node wiring to read `missing_block_bodies`, header hashes, and size hints. + QueryNeededBlocks { + /// Current verified body tip. + verified_block_tip: block::Height, + /// Current best header target. + best_header_tip: block::Height, + }, + /// Ask node wiring to read committed bodies for an inbound `GetBlocks`. + QueryBlocksByHeightRange { + /// Peer that requested the range. + peer: ZakuraPeerId, + /// First height. + start: block::Height, + /// Maximum count. + count: u32, + }, + /// Parent-first body ready for B3's verifier/commit driver. + SubmitBlock { + /// Submission token to echo in [`BlockSyncEvent::BlockApplyFinished`]. + token: BlockApplyToken, + /// Block body that is contiguous above `verified_block_tip`. + block: Arc, + }, + /// Report peer misbehavior to the supervisor. + Misbehavior { + /// Misbehaving peer. + peer: ZakuraPeerId, + /// Reason for reporting. + reason: BlockSyncMisbehavior, + }, +} + +/// Block-sync peer-accounting violations. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum BlockSyncMisbehavior { + /// A stream-6 payload was malformed before semantic handling. + MalformedMessage, + /// A peer sent blocks that were not requested. + UnsolicitedBlock, + /// A peer requested more blocks than this node advertised it can serve. + GetBlocksTooLong, + /// A peer exceeded this node's inbound `GetBlocks` serving budget. + GetBlocksSpam, + /// A peer supplied a body whose hash or size does not match committed metadata. + InvalidBlock, + /// A peer supplied a body outside the tolerated scheduling-size deviation. + SizeMismatch, + /// Peer status is internally impossible. + InvalidStatus, + /// A response terminator arrived without an outstanding range. + UnsolicitedDone, + /// A peer reported a requested range unavailable. + RangeUnavailable, + /// A peer sent too many status frames. + StatusSpam, +} diff --git a/zebra-network/src/zakura/block_sync/mod.rs b/zebra-network/src/zakura/block_sync/mod.rs new file mode 100644 index 00000000000..e4aafc9c7e8 --- /dev/null +++ b/zebra-network/src/zakura/block_sync/mod.rs @@ -0,0 +1,59 @@ +//! Native Zakura block-sync stream messages and service scaffold. + +use std::{ + collections::{BTreeMap, HashMap, HashSet, VecDeque}, + io::{self, Cursor, Read, Write}, + sync::{Arc, Mutex as StdMutex}, + time::{Duration, Instant}, +}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{ + sync::{mpsc, watch}, + task::{self, JoinHandle}, + time, +}; +use tokio_util::sync::CancellationToken; +use zebra_chain::{ + block, + serialization::{SerializationError, ZcashDeserialize, ZcashSerialize}, +}; + +use super::{ + trace::{block_sync_trace as bs_trace, peer_label as trace_peer_label, BLOCK_SYNC_TABLE}, + Frame, ServicePeerDirection, ServicePeerLimits, ZakuraPeerId, ZakuraTrace, +}; + +mod config; +mod error; +mod events; +mod pipe; +mod reactor; +mod reorder; +mod scheduler; +mod service; +mod state; +#[cfg(test)] +mod tests; +mod wire; + +pub use config::{BlockSyncStatus, ZakuraBlockSyncConfig, MAX_BS_RESPONSE_BYTES}; +pub use error::BlockSyncWireError; +pub use events::{ + BlockApplyResult, BlockApplyToken, BlockSyncAction, BlockSyncBlockMeta, BlockSyncEvent, + BlockSyncMisbehavior, +}; +pub use reactor::spawn_block_sync_reactor; +pub use scheduler::BlockSizeEstimate; +#[cfg(test)] +pub(crate) use service::block_sync_streams; +pub use service::BlockSyncPeerSession; +pub(crate) use service::{BlockSyncService, MAX_BS_FRAME_BYTES}; +pub use state::{BlockSyncFrontiers, BlockSyncHandle, BlockSyncStartup}; +pub use wire::{ + BlockSyncMessage, MAX_BS_BLOCKS_PER_REQUEST, MAX_BS_MESSAGE_BYTES, MSG_BS_BLOCK, + MSG_BS_BLOCKS_DONE, MSG_BS_GET_BLOCKS, MSG_BS_RANGE_UNAVAILABLE, MSG_BS_STATUS, + ZAKURA_BLOCK_SYNC_STREAM_VERSION, ZAKURA_CAP_BLOCK_SYNC, ZAKURA_STREAM_BLOCK_SYNC, +}; diff --git a/zebra-network/src/zakura/block_sync/pipe.rs b/zebra-network/src/zakura/block_sync/pipe.rs new file mode 100644 index 00000000000..0486372a6e6 --- /dev/null +++ b/zebra-network/src/zakura/block_sync/pipe.rs @@ -0,0 +1,452 @@ +//! block_sync/pipe.rs — the per-peer block-sync pipe (stream 6). +//! +//! THE PHASE-3A DAG SLICE IS THIS DIAGRAM. The code below is a mechanical +//! transcription; the [`PIPE_SHAPE`] const is the inspectable, drift-checked +//! copy of it. +//! +//! recv ─▶ guard ─▶ decode ─▶ branch(msg) +//! ├─ Status ─▶ emit(WireMessage) +//! ├─ GetBlocks ─▶ emit(WireMessage) +//! ├─ Block ─▶ emit(WireMessage) +//! ├─ BlocksDone ─▶ emit(WireMessage) +//! └─ RangeUnavailable ─▶ emit(WireMessage) +//! +//! Phase 3a lifts stream-6 inbound processing onto the shared pipe while +//! retaining the compatibility reactor for semantic validate/mutate/emit work. +//! The pipe owns the single frame decode path used by both production and +//! `Service::deliver_frame`; decoded messages still flow to the reactor as +//! `WireMessage` events. + +use std::sync::Arc; + +use super::{events::*, wire::*, *}; +use crate::zakura::{ + Admit, Edge, Flow, FramedRecv, Node, NodeKind, PipeCx, PipeShape, SessionGuard, SinkReject, + ZakuraPeerId, +}; + +pub(super) fn block_sync_guard() -> SessionGuard { + // The transport already applies the per-connection count bucket and frame + // cap; this guard adds the same payload cap the codec enforces. Type + // validity is left to the decode stage on purpose: a disallowed or unknown + // stream-6 type must surface as `WireDecodeFailed` so the reactor records + // `MalformedMessage` misbehavior, rather than a pre-decode guard reject + // dropping that signal (see BS1). The block-sync byte budget likewise stays + // in the reactor scheduler/reorder state so existing request/retry + // accounting is not double-counted. + SessionGuard::oversize_only(MAX_BS_MESSAGE_BYTES as u32) +} + +/// Per-peer block-sync local state. +/// +/// Phase 3a has no peer-local block-sync state to move yet; the semantic peer +/// state remains in the compatibility reactor. +pub(super) struct BsLocal; + +/// Shared environment handed to every block-sync pipe. +#[derive(Clone)] +pub(super) struct BsEnv { + /// Bounded reactor event queue used for decoded stream-6 wire events. + events: mpsc::Sender, +} + +impl BsEnv { + /// Wrap the cloneable reactor event sender as the pipe's shared environment. + pub(super) fn new(events: mpsc::Sender) -> Self { + Self { events } + } +} + +/// The Phase-3a block-sync pipe DAG slice, as checked documentation. +pub(super) const PIPE_SHAPE: PipeShape = PipeShape { + service: "block-sync", + nodes: &[ + Node { + id: "guard", + kind: NodeKind::Guard, + }, + Node { + id: "decode", + kind: NodeKind::Decode, + }, + Node { + id: "branch", + kind: NodeKind::Branch, + }, + Node { + id: "emit", + kind: NodeKind::Emit, + }, + ], + edges: &[ + Edge { + from: "guard", + to: "decode", + on: "Pass", + }, + Edge { + from: "decode", + to: "branch", + on: "Ok", + }, + Edge { + from: "branch", + to: "emit", + on: "Status", + }, + Edge { + from: "branch", + to: "emit", + on: "GetBlocks", + }, + Edge { + from: "branch", + to: "emit", + on: "Block", + }, + Edge { + from: "branch", + to: "emit", + on: "BlocksDone", + }, + Edge { + from: "branch", + to: "emit", + on: "RangeUnavailable", + }, + ], +}; + +/// Executable transcription of [`PIPE_SHAPE`] — the production entry function. +/// +/// The shared guard has already admitted this frame before `run_inbound` is +/// reached. Decode failures are forwarded as `WireDecodeFailed` events and +/// reject the peer, matching the old sink path. Successful decodes branch by +/// message variant, then every compatibility branch emits the same `WireMessage` +/// event for the retained reactor to handle semantically. +pub(super) fn run_inbound(cx: &mut PipeCx<'_, BsLocal, BsEnv>, frame: Frame) -> Flow<()> { + let msg = match decode(&cx.env.events, cx.peer_id.clone(), frame) { + Flow::Continue(msg) => msg, + Flow::Done => return Flow::Done, + Flow::Reject(reject) => return Flow::Reject(reject), + }; + + match msg { + BlockSyncMessage::Status(_) + | BlockSyncMessage::GetBlocks { .. } + | BlockSyncMessage::Block(_) + | BlockSyncMessage::BlocksDone { .. } + | BlockSyncMessage::RangeUnavailable { .. } => forward( + &cx.env.events, + BlockSyncEvent::WireMessage { + peer: cx.peer_id.clone(), + msg, + }, + ), + } +} + +/// Run one production block-sync peer until stream close, cancellation, or reject. +/// +/// Unlike the generic synchronous pipe runner, this loop awaits the reactor +/// event queue after decoding a frame. That preserves ordered-stream semantics: +/// if the reactor is full, the stream reader backpressures before reading the +/// next QUIC frame instead of consuming a block body and then losing it on a +/// failed `try_send`. +pub(super) async fn run_peer( + peer_id: ZakuraPeerId, + mut recv: FramedRecv, + events: mpsc::Sender, + cancel: CancellationToken, +) -> Result<(), SinkReject> { + let mut guard = block_sync_guard(); + + loop { + let frame = tokio::select! { + () = cancel.cancelled() => return Ok(()), + frame = recv.recv() => frame, + }; + let Some(frame) = frame else { + return Ok(()); + }; + + match guard.admit(&frame) { + Admit::Pass => {} + Admit::Throttle => { + return Err(SinkReject::local( + "block-sync guard unexpectedly throttled an inbound frame", + )); + } + Admit::Reject(reason) => { + return Err(SinkReject::protocol(std::io::Error::new( + std::io::ErrorKind::InvalidData, + reason, + ))); + } + } + + let msg = match BlockSyncMessage::decode_frame(frame) { + Ok(msg) => msg, + Err(error) => { + let protocol_error = + std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); + send_event( + &events, + BlockSyncEvent::WireDecodeFailed { + peer: peer_id.clone(), + error: Arc::new(error), + }, + ) + .await?; + return Err(SinkReject::protocol(protocol_error)); + } + }; + + send_event( + &events, + BlockSyncEvent::WireMessage { + peer: peer_id.clone(), + msg, + }, + ) + .await?; + } +} + +async fn send_event( + events: &mpsc::Sender, + event: BlockSyncEvent, +) -> Result<(), SinkReject> { + events + .send(event) + .await + .map_err(|error| SinkReject::local(format!("block-sync queue closed: {error}"))) +} + +/// The single frame decode stage shared by production and `deliver_frame`. +fn decode( + events: &mpsc::Sender, + peer_id: ZakuraPeerId, + frame: Frame, +) -> Flow { + match BlockSyncMessage::decode_frame(frame) { + Ok(msg) => Flow::Continue(msg), + Err(error) => { + // Block bodies are validated against committed headers in B1+. + let protocol_error = + std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); + let _ = events.try_send(BlockSyncEvent::WireDecodeFailed { + peer: peer_id, + error: Arc::new(error), + }); + Flow::Reject(SinkReject::protocol(protocol_error)) + } + } +} + +/// Forward a successfully decoded inbound event to the compatibility reactor. +fn forward(events: &mpsc::Sender, event: BlockSyncEvent) -> Flow<()> { + match events.try_send(event) { + Ok(()) => Flow::Continue(()), + Err(error) => Flow::Reject(SinkReject::local(format!( + "block-sync queue closed: {error}" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::super::service::block_sync_pipe; + use super::*; + use crate::zakura::transport::framed_channel; + use std::time::Duration; + use tokio::time; + + const MESSAGE_BRANCHES: [&str; 5] = [ + "Status", + "GetBlocks", + "Block", + "BlocksDone", + "RangeUnavailable", + ]; + + fn peer() -> ZakuraPeerId { + ZakuraPeerId::new(vec![7; 32]).expect("test peer id is within bounds") + } + + /// A disallowed/unknown stream-6 frame type must reach the decode stage and + /// surface as `WireDecodeFailed` (so the reactor records `MalformedMessage`), + /// not get silently dropped by a pre-decode guard reject. This is the BS1 + /// regression guard: it builds the *real* production pipe, so reverting the + /// guard back to an allowed-type filter would make this fail. + #[test] + fn run_one_unknown_type_reaches_decode_and_emits_wire_decode_failed() { + let (events_tx, mut events_rx) = mpsc::channel(4); + let mut pipe = block_sync_pipe(peer(), events_tx); + + let flow = pipe.run_one(Frame { + // 99 is outside the stream-6 message-type set; the codec rejects it. + message_type: 99, + flags: 0, + payload: Vec::new(), + }); + + assert!( + matches!(flow, Flow::Reject(SinkReject::Protocol(_))), + "unknown type rejects the peer" + ); + assert!( + matches!( + events_rx.try_recv(), + Ok(BlockSyncEvent::WireDecodeFailed { .. }) + ), + "unknown type still reports malformed-message misbehavior" + ); + } + + /// A malformed payload of an otherwise-allowed type also decodes-then-rejects + /// with `WireDecodeFailed`, unchanged by the BS1 fix. + #[test] + fn run_one_malformed_payload_emits_wire_decode_failed() { + let (events_tx, mut events_rx) = mpsc::channel(4); + let mut pipe = block_sync_pipe(peer(), events_tx); + + let flow = pipe.run_one(Frame { + message_type: u16::from(MSG_BS_STATUS), + flags: 0, + payload: Vec::new(), + }); + + assert!(matches!(flow, Flow::Reject(SinkReject::Protocol(_)))); + assert!(matches!( + events_rx.try_recv(), + Ok(BlockSyncEvent::WireDecodeFailed { .. }) + )); + } + + /// A well-formed stream-6 message decodes and forwards a `WireMessage` to the + /// reactor, continuing the peer. + #[test] + fn run_one_valid_message_forwards_wire_message() { + let (events_tx, mut events_rx) = mpsc::channel(4); + let peer = peer(); + let mut pipe = block_sync_pipe(peer.clone(), events_tx); + + let message = BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 4, + }; + let frame = message.clone().encode_frame().expect("message encodes"); + + let flow = pipe.run_one(frame); + + assert!(matches!(flow, Flow::Continue(()))); + match events_rx.try_recv() { + Ok(BlockSyncEvent::WireMessage { peer: got, msg }) => { + assert_eq!(got, peer); + assert_eq!(msg, message); + } + other => panic!("expected WireMessage, got {other:?}"), + } + } + + #[tokio::test] + async fn run_peer_waits_when_reactor_queue_is_full() { + let (events_tx, mut events_rx) = mpsc::channel(1); + let peer = peer(); + events_tx + .send(BlockSyncEvent::PeerDisconnected(peer.clone())) + .await + .expect("test event queue has capacity"); + + let (send, recv) = framed_channel(4); + let cancel = CancellationToken::new(); + let task = tokio::spawn(run_peer(peer.clone(), recv, events_tx, cancel.clone())); + + let message = BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 4, + }; + send.send(message.clone().encode_frame().expect("message encodes")) + .await + .expect("test stream has capacity"); + + tokio::task::yield_now().await; + assert!( + !task.is_finished(), + "full reactor queue must backpressure the peer task, not drop/reject the frame" + ); + + assert!(matches!( + events_rx.recv().await, + Some(BlockSyncEvent::PeerDisconnected(_)) + )); + + match time::timeout(Duration::from_secs(1), events_rx.recv()) + .await + .expect("frame should be forwarded after queue space opens") + { + Some(BlockSyncEvent::WireMessage { peer: got, msg }) => { + assert_eq!(got, peer); + assert_eq!(msg, message); + } + other => panic!("expected WireMessage, got {other:?}"), + } + + drop(send); + assert!( + time::timeout(Duration::from_secs(1), task) + .await + .expect("peer task exits after stream close") + .expect("peer task does not panic") + .is_ok(), + "peer task exits cleanly" + ); + cancel.cancel(); + } + + #[test] + fn pipe_shape_matches_runtime() { + // (a) The declared shape is internally consistent. + PIPE_SHAPE + .validate() + .expect("block-sync PIPE_SHAPE edges name only real nodes"); + + // (b) Phase 3a's real runtime branch is the decoded stream-6 message + // variant. Semantic handling remains in the compatibility reactor, so + // every branch currently terminates at the single forward/emit stage. + let branches: Vec<&str> = PIPE_SHAPE + .edges + .iter() + .filter(|edge| edge.from == "branch") + .map(|edge| edge.on) + .collect(); + + assert_eq!( + branches.len(), + MESSAGE_BRANCHES.len(), + "branch has exactly one edge per decoded stream-6 message variant" + ); + for branch in MESSAGE_BRANCHES { + assert!( + branches.contains(&branch), + "branch edge missing for runtime message variant {branch}" + ); + } + + assert!( + PIPE_SHAPE + .edges + .iter() + .any(|edge| edge.from == "guard" && edge.to == "decode"), + "admitted frames decode before branch" + ); + assert!( + PIPE_SHAPE + .nodes + .iter() + .any(|node| node.id == "emit" && matches!(node.kind, NodeKind::Emit)), + "the pipe terminates at a single compatibility emit node" + ); + } +} diff --git a/zebra-network/src/zakura/block_sync/reactor.rs b/zebra-network/src/zakura/block_sync/reactor.rs new file mode 100644 index 00000000000..f66f52a8e9f --- /dev/null +++ b/zebra-network/src/zakura/block_sync/reactor.rs @@ -0,0 +1,2125 @@ +use super::{config::*, events::*, reorder::*, scheduler::*, state::*, wire::*, *}; +use crate::zakura::{ + FrontierChange, FrontierUpdate, ServiceAdmissionDecision, ServicePeerDirection, + ServicePeerSnapshot, ZakuraBlockSyncCandidateState, +}; +use iroh::NodeId; + +const SOFT_MISBEHAVIOR_DISCONNECT_THRESHOLD: u32 = 3; + +/// Upper bound on how long the reactor will wait to enqueue a data-plane action +/// before abandoning it. The bounded `actions` channel is normally drained by +/// the action driver almost immediately; this deadline only trips when that +/// driver is genuinely stalled on backend/verifier work, and it keeps a stalled +/// driver from wedging the reactor's control plane — peer-lifecycle draining, +/// request timeouts, and above all misbehavior disconnects. +const ACTION_SEND_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +enum OutstandingRangeDisposition { + Satisfied, + RetryOriginal, + RetryMissing, +} + +/// Spawn a block-sync reactor and return its handle plus action stream. +pub fn spawn_block_sync_reactor( + startup: BlockSyncStartup, +) -> ( + BlockSyncHandle, + mpsc::Receiver, + JoinHandle<()>, +) { + debug_assert!( + !startup.state_queries_enabled + || (startup.header_tip.is_some() ^ startup.frontier_updates.is_some()), + "state-backed block sync must have exactly one frontier source", + ); + + let state = BlockSyncState::new(&startup); + let (events_tx, events_rx) = + mpsc::channel(startup.config.peer_limits.inbound_queue_depth.max(1)); + let (lifecycle_tx, lifecycle_rx) = mpsc::unbounded_channel(); + let (actions_tx, actions_rx) = mpsc::channel(128); + let (peers_tx, peers_rx) = watch::channel(state.peer_snapshot(startup.config.peer_limits)); + let (status_tx, status_rx) = watch::channel(state.last_advertised_status); + let (candidates_tx, candidates_rx) = watch::channel(ZakuraBlockSyncCandidateState::default()); + + let handle = BlockSyncHandle { + events: events_tx, + lifecycle: lifecycle_tx, + peers: peers_rx, + status: status_rx, + candidates: candidates_rx, + }; + let reactor = BlockSyncReactor { + startup, + state, + events: events_rx, + lifecycle: lifecycle_rx, + actions: actions_tx, + peers: peers_tx, + status: status_tx, + candidates: candidates_tx, + }; + let task = tokio::spawn(reactor.run()); + + (handle, actions_rx, task) +} + +#[derive(Debug)] +pub(super) struct BlockSyncReactor { + startup: BlockSyncStartup, + state: BlockSyncState, + events: mpsc::Receiver, + lifecycle: mpsc::UnboundedReceiver, + actions: mpsc::Sender, + peers: watch::Sender, + status: watch::Sender, + candidates: watch::Sender, +} + +impl BlockSyncReactor { + async fn run(mut self) { + let mut header_tip = self.startup.header_tip.clone(); + let mut header_tip_open = header_tip.is_some(); + let mut frontier_updates = self.startup.frontier_updates.clone(); + let mut frontier_updates_open = frontier_updates.is_some(); + let mut ticks = time::interval(self.startup.config.request_timeout); + let mut status_ticks = time::interval( + self.startup + .config + .status_refresh_interval + .max(Duration::from_millis(1)), + ); + + if !self.query_needed_blocks().await { + self.pause_new_body_downloads(); + } + self.release_caught_up_block_sync_peers(); + self.publish_metrics(); + self.trace_sync_state(); + loop { + tokio::select! { + biased; + _ = self.startup.shutdown.cancelled() => break, + event = self.lifecycle.recv() => { + let Some(event) = event else { break }; + self.handle_event(event).await; + } + event = self.events.recv() => { + let Some(event) = event else { break }; + self.handle_event(event).await; + } + changed = async { + match header_tip.as_mut() { + Some(header_tip) => header_tip.changed().await, + None => std::future::pending().await, + } + }, if header_tip_open => { + match changed { + Ok(()) => { + let header_tip = header_tip + .as_mut() + .expect("header tip receiver exists while header_tip_open is true"); + let (height, hash) = *header_tip.borrow_and_update(); + self.handle_header_tip_changed(height, hash).await; + self.publish_metrics(); + } + Err(_) => header_tip_open = false, + } + } + changed = async { + match frontier_updates.as_mut() { + Some(frontier_updates) => frontier_updates.changed().await, + None => std::future::pending().await, + } + }, if frontier_updates_open => { + match changed { + Ok(()) => { + let frontier_updates = frontier_updates + .as_mut() + .expect("frontier update receiver exists while frontier_updates_open is true"); + let update = *frontier_updates.borrow_and_update(); + self.handle_frontier_update(update).await; + self.publish_metrics(); + } + Err(_) => frontier_updates_open = false, + } + } + _ = ticks.tick() => { + self.handle_timeouts().await; + self.publish_metrics(); + self.trace_sync_state(); + } + _ = status_ticks.tick() => self.flush_status_refresh().await, + } + } + } + + async fn handle_event(&mut self, event: BlockSyncEvent) { + self.trace_event_received(&event); + match event { + BlockSyncEvent::PeerConnected(session) => self.handle_peer_connected(session).await, + BlockSyncEvent::PeerDisconnected(peer) => self.handle_peer_disconnected(peer), + BlockSyncEvent::WireMessage { peer, msg } => self.handle_wire_message(peer, msg).await, + BlockSyncEvent::WireDecodeFailed { peer, error } => { + self.handle_wire_decode_failed(peer, error).await + } + BlockSyncEvent::HeaderTipChanged { height, hash } => { + self.handle_header_tip_changed(height, hash).await + } + BlockSyncEvent::StateFrontiersChanged(frontiers) => { + self.handle_state_frontiers_changed(frontiers).await + } + BlockSyncEvent::ChainTipGrow(frontiers) => { + self.handle_state_frontiers_changed(frontiers).await + } + BlockSyncEvent::ChainTipReset(frontiers) => { + self.handle_chain_tip_reset(frontiers).await + } + BlockSyncEvent::NeededBlocks(blocks) => { + self.handle_needed_blocks(blocks).await; + } + BlockSyncEvent::BlockApplyFinished { + token, + height, + hash, + result, + local_frontier, + } => { + self.handle_block_apply_finished(token, height, hash, result, local_frontier) + .await + } + BlockSyncEvent::BlockRangeResponseReady { + peer, + start_height, + requested_count, + blocks, + } => { + self.handle_block_range_response_ready(peer, start_height, requested_count, blocks) + .await; + } + BlockSyncEvent::BlockRangeResponseFinished { + peer, + start_height, + requested_count, + returned_count, + } => { + self.handle_block_range_response_finished( + peer, + start_height, + requested_count, + returned_count, + ) + .await; + } + } + self.publish_metrics(); + } + + fn admission_decision_for( + &self, + peer: &ZakuraPeerId, + direction: ServicePeerDirection, + ) -> ServiceAdmissionDecision { + if self.state.peers.contains_key(peer) { + return ServiceAdmissionDecision::Admit; + } + + let limits = self.startup.config.peer_limits; + let admitted = self.admitted_count(direction); + let cap = match direction { + ServicePeerDirection::Inbound => limits.max_inbound_peers, + ServicePeerDirection::Outbound => limits.max_outbound_peers, + }; + + if admitted >= cap { + ServiceAdmissionDecision::RejectFull + } else { + ServiceAdmissionDecision::Admit + } + } + + fn admitted_count(&self, direction: ServicePeerDirection) -> usize { + self.state + .peers + .values() + .filter(|peer| peer.direction == direction) + .count() + } + + fn publish_peer_snapshot(&self) { + let _ = self + .peers + .send(self.state.peer_snapshot(self.startup.config.peer_limits)); + } + + fn publish_candidate_state(&self) { + let has_body_gaps = !self.state.needed_heights.is_empty(); + let mut admitted_node_ids: Vec<_> = self + .state + .peers + .iter() + .filter_map(|(peer_id, peer)| { + if has_body_gaps && !peer.can_serve_any(&self.state.needed_heights) { + return None; + } + + node_id_from_block_peer_id(peer_id) + }) + .collect(); + admitted_node_ids.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes())); + admitted_node_ids.dedup(); + + let _ = self.candidates.send(ZakuraBlockSyncCandidateState { + missing_block_bodies: self.state.needed_heights.clone(), + admitted_node_ids, + }); + } + + async fn handle_peer_connected(&mut self, session: BlockSyncPeerSession) { + let peer = session.peer_id().clone(); + let direction = session.direction(); + self.state.disconnected_peers.remove(&peer); + let decision = self.admission_decision_for(&peer, direction); + if decision != ServiceAdmissionDecision::Admit { + self.state.parked_peers.insert(peer); + session.cancel_token().cancel(); + self.publish_peer_snapshot(); + self.publish_candidate_state(); + return; + } + + self.state.parked_peers.remove(&peer); + self.state + .peers + .entry(peer.clone()) + .and_modify(|peer_state| { + peer_state.session = session.clone(); + peer_state.direction = direction; + }) + .or_insert_with(|| PeerBlockState::new(session, &self.startup.config)); + self.publish_peer_snapshot(); + self.publish_candidate_state(); + self.send_status(&peer).await; + self.schedule().await; + } + + fn handle_peer_disconnected(&mut self, peer: ZakuraPeerId) { + if let Some(peer_state) = self.state.peers.remove(&peer) { + for outstanding in peer_state.outstanding.into_iter().rev() { + self.finish_detached_outstanding( + outstanding, + OutstandingRangeDisposition::RetryMissing, + ); + } + self.state.disconnected_peers.insert(peer.clone()); + } + self.state.parked_peers.remove(&peer); + self.state.schedule.forget_peer(&peer); + self.publish_peer_snapshot(); + self.publish_candidate_state(); + } + + async fn handle_header_tip_changed(&mut self, height: block::Height, hash: block::Hash) { + self.state.best_header_tip = height; + self.state.best_header_hash = hash; + if !self.query_needed_blocks().await { + self.pause_new_body_downloads(); + } + self.release_caught_up_block_sync_peers(); + } + + async fn handle_frontier_update(&mut self, update: FrontierUpdate) { + let frontier = update.frontier; + let state_frontiers = BlockSyncFrontiers { + finalized_height: frontier.finalized.height, + verified_block_tip: frontier.verified_body.height, + verified_block_hash: frontier.verified_body.hash, + }; + match update.change { + FrontierChange::Snapshot => { + self.handle_header_tip_changed( + frontier.best_header.height, + frontier.best_header.hash, + ) + .await; + self.handle_state_frontiers_changed(state_frontiers).await; + } + FrontierChange::HeaderAdvanced => { + self.handle_header_tip_changed( + frontier.best_header.height, + frontier.best_header.hash, + ) + .await; + if frontier.verified_body.height > self.state.verified_block_tip { + self.handle_state_frontiers_changed(state_frontiers).await; + } + } + FrontierChange::HeaderReanchored => { + self.state.best_header_tip = frontier.best_header.height; + self.state.best_header_hash = frontier.best_header.hash; + self.handle_chain_tip_reset(state_frontiers).await; + } + FrontierChange::VerifiedGrow => { + self.handle_state_frontiers_changed(state_frontiers).await; + if frontier.best_header.height > self.state.best_header_tip { + self.handle_header_tip_changed( + frontier.best_header.height, + frontier.best_header.hash, + ) + .await; + } + } + FrontierChange::VerifiedReset => { + self.handle_chain_tip_reset(state_frontiers).await; + if frontier.best_header.height > self.state.best_header_tip { + self.handle_header_tip_changed( + frontier.best_header.height, + frontier.best_header.hash, + ) + .await; + } + } + } + } + + async fn handle_state_frontiers_changed(&mut self, frontiers: BlockSyncFrontiers) { + if let Some(old_serving_tip) = self.apply_state_frontiers_changed(frontiers, true).await { + self.finish_frontier_update(old_serving_tip).await; + } + } + + async fn apply_state_frontiers_changed( + &mut self, + frontiers: BlockSyncFrontiers, + release_applied: bool, + ) -> Option<(block::Height, block::Hash)> { + self.state.finalized_height = self.state.finalized_height.max(frontiers.finalized_height); + if frontiers.verified_block_tip < self.state.verified_block_tip { + tracing::debug!( + current = ?self.state.verified_block_tip, + stale = ?frontiers.verified_block_tip, + "ignoring stale Zakura block-sync frontier update" + ); + return None; + } + + let old_serving_tip = (self.state.servable_high, self.state.servable_hash); + self.state.servable_high = frontiers.verified_block_tip; + self.state.servable_hash = frontiers.verified_block_hash; + self.state.verified_block_hash = frontiers.verified_block_hash; + self.state.body_download_floor = self + .state + .body_download_floor + .max(frontiers.verified_block_tip); + if frontiers.verified_block_tip != self.state.verified_block_tip { + self.state + .reorder + .drop_through(frontiers.verified_block_tip, &mut self.state.budget); + self.state + .schedule + .drop_through(frontiers.verified_block_tip); + if release_applied { + self.release_applied_blocks_through(frontiers.verified_block_tip); + } + self.drop_outstanding_through(frontiers.verified_block_tip); + self.state.verified_block_tip = frontiers.verified_block_tip; + self.trace_frontiers_changed(frontiers.verified_block_tip); + self.release_contiguous_blocks().await; + } + Some(old_serving_tip) + } + + async fn finish_frontier_update(&mut self, old_serving_tip: (block::Height, block::Hash)) { + self.queue_status_refresh_if_changed(old_serving_tip); + self.flush_status_refresh().await; + if !self.query_needed_blocks().await { + self.pause_new_body_downloads(); + } + self.release_caught_up_block_sync_peers(); + } + + async fn handle_chain_tip_reset(&mut self, frontiers: BlockSyncFrontiers) { + metrics::counter!("sync.block.reorg.reset").increment(1); + self.trace_chain_tip_reset(frontiers.verified_block_tip); + + let reset_tip_matches_submitted_body = self + .state + .applying + .get(&frontiers.verified_block_tip) + .is_none_or(|applying| applying.hash == frontiers.verified_block_hash); + + // A `Reset` can also be a coalesced forward state update. Preserve + // successor bodies only if the new tip is within our contiguous + // submitted/downloaded body floor and does not conflict with a submitted + // body at the reset height. Otherwise a forward reset can still be a + // fork switch, so old-fork bodies must be discarded. + if frontiers.verified_block_tip > self.state.verified_block_tip + && frontiers.verified_block_tip <= self.state.body_download_floor + && reset_tip_matches_submitted_body + { + self.handle_state_frontiers_changed(frontiers).await; + return; + } + + self.state.finalized_height = frontiers.finalized_height; + self.state.verified_block_tip = frontiers.verified_block_tip; + self.state.verified_block_hash = frontiers.verified_block_hash; + self.state.body_download_floor = frontiers.verified_block_tip; + let old_serving_tip = (self.state.servable_high, self.state.servable_hash); + self.state.servable_high = frontiers.verified_block_tip; + self.state.servable_hash = frontiers.verified_block_hash; + + self.state.reorder.clear(&mut self.state.budget); + self.release_all_applying_blocks(); + self.state.schedule.clear_covered_from(block::Height::MIN); + self.drop_ranges_not_in_needed(&HashMap::new()); + self.state.schedule.retain_matching_needed(&HashMap::new()); + + self.queue_status_refresh_if_changed(old_serving_tip); + self.flush_status_refresh().await; + if !self.query_needed_blocks().await { + self.pause_new_body_downloads(); + } + self.release_caught_up_block_sync_peers(); + } + + async fn handle_needed_blocks(&mut self, blocks: Vec) { + // The state reports every header-known, body-missing height above the + // download floor, but it has no visibility into our in-memory buffers. + // Heights already held in the reorder buffer (received, waiting for a + // lower gap to fill) or in `applying` (submitted, awaiting commit) must + // not be scheduled again: `refresh_needed` builds one maximal contiguous + // range and `ensure` rejects any range overlapping a queued/assigned + // one, so a held run sitting above an open gap would otherwise block the + // gap below it from ever being queued, freezing `body_download_floor` + // and re-requesting already-held blocks forever. Only schedule heights + // we do not already hold in memory. + let blocks: Vec<_> = blocks + .into_iter() + .filter(|block| { + !self.state.reorder.contains(block.height) + && !self.state.applying.contains_key(&block.height) + }) + .collect(); + + self.state.needed_heights = blocks.iter().map(|block| block.height).collect(); + self.state.needed_heights.sort_unstable(); + self.state.needed_heights.dedup(); + self.publish_candidate_state(); + + let needed = blocks + .into_iter() + .map(|block| NeededBlock { + height: block.height, + hash: block.hash, + size: block.size, + }) + .collect::>(); + let needed_hashes = needed + .iter() + .map(|block| (block.height, block.hash)) + .collect::>(); + // State queries are snapshots taken while peer responses are still in + // flight. A newer snapshot can omit heights from an active request + // because those bodies are already buffered, applying, verified, or the + // query simply raced with the response. Keep active ranges correlated + // unless state explicitly reports a different hash for one of their + // heights; otherwise valid late bodies become `UnsolicitedBlock` and + // cause avoidable peer churn. + let retention_hashes = self.needed_hashes_with_active_outstanding(&needed_hashes); + self.drop_ranges_not_in_needed(&retention_hashes); + self.state + .schedule + .retain_matching_needed(&retention_hashes); + self.state.schedule.refresh_needed(needed); + if self.should_pause_new_body_downloads() { + self.pause_new_body_downloads(); + self.release_caught_up_block_sync_peers(); + return; + } + self.schedule().await; + } + + fn clear_needed_heights(&mut self) { + if self.state.needed_heights.is_empty() { + return; + } + self.state.needed_heights.clear(); + self.publish_candidate_state(); + } + + fn pause_new_body_downloads(&mut self) { + self.clear_needed_heights(); + self.state.schedule.clear_queued(); + } + + fn body_lag(&self) -> u32 { + self.state + .best_header_tip + .0 + .saturating_sub(self.state.verified_block_tip.0) + } + + fn should_pause_new_body_downloads(&self) -> bool { + let lag = self.body_lag(); + lag == 0 + || lag <= self.startup.config.near_tip_body_download_pause_blocks + || self.state.budget.available() == 0 + } + + fn release_caught_up_block_sync_peers(&mut self) { + // Keep block-sync streams open even when this node is locally caught + // up. A synced node can still be the server a fresh peer needs for + // historical bodies, and closing the stream after every local catch-up + // starves fresh Zakura-only nodes between checkpoint windows. + } + + async fn handle_wire_decode_failed( + &mut self, + peer: ZakuraPeerId, + error: Arc, + ) { + tracing::debug!(?peer, ?error, "malformed Zakura block-sync frame"); + self.report_misbehavior(peer, BlockSyncMisbehavior::MalformedMessage) + .await; + } + + async fn handle_wire_message(&mut self, peer: ZakuraPeerId, msg: BlockSyncMessage) { + if self.state.parked_peers.contains(&peer) { + return; + } + + match msg { + BlockSyncMessage::Status(status) => self.handle_status(peer, status).await, + BlockSyncMessage::Block(block) => self.handle_block(peer, block).await, + BlockSyncMessage::BlocksDone { + start_height, + returned: _, + } => self.handle_blocks_done(peer, start_height).await, + BlockSyncMessage::RangeUnavailable { + start_height, + count, + } => { + self.handle_range_unavailable(peer, start_height, count) + .await; + } + BlockSyncMessage::GetBlocks { + start_height, + count, + } => { + self.handle_get_blocks(peer, start_height, count).await; + } + } + } + + async fn handle_status(&mut self, peer: ZakuraPeerId, status: BlockSyncStatus) { + if status.servable_low > status.servable_high { + self.report_misbehavior(peer, BlockSyncMisbehavior::InvalidStatus) + .await; + return; + } + let Some(peer_state) = self.state.peers.get_mut(&peer) else { + return; + }; + let servable_range_grew = status.servable_high > peer_state.servable_high + || status.servable_low < peer_state.servable_low; + if !peer_state.inbound_status.try_take(Instant::now()) && !servable_range_grew { + self.report_misbehavior(peer, BlockSyncMisbehavior::StatusSpam) + .await; + return; + } + peer_state.servable_low = status.servable_low; + peer_state.servable_high = status.servable_high; + peer_state.max_blocks_per_response = + clamp_advertised_blocks(status.max_blocks_per_response); + peer_state.max_inflight_requests = clamp_advertised_inflight(status.max_inflight_requests); + peer_state.max_response_bytes = clamp_advertised_response_bytes(status.max_response_bytes); + peer_state.received_status = true; + self.trace_status_received(&peer, status); + self.publish_candidate_state(); + self.schedule().await; + } + + async fn handle_block(&mut self, peer: ZakuraPeerId, block: Arc) { + let hash = block.hash(); + let Some(height) = block.coinbase_height() else { + self.report_misbehavior(peer, BlockSyncMisbehavior::InvalidBlock) + .await; + return; + }; + + let Some(peer_state) = self.state.peers.get_mut(&peer) else { + if self.ignore_disconnected_peer_response(&peer, "body") { + return; + } + if self + .ignore_stale_response(&peer, height, "body from inactive peer") + .await + { + return; + } + self.report_misbehavior(peer, BlockSyncMisbehavior::UnsolicitedBlock) + .await; + return; + }; + let Some(index) = peer_state.outstanding_index_for_height(height) else { + if self.ignore_stale_response(&peer, height, "body").await { + return; + } + if self + .ignore_unmatched_needed_response(&peer, height, "body") + .await + { + return; + } + self.report_misbehavior(peer, BlockSyncMisbehavior::UnsolicitedBlock) + .await; + return; + }; + let outstanding = &peer_state.outstanding[index]; + if outstanding.has_received(height) { + tracing::debug!(?peer, ?height, "ignoring duplicate block-sync body frame"); + return; + } + + if outstanding.request.expected_hash(height) != Some(hash) { + self.report_misbehavior(peer, BlockSyncMisbehavior::InvalidBlock) + .await; + return; + } + let estimated_bytes = outstanding.estimated_bytes_for_height(height).unwrap_or(0); + let retry_request = outstanding.request.single_height_retry(height); + + if !block_merkle_root_matches_header(block.clone()).await { + self.finish_peer_outstanding_at( + &peer, + index, + OutstandingRangeDisposition::RetryOriginal, + ); + self.report_misbehavior(peer, BlockSyncMisbehavior::InvalidBlock) + .await; + return; + } + + let serialized_bytes = match block.zcash_serialize_to_vec() { + Ok(bytes) => bytes.len() as u64, + Err(error) => { + tracing::debug!(?error, "failed to serialize decoded block-sync body"); + self.finish_peer_outstanding_at( + &peer, + index, + OutstandingRangeDisposition::RetryOriginal, + ); + self.report_misbehavior(peer, BlockSyncMisbehavior::InvalidBlock) + .await; + return; + } + }; + if serialized_bytes + > tolerated_bytes( + estimated_bytes, + self.startup.config.size_deviation_tolerance, + ) + { + self.report_misbehavior(peer.clone(), BlockSyncMisbehavior::SizeMismatch) + .await; + } + + metrics::counter!("sync.block.body.received").increment(1); + self.trace_body_received(&peer, height, serialized_bytes); + self.state.budget.release(estimated_bytes); + let mut completed = None; + if let Some(peer_state) = self.state.peers.get_mut(&peer) { + if let Some(outstanding) = peer_state.outstanding.get_mut(index) { + outstanding.mark_received(height); + if outstanding.is_complete() { + completed = Some(peer_state.outstanding.remove(index)); + } + } + } + if let Some(outstanding) = completed { + self.finish_detached_outstanding(outstanding, OutstandingRangeDisposition::Satisfied); + } + + if height <= self.state.verified_block_tip + || self.state.reorder.contains(height) + || self.state.applying.contains_key(&height) + { + self.release_contiguous_blocks().await; + self.schedule().await; + self.release_caught_up_block_sync_peers(); + return; + } + + match self + .state + .reorder + .insert(height, block, serialized_bytes, &mut self.state.budget) + { + ReorderInsertResult::Inserted => { + // A received body is now held in memory. Mark it covered so the + // retry path stops re-requesting it. `refresh_needed` already + // drops buffered heights from `needed`, but the retry path + // (`handle_timeouts` / `handle_blocks_done` -> `retry`) bypasses + // that filter and re-queues buffered heights via `push_front`. + // A run buffered above an open gap was otherwise re-fetched + // indefinitely (in production a single height was re-requested + // thousands of times), pinning the queue front and every peer + // slot so the gap below the run never got a request and the + // download floor never advanced. `retry`, `ensure`, and + // `prune_covered` all skip covered heights, so this stops the + // churn and lets the gap be scheduled. Covered is cleared if the + // block is later rolled back (apply `Rejected`/`TimedOut` -> + // `clear_covered_from`) or the chain tip resets, both of which + // also drop the buffer, so a dropped body becomes requestable + // again. + self.state.schedule.mark_height_covered(height); + } + ReorderInsertResult::Duplicate => {} + ReorderInsertResult::BudgetFull => { + if let Some(request) = retry_request { + self.state.schedule.retry(request); + } + tracing::debug!( + ?peer, + ?height, + serialized_bytes, + "dropping block-sync body because local byte budget is full" + ); + self.schedule().await; + return; + } + } + self.release_contiguous_blocks().await; + self.schedule().await; + self.release_caught_up_block_sync_peers(); + } + + async fn handle_get_blocks( + &mut self, + peer: ZakuraPeerId, + start_height: block::Height, + count: u32, + ) { + let local_inflight_cap = self.startup.config.advertised_max_inflight_requests(); + let Some(peer_state) = self.state.peers.get_mut(&peer) else { + self.report_misbehavior(peer, BlockSyncMisbehavior::GetBlocksSpam) + .await; + return; + }; + + if !peer_state.received_status { + self.report_misbehavior(peer, BlockSyncMisbehavior::GetBlocksSpam) + .await; + return; + } + + if count == 0 { + self.report_misbehavior(peer, BlockSyncMisbehavior::GetBlocksTooLong) + .await; + return; + } + + if !peer_state.try_start_serving_blocks(local_inflight_cap) { + let unavailable_count = count.min(inbound_get_blocks_count_limit(&self.startup.config)); + self.send_range_unavailable(&peer, start_height, unavailable_count); + return; + } + + let requested_count = self.clamp_served_block_count(start_height, count); + if requested_count == 0 { + let unavailable_count = count.min(inbound_get_blocks_count_limit(&self.startup.config)); + self.send_range_unavailable(&peer, start_height, unavailable_count); + self.finish_serving_blocks(&peer); + return; + } + + if !self + .dispatch_action(BlockSyncAction::QueryBlocksByHeightRange { + peer: peer.clone(), + start: start_height, + count: requested_count, + }) + .await + { + self.finish_serving_blocks(&peer); + } + } + + fn finish_peer_outstanding_at( + &mut self, + peer: &ZakuraPeerId, + index: usize, + disposition: OutstandingRangeDisposition, + ) { + let Some(peer_state) = self.state.peers.get_mut(peer) else { + return; + }; + if index >= peer_state.outstanding.len() { + return; + } + + let outstanding = peer_state.outstanding.remove(index); + self.finish_detached_outstanding(outstanding, disposition); + } + + fn finish_detached_outstanding( + &mut self, + outstanding: OutstandingBlockRange, + disposition: OutstandingRangeDisposition, + ) { + self.state.budget.release(outstanding.reserved_bytes()); + match disposition { + OutstandingRangeDisposition::Satisfied => { + self.state.schedule.clear_assignment(&outstanding.request); + } + OutstandingRangeDisposition::RetryOriginal => { + self.state.schedule.retry(outstanding.request); + } + OutstandingRangeDisposition::RetryMissing => { + self.state.schedule.clear_assignment(&outstanding.request); + for request in outstanding.missing_retry_requests().into_iter().rev() { + self.state.schedule.retry(request); + } + } + } + } + + async fn handle_range_unavailable( + &mut self, + peer: ZakuraPeerId, + start_height: block::Height, + _count: u32, + ) { + let Some(index) = self.outstanding_index_for_start(&peer, start_height) else { + if self.ignore_disconnected_peer_response(&peer, "unavailable range") { + return; + } + if self + .ignore_stale_response(&peer, start_height, "unavailable range") + .await + { + return; + } + + self.trace_range_unavailable(&peer, start_height); + return; + }; + + self.trace_range_unavailable(&peer, start_height); + self.finish_peer_outstanding_at(&peer, index, OutstandingRangeDisposition::RetryMissing); + self.schedule().await; + self.release_caught_up_block_sync_peers(); + } + + fn outstanding_index_for_start( + &self, + peer: &ZakuraPeerId, + start_height: block::Height, + ) -> Option { + self.state + .peers + .get(peer)? + .outstanding_index_for_start(start_height) + } + + fn is_stale_response_height(&self, height: block::Height) -> bool { + height <= self.state.body_download_floor + || self.state.reorder.contains(height) + || self.state.applying.contains_key(&height) + } + + async fn ignore_stale_response( + &mut self, + peer: &ZakuraPeerId, + height: block::Height, + response_kind: &'static str, + ) -> bool { + if !self.is_stale_response_height(height) { + return false; + } + + tracing::debug!( + ?peer, + ?height, + response_kind, + "ignoring stale block-sync response" + ); + self.release_contiguous_blocks().await; + self.schedule().await; + self.release_caught_up_block_sync_peers(); + true + } + + async fn ignore_unmatched_needed_response( + &mut self, + peer: &ZakuraPeerId, + height: block::Height, + response_kind: &'static str, + ) -> bool { + if self.state.needed_heights.binary_search(&height).is_err() { + return false; + } + + metrics::counter!("sync.block.response.unmatched_needed_ignored").increment(1); + tracing::debug!( + ?peer, + ?height, + response_kind, + "ignoring unmatched block-sync response for currently needed height" + ); + self.release_contiguous_blocks().await; + self.schedule().await; + self.release_caught_up_block_sync_peers(); + true + } + + fn ignore_disconnected_peer_response( + &self, + peer: &ZakuraPeerId, + response_kind: &'static str, + ) -> bool { + if !self.state.disconnected_peers.contains(peer) { + return false; + } + + metrics::counter!("sync.block.response.disconnected_peer_ignored").increment(1); + tracing::debug!( + ?peer, + response_kind, + "ignoring late block-sync response from disconnected peer" + ); + true + } + + async fn handle_blocks_done(&mut self, peer: ZakuraPeerId, start_height: block::Height) { + if !self.state.peers.contains_key(&peer) { + if self.ignore_disconnected_peer_response(&peer, "terminator") { + return; + } + if self + .ignore_stale_response(&peer, start_height, "terminator from inactive peer") + .await + { + return; + } + + self.report_misbehavior(peer, BlockSyncMisbehavior::UnsolicitedDone) + .await; + return; + } + + let Some(index) = self.outstanding_index_for_start(&peer, start_height) else { + if self + .ignore_stale_response(&peer, start_height, "terminator") + .await + { + return; + } + if self + .ignore_unmatched_needed_response(&peer, start_height, "terminator") + .await + { + return; + } + // A known, active peer sent a response terminator that correlates to no + // outstanding range. Fail closed: report `UnsolicitedDone` (a hard + // block-sync misbehavior) instead of silently rescheduling. + self.report_misbehavior(peer, BlockSyncMisbehavior::UnsolicitedDone) + .await; + return; + }; + self.finish_peer_outstanding_at(&peer, index, OutstandingRangeDisposition::RetryMissing); + self.schedule().await; + self.release_caught_up_block_sync_peers(); + } + + async fn handle_block_range_response_ready( + &mut self, + peer: ZakuraPeerId, + start_height: block::Height, + requested_count: u32, + blocks: Vec<(block::Height, Arc, usize)>, + ) { + let max_response_bytes = u64::from(self.startup.config.advertised_max_response_bytes()); + let mut sent_blocks = 0u32; + let mut sent_bytes = 0u64; + + for (height, block, size) in blocks { + let Ok(size) = u64::try_from(size) else { + break; + }; + let Some(next_bytes) = sent_bytes.checked_add(size) else { + break; + }; + if next_bytes > max_response_bytes { + break; + } + if height_after_count(start_height, sent_blocks) != Some(height) { + break; + } + + if !self.send_block(&peer, block) { + break; + } + sent_blocks = sent_blocks.saturating_add(1); + sent_bytes = next_bytes; + } + + if sent_blocks == 0 { + self.send_range_unavailable(&peer, start_height, requested_count); + } else { + self.send_blocks_done(&peer, start_height, sent_blocks); + } + self.finish_serving_blocks(&peer); + } + + async fn handle_block_range_response_finished( + &mut self, + peer: ZakuraPeerId, + start_height: block::Height, + requested_count: u32, + returned_count: u32, + ) { + if returned_count == 0 { + self.send_range_unavailable(&peer, start_height, requested_count); + } + self.finish_serving_blocks(&peer); + } + + async fn handle_block_apply_finished( + &mut self, + token: BlockApplyToken, + height: block::Height, + hash: block::Hash, + result: BlockApplyResult, + local_frontier: Option, + ) { + let (accepted_local_frontier, old_serving_tip) = if let Some(frontiers) = local_frontier { + let old_serving_tip = self.apply_state_frontiers_changed(frontiers, false).await; + (old_serving_tip.map(|_| frontiers), old_serving_tip) + } else { + (None, None) + }; + + let Some(applying) = self.state.applying.get(&height) else { + if let Some(frontiers) = accepted_local_frontier { + self.release_applied_blocks_through(frontiers.verified_block_tip); + } + if let Some(old_serving_tip) = old_serving_tip { + self.finish_frontier_update(old_serving_tip).await; + } + return; + }; + if applying.hash != hash || applying.token != token { + if let Some(frontiers) = accepted_local_frontier { + self.release_applied_blocks_through(frontiers.verified_block_tip); + } + if let Some(old_serving_tip) = old_serving_tip { + self.finish_frontier_update(old_serving_tip).await; + } + return; + } + let applying = self + .state + .applying + .remove(&height) + .expect("applying entry exists because it was just checked"); + + self.state.budget.release(applying.bytes); + self.trace_apply_finished(height, token, result); + match result { + BlockApplyResult::Committed | BlockApplyResult::Duplicate => {} + BlockApplyResult::Rejected | BlockApplyResult::TimedOut + if height > self.state.verified_block_tip => + { + self.release_applying_blocks_from(height); + self.state.body_download_floor = previous_height(height) + .unwrap_or(block::Height::MIN) + .max(self.state.verified_block_tip); + self.state.schedule.clear_covered_from(height); + self.state.reorder.drop_from(height, &mut self.state.budget); + } + BlockApplyResult::Rejected | BlockApplyResult::TimedOut => {} + } + if let Some(frontiers) = accepted_local_frontier { + self.release_applied_blocks_through(frontiers.verified_block_tip); + } + + self.release_contiguous_blocks().await; + if let Some(old_serving_tip) = old_serving_tip { + self.queue_status_refresh_if_changed(old_serving_tip); + self.flush_status_refresh().await; + } + if !self.query_needed_blocks().await { + self.pause_new_body_downloads(); + } + self.schedule().await; + self.release_caught_up_block_sync_peers(); + } + + fn finish_serving_blocks(&mut self, peer: &ZakuraPeerId) { + if let Some(peer_state) = self.state.peers.get_mut(peer) { + peer_state.finish_serving_blocks(); + } + } + + async fn handle_timeouts(&mut self) { + let now = Instant::now(); + let mut timed_out = Vec::new(); + for peer in self.state.peers.values_mut() { + let mut index = 0; + while index < peer.outstanding.len() { + if peer.outstanding[index].deadline <= now { + timed_out.push(peer.outstanding.remove(index)); + } else { + index += 1; + } + } + } + + for outstanding in timed_out { + self.finish_detached_outstanding( + outstanding, + OutstandingRangeDisposition::RetryOriginal, + ); + } + self.schedule().await; + self.release_caught_up_block_sync_peers(); + } + + async fn query_needed_blocks(&mut self) -> bool { + if !self.startup.state_queries_enabled || self.should_pause_new_body_downloads() { + return false; + } + let _ = self + .dispatch_action(BlockSyncAction::QueryNeededBlocks { + verified_block_tip: self.state.body_download_floor, + best_header_tip: self.state.best_header_tip, + }) + .await; + true + } + + fn drop_ranges_not_in_needed(&mut self, needed: &HashMap) { + let mut dropped = Vec::new(); + for peer in self.state.peers.values_mut() { + let mut index = 0; + while index < peer.outstanding.len() { + if peer.outstanding[index].request.matches_needed(needed) { + index += 1; + } else { + dropped.push(peer.outstanding.remove(index)); + } + } + } + + for outstanding in dropped { + self.finish_detached_outstanding(outstanding, OutstandingRangeDisposition::Satisfied); + } + } + + fn needed_hashes_with_active_outstanding( + &self, + needed: &HashMap, + ) -> HashMap { + let mut retained = needed.clone(); + for peer in self.state.peers.values() { + for outstanding in &peer.outstanding { + for (height, hash) in &outstanding.request.expected_hashes { + retained.entry(*height).or_insert(*hash); + } + } + } + retained + } + + fn drop_outstanding_through(&mut self, tip: block::Height) { + let mut completed = Vec::new(); + let mut released_bytes = 0; + for peer in self.state.peers.values_mut() { + let mut index = 0; + while index < peer.outstanding.len() { + if peer.outstanding[index].request.start_height <= tip { + released_bytes += peer.outstanding[index].mark_received_through(tip); + if peer.outstanding[index].is_complete() { + completed.push(peer.outstanding.remove(index)); + } else { + index += 1; + } + } else { + index += 1; + } + } + } + + self.state.budget.release(released_bytes); + for outstanding in completed { + self.finish_detached_outstanding(outstanding, OutstandingRangeDisposition::Satisfied); + } + } + + async fn schedule(&mut self) { + self.submit_pending_blocks().await; + + if self.should_pause_new_body_downloads() { + let reason = if self.body_lag() == 0 { + "lag_zero" + } else if self.body_lag() <= self.startup.config.near_tip_body_download_pause_blocks { + "near_tip" + } else { + "budget_full" + }; + self.trace_downloads_paused(reason); + return; + } + + let mut peer_ids: Vec<_> = self.state.peers.keys().cloned().collect(); + peer_ids.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes())); + + let per_peer_byte_cap = self.startup.config.per_peer_byte_cap(); + + for peer_id in peer_ids { + // Fill this peer's available slots in one pass, letting the byte + // budget (re-checked each iteration) be the congestion window. A + // raised slot cap is only useful if we can open the window promptly + // rather than one slot per scheduling event. + loop { + if self.should_pause_new_body_downloads() { + return; + } + let Some(peer) = self.state.peers.get(&peer_id) else { + break; + }; + if !peer.received_status || peer.available_slots() == 0 { + break; + } + let Some(request) = self.state.schedule.next_for_peer( + &peer_id, + peer, + &mut self.state.budget, + per_peer_byte_cap, + ) else { + break; + }; + + let Some(peer) = self.state.peers.get(&peer_id) else { + break; + }; + if let Err(error) = peer + .session + .try_send_get_blocks(request.start_height, request.count) + { + tracing::debug!( + peer = ?peer_id, + start_height = ?request.start_height, + count = request.count, + ?error, + "failed to queue Zakura block-sync GetBlocks" + ); + self.state.budget.release(request.estimated_bytes); + self.state.schedule.retry(request); + break; + } + + metrics::counter!("sync.block.request.sent").increment(1); + self.trace_get_blocks_sent( + &peer_id, + request.start_height, + request.count, + request.estimated_bytes, + ); + let deadline = Instant::now() + self.startup.config.request_timeout; + if let Some(peer) = self.state.peers.get_mut(&peer_id) { + peer.outstanding.push(OutstandingBlockRange { + request: request.clone(), + deadline, + received: HashSet::new(), + }); + } + let _ = self + .dispatch_action(BlockSyncAction::SendMessage { + peer: peer_id.clone(), + msg: BlockSyncMessage::GetBlocks { + start_height: request.start_height, + count: request.count, + }, + }) + .await; + } + } + } + + async fn release_contiguous_blocks(&mut self) { + let released = self + .state + .reorder + .drain_contiguous_prefix(self.state.body_download_floor); + for (height, block, bytes) in released { + let hash = block.hash(); + self.state.body_download_floor = height; + self.state.schedule.mark_height_covered(height); + self.state.applying.insert( + height, + ApplyingBlock { + token: 0, + hash, + block, + bytes, + submitted: false, + }, + ); + } + + self.submit_pending_blocks().await; + } + + async fn submit_pending_blocks(&mut self) { + let pending: Vec<_> = self + .state + .applying + .iter() + .filter_map(|(height, applying)| (!applying.submitted).then_some(*height)) + .collect(); + + for height in pending { + let Some(block) = self + .state + .applying + .get(&height) + .map(|applying| applying.block.clone()) + else { + continue; + }; + + let token = self.next_apply_token(); + if let Some(applying) = self.state.applying.get_mut(&height) { + applying.token = token; + applying.submitted = true; + } + + metrics::counter!("sync.block.submit.sent").increment(1); + if !self + .dispatch_action(BlockSyncAction::SubmitBlock { token, block }) + .await + { + if let Some(applying) = self.state.applying.get_mut(&height) { + if applying.token == token { + applying.token = 0; + applying.submitted = false; + } + } + return; + } + self.trace_body_submitted(height, token); + } + } + + fn next_apply_token(&mut self) -> BlockApplyToken { + let token = self.state.next_apply_token; + self.state.next_apply_token = self.state.next_apply_token.checked_add(1).unwrap_or(1); + token + } + + fn release_applied_blocks_through(&mut self, tip: block::Height) { + let applied: Vec<_> = self + .state + .applying + .range(..=tip) + .map(|(height, _)| *height) + .collect(); + for height in applied { + if let Some(applying) = self.state.applying.remove(&height) { + self.state.budget.release(applying.bytes); + } + } + } + + fn release_all_applying_blocks(&mut self) { + let bytes = self + .state + .applying + .values() + .map(|applying| applying.bytes) + .sum(); + self.state.budget.release(bytes); + self.state.applying.clear(); + } + + fn release_applying_blocks_from(&mut self, from: block::Height) { + let heights: Vec<_> = self + .state + .applying + .range(from..) + .map(|(height, _)| *height) + .collect(); + for height in heights { + if let Some(applying) = self.state.applying.remove(&height) { + self.state.budget.release(applying.bytes); + } + } + } + + async fn send_status(&self, peer: &ZakuraPeerId) { + let Some(peer_state) = self.state.peers.get(peer) else { + return; + }; + let status = self.local_status(); + let _ = peer_state.session.try_send_status(status); + let _ = self + .dispatch_action(BlockSyncAction::SendMessage { + peer: peer.clone(), + msg: BlockSyncMessage::Status(status), + }) + .await; + } + + fn send_block(&self, peer: &ZakuraPeerId, block: Arc) -> bool { + let Some(peer_state) = self.state.peers.get(peer) else { + return false; + }; + if let Err(error) = peer_state.session.try_send_block(block) { + tracing::debug!(?peer, ?error, "failed to queue Zakura block-sync Block"); + return false; + } + metrics::counter!("sync.block.body.served").increment(1); + true + } + + fn send_blocks_done(&self, peer: &ZakuraPeerId, start_height: block::Height, returned: u32) { + if returned == 0 { + return; + } + let Some(peer_state) = self.state.peers.get(peer) else { + return; + }; + if let Err(error) = peer_state + .session + .try_send_blocks_done(start_height, returned) + { + tracing::debug!( + ?peer, + ?error, + "failed to queue Zakura block-sync BlocksDone" + ); + } + } + + fn send_range_unavailable(&self, peer: &ZakuraPeerId, start_height: block::Height, count: u32) { + let count = count.max(1); + let Some(peer_state) = self.state.peers.get(peer) else { + return; + }; + if let Err(error) = peer_state + .session + .try_send_range_unavailable(start_height, count) + { + tracing::debug!( + ?peer, + ?error, + "failed to queue Zakura block-sync RangeUnavailable" + ); + } + } + + async fn flush_status_refresh(&mut self) { + if !self.state.pending_status_refresh { + return; + } + let now = Instant::now(); + if !self.state.status_refresh.try_take(now) { + return; + } + let status = self.local_status(); + if status == self.state.last_advertised_status { + self.state.pending_status_refresh = false; + return; + } + + self.state.pending_status_refresh = false; + self.state.last_advertised_status = status; + let _ = self.status.send(status); + + let peer_ids: Vec<_> = self + .state + .peers + .iter_mut() + .filter_map(|(peer_id, peer)| peer.unsolicited.try_take(now).then(|| peer_id.clone())) + .collect(); + + for peer in peer_ids { + self.send_status(&peer).await; + } + } + + fn queue_status_refresh_if_changed(&mut self, old_serving_tip: (block::Height, block::Hash)) { + if old_serving_tip != (self.state.servable_high, self.state.servable_hash) + && self.local_status() != self.state.last_advertised_status + { + self.state.pending_status_refresh = true; + } + } + + fn emit_trace( + &self, + event: &'static str, + build: impl FnOnce(&mut serde_json::Map), + ) { + self.startup.trace.emit_with(BLOCK_SYNC_TABLE, |row| { + row.insert( + bs_trace::EVENT.to_string(), + serde_json::Value::String(event.to_string()), + ); + build(row); + }); + } + + /// Emit the periodic reactor snapshot used to diagnose body-sync stalls. + /// + /// This is the highest-signal row: a stall shows up as `body_download_floor` + /// and `verified_block_tip` frozen while `best_header_tip` climbs, plus + /// whichever resource is pinned (`budget_available == 0`, `applying`/`reorder` + /// growing, or `peers_with_status == 0`). + fn trace_sync_state(&self) { + if !self.startup.trace.is_enabled() { + return; + } + let outstanding: usize = self + .state + .peers + .values() + .map(|peer| peer.outstanding.len()) + .sum(); + let peers_with_status = self + .state + .peers + .values() + .filter(|peer| peer.received_status) + .count(); + self.emit_trace(bs_trace::BLOCK_SYNC_STATE, |row| { + bs_insert_height( + row, + bs_trace::BODY_DOWNLOAD_FLOOR, + self.state.body_download_floor, + ); + bs_insert_height( + row, + bs_trace::VERIFIED_BLOCK_TIP, + self.state.verified_block_tip, + ); + bs_insert_height(row, bs_trace::BEST_HEADER_TIP, self.state.best_header_tip); + bs_insert_u64(row, bs_trace::BODY_LAG, u64::from(self.body_lag())); + bs_insert_u64(row, bs_trace::APPLYING, self.state.applying.len() as u64); + bs_insert_u64(row, bs_trace::REORDER, self.state.reorder.len() as u64); + bs_insert_u64(row, bs_trace::OUTSTANDING, outstanding as u64); + bs_insert_u64( + row, + bs_trace::BUDGET_AVAILABLE, + self.state.budget.available(), + ); + bs_insert_u64(row, bs_trace::BUDGET_RESERVED, self.state.budget.reserved()); + bs_insert_u64(row, bs_trace::PEERS, self.state.peers.len() as u64); + bs_insert_u64(row, bs_trace::PEERS_WITH_STATUS, peers_with_status as u64); + // Scheduling visibility: distinguishes "gap not in `needed`" + // (state/filter) from "gap in `needed` but never queued" (`ensure` + // rejected it) from "queued but never requested" (starvation). + if let Some(min) = self.state.needed_heights.first() { + bs_insert_height(row, bs_trace::NEEDED_MIN, *min); + } + bs_insert_u64( + row, + bs_trace::NEEDED_COUNT, + self.state.needed_heights.len() as u64, + ); + bs_insert_u64( + row, + bs_trace::QUEUE_LEN, + self.state.schedule.queued_range_count() as u64, + ); + if let Some(start) = self.state.schedule.queued_min_start() { + bs_insert_height(row, bs_trace::QUEUE_MIN_START, start); + } + bs_insert_u64( + row, + bs_trace::ASSIGNED_LEN, + self.state.schedule.assigned_key_count() as u64, + ); + if let Some(end) = self.state.schedule.covered_max_end() { + bs_insert_height(row, bs_trace::COVERED_MAX_END, end); + } + }); + } + + fn trace_status_received(&self, peer: &ZakuraPeerId, status: BlockSyncStatus) { + self.emit_trace(bs_trace::BLOCK_STATUS_RECEIVED, |row| { + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_height(row, bs_trace::RANGE_START, status.servable_low); + bs_insert_height(row, bs_trace::HEIGHT, status.servable_high); + }); + } + + fn trace_get_blocks_sent( + &self, + peer: &ZakuraPeerId, + start_height: block::Height, + count: u32, + estimated_bytes: u64, + ) { + self.emit_trace(bs_trace::BLOCK_GET_BLOCKS_SENT, |row| { + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_height(row, bs_trace::RANGE_START, start_height); + bs_insert_u64(row, bs_trace::RANGE_COUNT, u64::from(count)); + bs_insert_u64(row, bs_trace::ESTIMATED_BYTES, estimated_bytes); + }); + } + + fn trace_body_received( + &self, + peer: &ZakuraPeerId, + height: block::Height, + serialized_bytes: u64, + ) { + self.emit_trace(bs_trace::BLOCK_BODY_RECEIVED, |row| { + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_height(row, bs_trace::HEIGHT, height); + bs_insert_u64(row, bs_trace::SERIALIZED_BYTES, serialized_bytes); + }); + } + + fn trace_body_submitted(&self, height: block::Height, token: BlockApplyToken) { + self.emit_trace(bs_trace::BLOCK_BODY_SUBMITTED, |row| { + bs_insert_height(row, bs_trace::HEIGHT, height); + bs_insert_u64(row, bs_trace::APPLY_TOKEN, token); + }); + } + + fn trace_apply_finished( + &self, + height: block::Height, + token: BlockApplyToken, + result: BlockApplyResult, + ) { + self.emit_trace(bs_trace::BLOCK_APPLY_FINISHED, |row| { + bs_insert_height(row, bs_trace::HEIGHT, height); + bs_insert_u64(row, bs_trace::APPLY_TOKEN, token); + bs_insert_str(row, bs_trace::RESULT, block_apply_result_label(result)); + }); + } + + fn trace_range_unavailable(&self, peer: &ZakuraPeerId, start_height: block::Height) { + self.emit_trace(bs_trace::BLOCK_RANGE_UNAVAILABLE, |row| { + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_height(row, bs_trace::RANGE_START, start_height); + }); + } + + fn trace_downloads_paused(&self, reason: &'static str) { + self.emit_trace(bs_trace::BLOCK_DOWNLOADS_PAUSED, |row| { + bs_insert_str(row, bs_trace::REASON, reason); + bs_insert_u64(row, bs_trace::BODY_LAG, u64::from(self.body_lag())); + bs_insert_u64( + row, + bs_trace::BUDGET_AVAILABLE, + self.state.budget.available(), + ); + }); + } + + fn trace_frontiers_changed(&self, verified_block_tip: block::Height) { + self.emit_trace(bs_trace::BLOCK_FRONTIERS_CHANGED, |row| { + bs_insert_height(row, bs_trace::VERIFIED_BLOCK_TIP, verified_block_tip); + bs_insert_height(row, bs_trace::BEST_HEADER_TIP, self.state.best_header_tip); + }); + } + + fn trace_chain_tip_reset(&self, verified_block_tip: block::Height) { + self.emit_trace(bs_trace::BLOCK_CHAIN_TIP_RESET, |row| { + bs_insert_height(row, bs_trace::VERIFIED_BLOCK_TIP, verified_block_tip); + }); + } + + fn publish_metrics(&self) { + // These lossy casts are metrics-only gauges; consensus and scheduling + // continue to use the original integer values. + metrics::gauge!("sync.block.best_header_tip.height") + .set(self.state.best_header_tip.0 as f64); + metrics::gauge!("sync.block.verified_tip.height") + .set(self.state.verified_block_tip.0 as f64); + metrics::gauge!("sync.block.missing_bodies").set(self.state.needed_heights.len() as f64); + metrics::gauge!("sync.block.budget.reserved_bytes") + .set(self.state.budget.reserved() as f64); + metrics::gauge!("sync.block.reorder.buffered_bytes") + .set(self.state.reorder.buffered_bytes() as f64); + metrics::gauge!("sync.block.applying").set(self.state.applying.len() as f64); + metrics::gauge!("sync.block.outstanding").set( + self.state + .peers + .values() + .map(|peer| peer.outstanding.len()) + .sum::() as f64, + ); + } + + fn clamp_served_block_count(&self, start_height: block::Height, count: u32) -> u32 { + if start_height > self.state.servable_high { + return 0; + } + + let available = self + .state + .servable_high + .0 + .checked_sub(start_height.0) + .and_then(|diff| diff.checked_add(1)) + .unwrap_or(0); + + count + .min(inbound_get_blocks_count_limit(&self.startup.config)) + .min(available) + } + + fn local_status(&self) -> BlockSyncStatus { + BlockSyncStatus { + servable_low: block::Height::MIN, + servable_high: self.state.servable_high, + tip_hash: self.state.servable_hash, + max_blocks_per_response: self.startup.config.advertised_max_blocks_per_response(), + max_inflight_requests: self.startup.config.advertised_max_inflight_requests(), + max_response_bytes: self.startup.config.advertised_max_response_bytes(), + } + } + + /// Hand a data-plane action to the action driver without letting a slow or + /// stalled driver wedge the reactor. A full channel is awaited only up to + /// [`ACTION_SEND_TIMEOUT`]; past that the action is dropped so the reactor + /// keeps draining peer-lifecycle events, request timeouts, and misbehavior + /// disconnects. Returns `true` only if the action was accepted. + async fn dispatch_action(&self, action: BlockSyncAction) -> bool { + self.trace_action_dispatched(&action); + match time::timeout(ACTION_SEND_TIMEOUT, self.actions.send(action)).await { + Ok(Ok(())) => true, + // Receiver dropped: the driver is gone, treat like a send failure. + Ok(Err(_)) => false, + // Driver stalled past the deadline: drop the action and stay live. + Err(_) => { + metrics::counter!("sync.block.action.send_timeout").increment(1); + false + } + } + } + + async fn report_misbehavior(&mut self, peer: ZakuraPeerId, reason: BlockSyncMisbehavior) { + let mut cancel_peer = None; + if let Some(peer_state) = self.state.peers.get_mut(&peer) { + peer_state.misbehavior = peer_state.misbehavior.saturating_add(1); + if block_sync_misbehavior_is_soft(reason) + && peer_state.misbehavior >= SOFT_MISBEHAVIOR_DISCONNECT_THRESHOLD + { + cancel_peer = Some(peer_state.session.cancel_token()); + } + } + if let Some(cancel_token) = cancel_peer { + cancel_token.cancel(); + } + // The Misbehavior action carries the hard-disconnect/scoring request to + // the supervisor. Deliver it without ever blocking the reactor: awaiting + // a full `actions` channel here was the backpressure stall that delayed + // misbehavior disconnects, request timeouts, and lifecycle draining + // whenever the action driver was slow. `try_send` keeps the reactor live + // so it can promptly tear down soft offenders at threshold and deliver + // the next disconnect as soon as the driver drains a slot. + let action = BlockSyncAction::Misbehavior { peer, reason }; + self.trace_action_dispatched(&action); + if self.actions.try_send(action).is_err() { + metrics::counter!("sync.block.peer.disconnect.action_dropped").increment(1); + } + } + + fn trace_event_received(&self, event: &BlockSyncEvent) { + self.emit_trace(bs_trace::BLOCK_EVENT_RECEIVED, |row| match event { + BlockSyncEvent::PeerConnected(session) => { + bs_insert_str(row, bs_trace::KIND, "peer_connected"); + bs_insert_peer(row, bs_trace::PEER, session.peer_id()); + } + BlockSyncEvent::PeerDisconnected(peer) => { + bs_insert_str(row, bs_trace::KIND, "peer_disconnected"); + bs_insert_peer(row, bs_trace::PEER, peer); + } + BlockSyncEvent::WireMessage { peer, msg } => { + bs_insert_str(row, bs_trace::KIND, "wire_message"); + bs_insert_str(row, bs_trace::REASON, block_sync_message_label(msg)); + bs_insert_peer(row, bs_trace::PEER, peer); + trace_block_sync_message_fields(row, msg); + } + BlockSyncEvent::WireDecodeFailed { peer, .. } => { + bs_insert_str(row, bs_trace::KIND, "wire_decode_failed"); + bs_insert_peer(row, bs_trace::PEER, peer); + } + BlockSyncEvent::HeaderTipChanged { height, hash } => { + bs_insert_str(row, bs_trace::KIND, "header_tip_changed"); + bs_insert_height(row, bs_trace::HEIGHT, *height); + bs_insert_hash(row, bs_trace::HASH, *hash); + } + BlockSyncEvent::StateFrontiersChanged(frontiers) => { + bs_insert_str(row, bs_trace::KIND, "state_frontiers_changed"); + bs_insert_frontiers(row, frontiers); + } + BlockSyncEvent::ChainTipGrow(frontiers) => { + bs_insert_str(row, bs_trace::KIND, "chain_tip_grow"); + bs_insert_frontiers(row, frontiers); + } + BlockSyncEvent::ChainTipReset(frontiers) => { + bs_insert_str(row, bs_trace::KIND, "chain_tip_reset"); + bs_insert_frontiers(row, frontiers); + } + BlockSyncEvent::NeededBlocks(blocks) => { + bs_insert_str(row, bs_trace::KIND, "needed_blocks"); + bs_insert_u64(row, bs_trace::RANGE_COUNT, blocks.len() as u64); + if let Some(first) = blocks.first() { + bs_insert_height(row, bs_trace::RANGE_START, first.height); + } + } + BlockSyncEvent::BlockApplyFinished { + token, + height, + hash, + result, + local_frontier, + } => { + bs_insert_str(row, bs_trace::KIND, "block_apply_finished"); + bs_insert_u64(row, bs_trace::APPLY_TOKEN, *token); + bs_insert_height(row, bs_trace::HEIGHT, *height); + bs_insert_hash(row, bs_trace::HASH, *hash); + bs_insert_str(row, bs_trace::RESULT, block_apply_result_label(*result)); + if let Some(frontiers) = local_frontier { + bs_insert_frontiers(row, frontiers); + } + } + BlockSyncEvent::BlockRangeResponseFinished { + peer, + start_height, + requested_count, + returned_count, + } => { + bs_insert_str(row, bs_trace::KIND, "block_range_response_finished"); + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_height(row, bs_trace::RANGE_START, *start_height); + bs_insert_u64(row, bs_trace::RANGE_COUNT, u64::from(*returned_count)); + bs_insert_u64(row, bs_trace::EXPECTED_COUNT, u64::from(*requested_count)); + } + BlockSyncEvent::BlockRangeResponseReady { + peer, + start_height, + requested_count, + blocks, + } => { + bs_insert_str(row, bs_trace::KIND, "block_range_response_ready"); + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_height(row, bs_trace::RANGE_START, *start_height); + bs_insert_u64(row, bs_trace::RANGE_COUNT, blocks.len() as u64); + bs_insert_u64(row, bs_trace::EXPECTED_COUNT, u64::from(*requested_count)); + } + }); + } + + fn trace_action_dispatched(&self, action: &BlockSyncAction) { + self.emit_trace(bs_trace::BLOCK_ACTION_DISPATCHED, |row| match action { + BlockSyncAction::SendMessage { peer, msg } => { + bs_insert_str(row, bs_trace::KIND, "send_message"); + bs_insert_str(row, bs_trace::REASON, block_sync_message_label(msg)); + bs_insert_peer(row, bs_trace::PEER, peer); + trace_block_sync_message_fields(row, msg); + } + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + bs_insert_str(row, bs_trace::KIND, "query_needed_blocks"); + bs_insert_height(row, bs_trace::VERIFIED_BLOCK_TIP, *verified_block_tip); + bs_insert_height(row, bs_trace::BEST_HEADER_TIP, *best_header_tip); + } + BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => { + bs_insert_str(row, bs_trace::KIND, "query_blocks_by_height_range"); + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_height(row, bs_trace::RANGE_START, *start); + bs_insert_u64(row, bs_trace::RANGE_COUNT, u64::from(*count)); + } + BlockSyncAction::SubmitBlock { token, block } => { + bs_insert_str(row, bs_trace::KIND, "submit_block"); + bs_insert_u64(row, bs_trace::APPLY_TOKEN, *token); + bs_insert_hash(row, bs_trace::HASH, block.hash()); + if let Some(height) = block.coinbase_height() { + bs_insert_height(row, bs_trace::HEIGHT, height); + } + } + BlockSyncAction::Misbehavior { peer, reason } => { + bs_insert_str(row, bs_trace::KIND, "misbehavior"); + bs_insert_peer(row, bs_trace::PEER, peer); + bs_insert_str(row, bs_trace::REASON, block_misbehavior_label(*reason)); + } + }); + } +} + +pub(super) fn node_id_from_block_peer_id(peer_id: &ZakuraPeerId) -> Option { + let bytes: [u8; 32] = peer_id.as_bytes().try_into().ok()?; + NodeId::from_bytes(&bytes).ok() +} + +fn block_apply_result_label(result: BlockApplyResult) -> &'static str { + match result { + BlockApplyResult::Committed => "committed", + BlockApplyResult::Duplicate => "duplicate", + BlockApplyResult::Rejected => "rejected", + BlockApplyResult::TimedOut => "timed_out", + } +} + +fn bs_insert_peer( + row: &mut serde_json::Map, + key: &'static str, + peer: &ZakuraPeerId, +) { + row.insert( + key.to_string(), + serde_json::Value::String(trace_peer_label(peer)), + ); +} + +fn bs_insert_height( + row: &mut serde_json::Map, + key: &'static str, + height: block::Height, +) { + bs_insert_u64(row, key, u64::from(height.0)); +} + +fn bs_insert_hash( + row: &mut serde_json::Map, + key: &'static str, + hash: block::Hash, +) { + row.insert( + key.to_string(), + serde_json::Value::String(format!("{hash}")), + ); +} + +fn bs_insert_u64( + row: &mut serde_json::Map, + key: &'static str, + value: u64, +) { + row.insert(key.to_string(), serde_json::Value::from(value)); +} + +fn bs_insert_frontiers( + row: &mut serde_json::Map, + frontiers: &BlockSyncFrontiers, +) { + bs_insert_height( + row, + bs_trace::VERIFIED_BLOCK_TIP, + frontiers.verified_block_tip, + ); + bs_insert_hash(row, bs_trace::HASH, frontiers.verified_block_hash); +} + +fn trace_block_sync_message_fields( + row: &mut serde_json::Map, + msg: &BlockSyncMessage, +) { + match msg { + BlockSyncMessage::Status(status) => { + bs_insert_height(row, bs_trace::RANGE_START, status.servable_low); + bs_insert_height(row, bs_trace::HEIGHT, status.servable_high); + } + BlockSyncMessage::Block(block) => { + bs_insert_hash(row, bs_trace::HASH, block.hash()); + if let Some(height) = block.coinbase_height() { + bs_insert_height(row, bs_trace::HEIGHT, height); + } + } + BlockSyncMessage::BlocksDone { + start_height, + returned, + } => { + bs_insert_height(row, bs_trace::RANGE_START, *start_height); + bs_insert_u64(row, bs_trace::RANGE_COUNT, u64::from(*returned)); + } + BlockSyncMessage::RangeUnavailable { + start_height, + count, + } + | BlockSyncMessage::GetBlocks { + start_height, + count, + } => { + bs_insert_height(row, bs_trace::RANGE_START, *start_height); + bs_insert_u64(row, bs_trace::RANGE_COUNT, u64::from(*count)); + } + } +} + +fn block_sync_message_label(msg: &BlockSyncMessage) -> &'static str { + match msg { + BlockSyncMessage::Status(_) => "status", + BlockSyncMessage::Block(_) => "block", + BlockSyncMessage::BlocksDone { .. } => "blocks_done", + BlockSyncMessage::RangeUnavailable { .. } => "range_unavailable", + BlockSyncMessage::GetBlocks { .. } => "get_blocks", + } +} + +fn block_misbehavior_label(reason: BlockSyncMisbehavior) -> &'static str { + match reason { + BlockSyncMisbehavior::MalformedMessage => "malformed_message", + BlockSyncMisbehavior::UnsolicitedBlock => "unsolicited_block", + BlockSyncMisbehavior::GetBlocksTooLong => "get_blocks_too_long", + BlockSyncMisbehavior::GetBlocksSpam => "get_blocks_spam", + BlockSyncMisbehavior::InvalidBlock => "invalid_block", + BlockSyncMisbehavior::SizeMismatch => "size_mismatch", + BlockSyncMisbehavior::InvalidStatus => "invalid_status", + BlockSyncMisbehavior::UnsolicitedDone => "unsolicited_done", + BlockSyncMisbehavior::RangeUnavailable => "range_unavailable", + BlockSyncMisbehavior::StatusSpam => "status_spam", + } +} + +fn bs_insert_str( + row: &mut serde_json::Map, + key: &'static str, + value: &str, +) { + row.insert(key.to_string(), serde_json::Value::from(value.to_string())); +} + +fn tolerated_bytes(reserved_bytes: u64, tolerance_percent: u32) -> u64 { + reserved_bytes.saturating_mul(u64::from(tolerance_percent.max(100))) / 100 +} + +fn block_sync_misbehavior_is_soft(reason: BlockSyncMisbehavior) -> bool { + matches!( + reason, + BlockSyncMisbehavior::SizeMismatch + | BlockSyncMisbehavior::RangeUnavailable + | BlockSyncMisbehavior::GetBlocksSpam + ) +} + +async fn block_merkle_root_matches_header(block: Arc) -> bool { + match task::spawn_blocking(move || { + block.transactions.iter().collect::() == block.header.merkle_root + }) + .await + { + Ok(matches) => matches, + Err(error) => { + tracing::debug!(?error, "block-sync merkle-root validation task failed"); + false + } + } +} diff --git a/zebra-network/src/zakura/block_sync/reorder.rs b/zebra-network/src/zakura/block_sync/reorder.rs new file mode 100644 index 00000000000..f4ec8cf2aed --- /dev/null +++ b/zebra-network/src/zakura/block_sync/reorder.rs @@ -0,0 +1,114 @@ +use super::{state::*, *}; + +#[derive(Clone, Debug)] +pub(crate) struct ReorderBuffer { + blocks: BTreeMap, + buffered_bytes: u64, +} + +impl ReorderBuffer { + pub(super) fn new() -> Self { + Self { + blocks: BTreeMap::new(), + buffered_bytes: 0, + } + } + + pub(super) fn buffered_bytes(&self) -> u64 { + self.buffered_bytes + } + + pub(super) fn len(&self) -> usize { + self.blocks.len() + } + + pub(super) fn contains(&self, height: block::Height) -> bool { + self.blocks.contains_key(&height) + } + + pub(super) fn insert( + &mut self, + height: block::Height, + block: Arc, + bytes: u64, + budget: &mut ByteBudget, + ) -> ReorderInsertResult { + if self.blocks.contains_key(&height) { + return ReorderInsertResult::Duplicate; + } + if !budget.try_reserve(bytes) { + return ReorderInsertResult::BudgetFull; + } + + self.blocks.insert(height, BufferedBlock { block, bytes }); + self.buffered_bytes = self.buffered_bytes.saturating_add(bytes); + ReorderInsertResult::Inserted + } + + pub(super) fn drain_contiguous_prefix( + &mut self, + verified_block_tip: block::Height, + ) -> Vec<(block::Height, Arc, u64)> { + let mut released = Vec::new(); + let mut next = match next_height(verified_block_tip) { + Some(next) => next, + None => return released, + }; + + while let Some(buffered) = self.blocks.remove(&next) { + self.buffered_bytes = self.buffered_bytes.saturating_sub(buffered.bytes); + released.push((next, buffered.block, buffered.bytes)); + let Some(after) = next_height(next) else { + break; + }; + next = after; + } + + released + } + + pub(crate) fn clear(&mut self, budget: &mut ByteBudget) { + self.drop_from(block::Height::MIN, budget); + } + + pub(crate) fn drop_through(&mut self, through: block::Height, budget: &mut ByteBudget) { + let heights: Vec<_> = self + .blocks + .range(..=through) + .map(|(height, _)| *height) + .collect(); + for height in heights { + if let Some(buffered) = self.blocks.remove(&height) { + self.buffered_bytes = self.buffered_bytes.saturating_sub(buffered.bytes); + budget.release(buffered.bytes); + } + } + } + + pub(crate) fn drop_from(&mut self, from: block::Height, budget: &mut ByteBudget) { + let heights: Vec<_> = self + .blocks + .range(from..) + .map(|(height, _)| *height) + .collect(); + for height in heights { + if let Some(buffered) = self.blocks.remove(&height) { + self.buffered_bytes = self.buffered_bytes.saturating_sub(buffered.bytes); + budget.release(buffered.bytes); + } + } + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(super) enum ReorderInsertResult { + Inserted, + Duplicate, + BudgetFull, +} + +#[derive(Clone, Debug)] +struct BufferedBlock { + block: Arc, + bytes: u64, +} diff --git a/zebra-network/src/zakura/block_sync/scheduler.rs b/zebra-network/src/zakura/block_sync/scheduler.rs new file mode 100644 index 00000000000..d9ffde2650f --- /dev/null +++ b/zebra-network/src/zakura/block_sync/scheduler.rs @@ -0,0 +1,530 @@ +use super::{state::*, *}; + +pub(super) const DEFAULT_BS_EWMA_SEED_BYTES: u64 = 256 * 1024; +pub(super) const DEFAULT_BS_SIZE_FLOOR_BYTES: u64 = 1024; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct NeededBlock { + pub(super) height: block::Height, + pub(super) hash: block::Hash, + pub(super) size: BlockSizeEstimate, +} + +/// Scheduling source for a block-body byte estimate. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum BlockSizeEstimate { + /// Confirmed serialized size from committed block metadata. + Confirmed(u32), + /// Untrusted advertised size hint from header sync. + Advertised(u32), + /// No size hint is known; use the EWMA fallback. + Unknown, +} + +#[derive(Clone, Debug)] +pub(super) struct BlockRangeScheduler { + queue: VecDeque, + assigned: HashMap>, + covered: Vec, + fanout: usize, + ewma_fallback_bytes: u64, + floor_estimate_bytes: u64, +} + +impl BlockRangeScheduler { + pub(super) fn new(fanout: usize) -> Self { + Self { + queue: VecDeque::new(), + assigned: HashMap::new(), + covered: Vec::new(), + fanout: fanout.max(1), + ewma_fallback_bytes: DEFAULT_BS_EWMA_SEED_BYTES, + floor_estimate_bytes: DEFAULT_BS_SIZE_FLOOR_BYTES, + } + } + + #[cfg(test)] + pub(super) fn set_estimator_for_tests(&mut self, ewma: u64, floor: u64) { + self.ewma_fallback_bytes = ewma.max(1); + self.floor_estimate_bytes = floor.max(1); + } + + pub(super) fn refresh_needed(&mut self, mut needed: Vec) { + needed.sort_by_key(|block| block.height); + needed.dedup_by_key(|block| block.height); + + let mut range: Option = None; + for block in needed { + let estimate = self.estimate_bytes(block.size); + if let Some(current) = range.as_mut() { + if current.end_height().0.checked_add(1) == Some(block.height.0) { + current.blocks.push(ScheduledBlock { + height: block.height, + hash: block.hash, + estimated_bytes: estimate, + }); + continue; + } + } + if let Some(current) = range.take() { + self.ensure(current); + } + range = Some(BlockRange { + blocks: vec![ScheduledBlock { + height: block.height, + hash: block.hash, + estimated_bytes: estimate, + }], + }); + } + if let Some(current) = range { + self.ensure(current); + } + self.prune_covered(); + } + + pub(super) fn retain_matching_needed(&mut self, needed: &HashMap) { + self.queue.retain(|range| range.matches_needed(needed)); + // Active assignments are paired with outstanding requests, which the + // reactor hash-prunes before retaining scheduler state. + self.assigned.retain(|range, _| { + needed + .get(&range.start) + .is_some_and(|_| needed.contains_key(&range.end)) + }); + } + + pub(super) fn clear_queued(&mut self) { + self.queue.clear(); + } + + pub(super) fn drop_through(&mut self, tip: block::Height) { + for range in &mut self.queue { + range.blocks.retain(|block| block.height > tip); + } + self.queue.retain(|range| !range.blocks.is_empty()); + self.assigned.retain(|range, _| range.end > tip); + } + + pub(super) fn next_for_peer( + &mut self, + peer_id: &ZakuraPeerId, + peer: &PeerBlockState, + budget: &mut ByteBudget, + per_peer_byte_cap: u64, + ) -> Option { + self.prune_covered(); + let index = self.queue.iter().position(|range| { + range.end_height() <= peer.servable_high + && range.start_height() >= peer.servable_low + && self.can_assign_peer_to_range(peer_id, range) + })?; + + let range = self.queue[index].clone(); + // Bound this peer's share of the global byte budget so one fast peer + // cannot reserve the whole window and starve the others. + let peer_reserved: u64 = peer + .outstanding + .iter() + .map(|outstanding| outstanding.reserved_bytes()) + .sum(); + let peer_headroom = per_peer_byte_cap.saturating_sub(peer_reserved); + let max_bytes = budget + .available() + .min(u64::from(peer.max_response_bytes.max(1))) + .min(peer_headroom); + let max_count = peer.max_blocks_per_response.max(1); + let mut estimated_bytes = 0u64; + let mut selected = Vec::new(); + + for block in range.blocks.iter().take(max_count as usize) { + let next_bytes = estimated_bytes.saturating_add(block.estimated_bytes); + if next_bytes > max_bytes { + break; + } + estimated_bytes = next_bytes; + selected.push(*block); + } + + if selected.is_empty() || !budget.try_reserve(estimated_bytes) { + return None; + } + + let count = + u32::try_from(selected.len()).expect("selected block count is capped by u32 status"); + let request = BlockRangeRequest { + start_height: selected[0].height, + count, + anchor_hash: selected[0].hash, + estimated_bytes, + expected_hashes: selected + .iter() + .map(|block| (block.height, block.hash)) + .collect(), + expected_bytes: selected + .iter() + .map(|block| (block.height, block.estimated_bytes)) + .collect(), + }; + + if selected.len() == range.blocks.len() { + if self + .assigned + .get(&range.key()) + .is_some_and(|peers| peers.len() + 1 >= self.fanout) + { + self.queue.remove(index); + } + } else if let Some(queued) = self.queue.get_mut(index) { + queued.blocks.drain(..selected.len()); + } + + self.assigned + .entry(request.key()) + .or_default() + .insert(peer_id.clone()); + Some(request) + } + + pub(super) fn retry(&mut self, range: BlockRangeRequest) { + if self.is_covered(range.start_height, range.end_height()) { + return; + } + self.clear_assignment(&range); + let retry_range = BlockRange { + blocks: (0..range.count) + .filter_map(|offset| { + range + .start_height + .0 + .checked_add(offset) + .map(|raw| ScheduledBlock { + height: block::Height(raw), + hash: range + .expected_hash(block::Height(raw)) + .unwrap_or(block::Hash([0; 32])), + estimated_bytes: range + .estimated_bytes_for_height(block::Height(raw)) + .unwrap_or(range.estimated_bytes / u64::from(range.count)), + }) + }) + .collect(), + }; + for segment in self.uncovered_segments(retry_range).into_iter().rev() { + self.queue.push_front(segment); + } + } + + #[cfg(test)] + pub(super) fn complete(&mut self, range: &BlockRangeRequest, budget: &mut ByteBudget) { + budget.release(range.estimated_bytes); + self.clear_assignment(range); + for (height, _) in &range.expected_hashes { + self.mark_height_covered(*height); + } + } + + #[cfg(test)] + pub(super) fn timeout(&mut self, range: BlockRangeRequest, budget: &mut ByteBudget) { + budget.release(range.estimated_bytes); + self.retry(range); + } + + pub(super) fn forget_peer(&mut self, peer: &ZakuraPeerId) { + for peers in self.assigned.values_mut() { + peers.remove(peer); + } + } + + pub(super) fn clear_assignment(&mut self, range: &BlockRangeRequest) { + self.assigned.remove(&range.key()); + } + + pub(super) fn mark_height_covered(&mut self, height: block::Height) { + self.mark_covered_interval(CoveredRange { + start: height, + end: height, + }); + self.prune_covered(); + } + + pub(super) fn clear_covered_from(&mut self, from: block::Height) { + let previous = previous_height(from); + self.covered.retain_mut(|covered| { + if covered.start >= from { + return false; + } + if covered.end >= from { + if let Some(previous) = previous { + covered.end = previous; + } else { + return false; + } + } + true + }); + } + + #[cfg(test)] + pub(super) fn release_cancelled(&mut self, budget: &mut ByteBudget) { + self.assigned.clear(); + self.queue.clear(); + let reserved = budget.reserved(); + budget.release(reserved); + } + + #[cfg(test)] + pub(super) fn assigned_range_count(&self) -> usize { + self.assigned.len() + } + + /// Diagnostics: number of queued (not-yet-assigned-to-fanout) ranges. + pub(super) fn queued_range_count(&self) -> usize { + self.queue.len() + } + + /// Diagnostics: lowest start height across all queued ranges. + /// + /// A frozen download floor with a non-empty `needed` set but a + /// `queued_min_start` above the gap means the gap range was rejected by + /// `ensure` (covered/queue/assigned overlap) rather than starved. + pub(super) fn queued_min_start(&self) -> Option { + self.queue.iter().map(|range| range.start_height()).min() + } + + /// Diagnostics: number of distinct assigned range keys (including any left + /// behind by `forget_peer` with an empty peer set). + pub(super) fn assigned_key_count(&self) -> usize { + self.assigned.len() + } + + /// Diagnostics: highest end height across all covered intervals. + pub(super) fn covered_max_end(&self) -> Option { + self.covered.iter().map(|covered| covered.end).max() + } + + fn ensure(&mut self, range: BlockRange) { + if range.blocks.is_empty() { + return; + } + for range in self.uncovered_segments(range) { + if self.queue.iter().any(|queued| queued.overlaps(&range)) + || self.assigned.keys().any(|assigned| { + assigned.start <= range.end_height() && assigned.end >= range.start_height() + }) + { + continue; + } + self.queue.push_back(range); + } + } + + fn estimate_bytes(&self, estimate: BlockSizeEstimate) -> u64 { + let hinted = match estimate { + BlockSizeEstimate::Confirmed(size) | BlockSizeEstimate::Advertised(size) => { + u64::from(size) + } + BlockSizeEstimate::Unknown => self.ewma_fallback_bytes, + }; + hinted + .max(self.floor_estimate_bytes) + .min(block::MAX_BLOCK_BYTES) + } + + fn mark_covered_interval(&mut self, mut interval: CoveredRange) { + let mut merged = Vec::with_capacity(self.covered.len().saturating_add(1)); + let mut inserted = false; + for covered in self.covered.drain(..) { + if covered.end.0.saturating_add(1) < interval.start.0 { + merged.push(covered); + } else if interval.end.0.saturating_add(1) < covered.start.0 { + if !inserted { + merged.push(interval); + inserted = true; + } + merged.push(covered); + } else { + interval.start = interval.start.min(covered.start); + interval.end = interval.end.max(covered.end); + } + } + if !inserted { + merged.push(interval); + } + self.covered = merged; + } + + fn prune_covered(&mut self) { + let mut queue = VecDeque::with_capacity(self.queue.len()); + for range in self.queue.drain(..) { + queue.extend(Self::uncovered_segments_for(&self.covered, range)); + } + self.queue = queue; + let covered = &self.covered; + self.assigned.retain(|range, _| { + !covered + .iter() + .any(|covered| covered.start <= range.start && covered.end >= range.end) + }); + } + + fn is_covered(&self, start: block::Height, end: block::Height) -> bool { + self.covered + .iter() + .any(|covered| covered.start <= start && covered.end >= end) + } + + fn can_assign_peer_to_range(&self, peer_id: &ZakuraPeerId, range: &BlockRange) -> bool { + let mut assigned_peers = HashSet::new(); + for (assigned, peers) in &self.assigned { + if assigned.start <= range.end_height() && assigned.end >= range.start_height() { + assigned_peers.extend(peers.iter()); + } + } + + assigned_peers.len() < self.fanout && !assigned_peers.contains(peer_id) + } + + fn uncovered_segments(&self, range: BlockRange) -> Vec { + Self::uncovered_segments_for(&self.covered, range) + } + + fn uncovered_segments_for(covered: &[CoveredRange], range: BlockRange) -> Vec { + let mut segments = Vec::new(); + let mut current = Vec::new(); + for block in range.blocks { + let is_covered = covered + .iter() + .any(|covered| covered.start <= block.height && covered.end >= block.height); + if is_covered { + if !current.is_empty() { + segments.push(BlockRange { blocks: current }); + current = Vec::new(); + } + } else { + current.push(block); + } + } + if !current.is_empty() { + segments.push(BlockRange { blocks: current }); + } + segments + } +} + +#[derive(Clone, Debug)] +struct BlockRange { + blocks: Vec, +} + +impl BlockRange { + fn start_height(&self) -> block::Height { + self.blocks + .first() + .expect("block range is never empty") + .height + } + + fn end_height(&self) -> block::Height { + self.blocks + .last() + .expect("block range is never empty") + .height + } + + fn key(&self) -> BlockRangeKey { + BlockRangeKey { + start: self.start_height(), + end: self.end_height(), + } + } + + fn overlaps(&self, other: &Self) -> bool { + self.start_height() <= other.end_height() && self.end_height() >= other.start_height() + } + + fn matches_needed(&self, needed: &HashMap) -> bool { + self.blocks + .iter() + .all(|block| needed.get(&block.height) == Some(&block.hash)) + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +struct ScheduledBlock { + height: block::Height, + hash: block::Hash, + estimated_bytes: u64, +} + +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] +struct BlockRangeKey { + start: block::Height, + end: block::Height, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +struct CoveredRange { + start: block::Height, + end: block::Height, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct BlockRangeRequest { + pub(super) start_height: block::Height, + pub(super) count: u32, + pub(super) anchor_hash: block::Hash, + pub(super) estimated_bytes: u64, + pub(super) expected_hashes: Vec<(block::Height, block::Hash)>, + pub(super) expected_bytes: Vec<(block::Height, u64)>, +} + +impl BlockRangeRequest { + pub(super) fn end_height(&self) -> block::Height { + height_after_count(self.start_height, self.count) + .and_then(previous_height) + .expect("range request count is non-zero") + } + + pub(super) fn contains(&self, height: block::Height) -> bool { + self.start_height <= height && height <= self.end_height() + } + + pub(super) fn expected_hash(&self, height: block::Height) -> Option { + self.expected_hashes + .iter() + .find_map(|(known_height, hash)| (*known_height == height).then_some(*hash)) + } + + pub(super) fn estimated_bytes_for_height(&self, height: block::Height) -> Option { + self.expected_bytes + .iter() + .find_map(|(known_height, bytes)| (*known_height == height).then_some(*bytes)) + } + + pub(super) fn single_height_retry(&self, height: block::Height) -> Option { + let hash = self.expected_hash(height)?; + let estimated_bytes = self.estimated_bytes_for_height(height)?; + Some(Self { + start_height: height, + count: 1, + anchor_hash: hash, + estimated_bytes, + expected_hashes: vec![(height, hash)], + expected_bytes: vec![(height, estimated_bytes)], + }) + } + + pub(super) fn matches_needed(&self, needed: &HashMap) -> bool { + self.expected_hashes + .iter() + .all(|(height, hash)| needed.get(height) == Some(hash)) + } + + fn key(&self) -> BlockRangeKey { + BlockRangeKey { + start: self.start_height, + end: self.end_height(), + } + } +} diff --git a/zebra-network/src/zakura/block_sync/service.rs b/zebra-network/src/zakura/block_sync/service.rs new file mode 100644 index 00000000000..8b29cc5f7b2 --- /dev/null +++ b/zebra-network/src/zakura/block_sync/service.rs @@ -0,0 +1,536 @@ +use super::{config::*, events::*, pipe::*, wire::*, *}; +use crate::zakura::{ + handle_pipe_exit, spawn_supervised_pipe, Flow, FramedSend, OrderedSendError, Peer, + PeerStreamSession, Pipe, Service, SinkReject, Stream, StreamMode, ZakuraPeerId, + FRAME_HEADER_BYTES, +}; +// The per-peer block-sync `Source` frame-pump is test-only scaffolding (see +// `BlockSyncPeerRecord` / `add_peer`); its trait and boxed-future alias are only +// referenced by that `cfg(test)` task. +#[cfg(test)] +use crate::zakura::{BoxRunFuture, Source}; + +/// Maximum frame bytes for one stream-6 body frame plus protocol framing. +/// +/// A block body is still decoded and validated against Zebra's +/// `MAX_BLOCK_BYTES`; this frame cap has extra slack so stream-6 can classify +/// oversized or incompatible block-sync payloads in the codec instead of +/// dropping them at the raw transport gate. +pub const MAX_BS_FRAME_BYTES: u32 = { + // This cast is safe: MAX_BS_MESSAGE_BYTES is asserted below 4 MiB. + (MAX_BS_MESSAGE_BYTES + FRAME_HEADER_BYTES) as u32 +}; + +const BLOCK_SYNC_SERVICE_STREAMS: [Stream; 1] = [Stream { + kind: ZAKURA_STREAM_BLOCK_SYNC, + version: ZAKURA_BLOCK_SYNC_STREAM_VERSION, + frame_cap: MAX_BS_FRAME_BYTES, + capability: ZAKURA_CAP_BLOCK_SYNC, + mode: StreamMode::Ordered, +}]; + +/// Service-declared streams for native block sync. +pub(crate) fn block_sync_streams() -> &'static [Stream] { + &BLOCK_SYNC_SERVICE_STREAMS +} + +/// Cloneable typed stream-6 sender. +#[derive(Clone, Debug)] +pub struct BlockSyncPeerSession { + peer_id: ZakuraPeerId, + direction: ServicePeerDirection, + send: FramedSend, + cancel_token: CancellationToken, +} + +impl BlockSyncPeerSession { + pub(crate) fn new(session: &PeerStreamSession, direction: ServicePeerDirection) -> Self { + Self { + peer_id: session.peer_id().clone(), + direction, + send: session.sender(), + cancel_token: session.cancel_token(), + } + } + + /// Authenticated peer identity for this block-sync session. + pub fn peer_id(&self) -> &ZakuraPeerId { + &self.peer_id + } + + /// Direction of the underlying Zakura connection. + pub fn direction(&self) -> ServicePeerDirection { + self.direction + } + + /// Peer disconnect/local shutdown cancellation token. + pub fn cancel_token(&self) -> CancellationToken { + self.cancel_token.clone() + } + + /// Send a typed status advertisement. + pub fn try_send_status(&self, status: BlockSyncStatus) -> Result<(), OrderedSendError> { + self.try_send_message(BlockSyncMessage::Status(status)) + } + + /// Send a typed block range request. + pub fn try_send_get_blocks( + &self, + start_height: block::Height, + count: u32, + ) -> Result<(), OrderedSendError> { + self.try_send_message(BlockSyncMessage::GetBlocks { + start_height, + count, + }) + } + + /// Send one typed block body frame. + pub fn try_send_block(&self, block: Arc) -> Result<(), OrderedSendError> { + self.try_send_message(BlockSyncMessage::Block(block)) + } + + /// Send a typed response terminator. + pub fn try_send_blocks_done( + &self, + start_height: block::Height, + returned: u32, + ) -> Result<(), OrderedSendError> { + self.try_send_message(BlockSyncMessage::BlocksDone { + start_height, + returned, + }) + } + + /// Send a typed unavailable-range response. + pub fn try_send_range_unavailable( + &self, + start_height: block::Height, + count: u32, + ) -> Result<(), OrderedSendError> { + self.try_send_message(BlockSyncMessage::RangeUnavailable { + start_height, + count, + }) + } + + fn try_send_message(&self, msg: BlockSyncMessage) -> Result<(), OrderedSendError> { + let frame = msg + .encode_frame() + .map_err(|error| OrderedSendError::Encode(Box::new(error)))?; + match self.send.try_send(frame) { + Ok(()) => Ok(()), + Err(mpsc::error::TrySendError::Full(_frame)) => Err(OrderedSendError::Full), + Err(mpsc::error::TrySendError::Closed(_frame)) => Err(OrderedSendError::Closed), + } + } +} + +/// Native stream-6 block-sync service scaffold. +#[derive(Debug)] +pub(crate) struct BlockSyncService { + inner: Arc, + _held_events: Option>>>, + _reactor_task: Option>, +} + +#[derive(Debug)] +struct BlockSyncServiceInner { + config: ZakuraBlockSyncConfig, + events: mpsc::Sender, + lifecycle: mpsc::UnboundedSender, + peers: StdMutex>, +} + +#[derive(Debug)] +struct BlockSyncPeerRecord { + direction: ServicePeerDirection, + cancel_token: CancellationToken, + // Production outbound block-sync sends are authoritative through + // `BlockSyncPeerSession`: the reactor calls `try_send_get_blocks`/etc. + // directly (see `reactor::schedule`). The per-peer `BlockSyncSource` action + // pump and its `actions` sender are test-only scaffolding — no non-test code + // produces into this channel, and `drive_block_sync_actions` deliberately + // ignores the reactor's duplicate `SendMessage` to avoid double-sending. + // Gating the sender and the task handle to `cfg(test)` keeps that contract + // compiler-enforced: production has no producer to wire and therefore cannot + // double-send, and it retains no idle frame-pump task/channel per peer. + #[cfg(test)] + actions: mpsc::Sender, + #[cfg(test)] + _tasks: Vec>, +} + +impl BlockSyncService { + pub(crate) fn new(config: ZakuraBlockSyncConfig) -> Self { + Self::new_with_startup(BlockSyncStartup::inert(config)) + } + + pub(crate) fn new_with_handle(config: ZakuraBlockSyncConfig, handle: BlockSyncHandle) -> Self { + Self { + inner: Arc::new(BlockSyncServiceInner { + config, + events: handle.events.clone(), + lifecycle: handle.lifecycle.clone(), + peers: StdMutex::new(HashMap::new()), + }), + _held_events: None, + _reactor_task: None, + } + } + + pub(crate) fn new_with_header_tip( + config: ZakuraBlockSyncConfig, + header_tip: watch::Receiver<(block::Height, block::Hash)>, + ) -> Self { + let best_header_tip = *header_tip.borrow(); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height::MIN, + verified_block_tip: block::Height::MIN, + verified_block_hash: block::Hash([0; 32]), + }, + best_header_tip, + header_tip, + config, + ); + Self::new_with_startup(startup) + } + + fn new_with_startup(startup: BlockSyncStartup) -> Self { + let config = startup.config.clone(); + let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup); + Self { + inner: Arc::new(BlockSyncServiceInner { + config, + events: handle.events.clone(), + lifecycle: handle.lifecycle.clone(), + peers: StdMutex::new(HashMap::new()), + }), + _held_events: None, + _reactor_task: Some(reactor_task), + } + } + + #[cfg(test)] + pub(crate) fn new_for_test( + config: ZakuraBlockSyncConfig, + ) -> (Self, mpsc::Receiver) { + let (events, event_rx) = mpsc::channel(config.peer_limits.inbound_queue_depth.max(1)); + let (lifecycle, mut lifecycle_rx) = mpsc::unbounded_channel(); + let events_for_lifecycle = events.clone(); + tokio::spawn(async move { + while let Some(event) = lifecycle_rx.recv().await { + let _ = events_for_lifecycle.send(event).await; + } + }); + ( + Self { + inner: Arc::new(BlockSyncServiceInner { + config, + events, + lifecycle, + peers: StdMutex::new(HashMap::new()), + }), + _held_events: None, + _reactor_task: None, + }, + event_rx, + ) + } + + #[cfg(test)] + pub(crate) fn new_with_handle_for_test( + config: ZakuraBlockSyncConfig, + handle: BlockSyncHandle, + ) -> Self { + Self::new_with_handle(config, handle) + } + + #[cfg(test)] + pub(crate) fn peer_count(&self) -> usize { + self.inner + .peers + .lock() + .expect("block-sync peer map mutex is never poisoned") + .len() + } + + #[cfg(test)] + pub(crate) async fn send_action( + &self, + action: BlockSyncAction, + ) -> Result<(), mpsc::error::SendError> { + let BlockSyncAction::SendMessage { peer, .. } = &action else { + return Err(mpsc::error::SendError(action)); + }; + let peer = peer.clone(); + let sender = match self.inner.peers.lock() { + Ok(peers) => peers.get(&peer).map(|record| record.actions.clone()), + Err(_) => return Err(mpsc::error::SendError(action)), + }; + let Some(sender) = sender else { + return Err(mpsc::error::SendError(action)); + }; + sender.send(action).await + } + + fn peer_slots_free(&self, direction: ServicePeerDirection) -> bool { + let peers = self + .inner + .peers + .lock() + .expect("block-sync peer map mutex is never poisoned"); + let count = peers + .values() + .filter(|record| record.direction == direction) + .count(); + let cap = match direction { + ServicePeerDirection::Inbound => self.inner.config.peer_limits.max_inbound_peers, + ServicePeerDirection::Outbound => self.inner.config.peer_limits.max_outbound_peers, + }; + count < cap + } +} + +impl Service for BlockSyncService { + fn name(&self) -> &'static str { + "block-sync" + } + + fn streams(&self) -> &[Stream] { + block_sync_streams() + } + + fn wants_peer( + &self, + _peer: &ZakuraPeerId, + _negotiated: u64, + direction: ServicePeerDirection, + ) -> bool { + self.peer_slots_free(direction) + } + + fn add_peer(&self, mut peer: Peer) { + if !self.peer_slots_free(peer.direction) { + return; + } + + let Some((recv, send)) = peer.take_stream(ZAKURA_STREAM_BLOCK_SYNC) else { + return; + }; + + let peer_id = peer.id.clone(); + let session = PeerStreamSession::new( + peer_id.clone(), + ZAKURA_STREAM_BLOCK_SYNC, + recv, + send, + peer.service_cancel_token(), + ); + let service_cancel_token = session.cancel_token(); + let connection_cancel_token = peer.cancel_token(); + let block_sync_session = BlockSyncPeerSession::new(&session, peer.direction); + let (_session_peer, _stream_kind, recv, send, _session_cancel) = session.into_parts(); + + // The per-peer block-sync source frame-pump is test-only scaffolding (see + // `BlockSyncPeerRecord`). Production outbound frames go directly through + // `BlockSyncPeerSession`, so only the test build spawns the source to + // exercise `send_action`; production drops the redundant transport sender. + // The outbound stream stays alive through the `BlockSyncPeerSession` clone + // the reactor holds, so nothing is lost by not retaining it here. + #[cfg(test)] + let (actions_tx, source_task) = { + let (actions_tx, actions_rx) = + mpsc::channel(self.inner.config.peer_limits.outbound_queue_depth.max(1)); + let source_task = spawn_block_sync_source( + peer_id.clone(), + actions_rx, + service_cancel_token.clone(), + send, + ); + (actions_tx, source_task) + }; + #[cfg(not(test))] + drop(send); + + let events = self.inner.events.clone(); + let run_peer_id = peer_id.clone(); + let run_cancel = service_cancel_token.clone(); + let on_teardown = { + let lifecycle = self.inner.lifecycle.clone(); + let peer_id = peer_id.clone(); + let inner = self.inner.clone(); + move || { + if let Ok(mut peers) = inner.peers.lock() { + peers.remove(&peer_id); + } + let _ = lifecycle.send(BlockSyncEvent::PeerDisconnected(peer_id)); + } + }; + let on_panic = { + let connection_cancel_token = connection_cancel_token.clone(); + move || connection_cancel_token.cancel() + }; + // A protocol reject is fatal to the whole connection; a normal/parked exit + // leaves it for the other services riding on it. Panic teardown is in + // `on_panic`. + let pipe = async move { + handle_pipe_exit( + "block-sync", + &connection_cancel_token, + run_peer(run_peer_id, recv, events, run_cancel).await, + ); + }; + // Let the returned handle drop to detach the supervised task (like + // `tokio::spawn`); the `PipeTeardown` still runs on every exit path. + spawn_supervised_pipe( + peer_id.clone(), + service_cancel_token.clone(), + on_teardown, + on_panic, + pipe, + ); + + { + let mut peers = self + .inner + .peers + .lock() + .expect("block-sync peer map mutex is never poisoned"); + peers.insert( + peer_id.clone(), + BlockSyncPeerRecord { + direction: peer.direction, + cancel_token: service_cancel_token, + #[cfg(test)] + actions: actions_tx, + #[cfg(test)] + _tasks: vec![source_task], + }, + ); + } + + let _ = self + .inner + .lifecycle + .send(BlockSyncEvent::PeerConnected(block_sync_session)); + } + + fn remove_peer(&self, peer: &ZakuraPeerId) { + let removed = self + .inner + .peers + .lock() + .expect("block-sync peer map mutex is never poisoned") + .remove(peer); + if let Some(record) = removed { + record.cancel_token.cancel(); + } + let _ = self + .inner + .lifecycle + .send(BlockSyncEvent::PeerDisconnected(peer.clone())); + } + + fn deliver_frame( + &self, + peer_id: ZakuraPeerId, + stream_kind: u16, + frame: Frame, + ) -> Result<(), SinkReject> { + if stream_kind != ZAKURA_STREAM_BLOCK_SYNC { + return Ok(()); + } + + let mut pipe = block_sync_pipe(peer_id, self.inner.events.clone()); + match pipe.run_one(frame) { + Flow::Continue(()) | Flow::Done => Ok(()), + Flow::Reject(reject) => Err(reject), + } + } +} + +// Test-only per-peer outbound frame-pump. Production block-sync sends go through +// `BlockSyncPeerSession` directly (see `add_peer` / `BlockSyncPeerRecord`); this +// source exists solely so tests can drive outbound frames via `send_action`. +#[cfg(test)] +fn spawn_block_sync_source( + peer_id: ZakuraPeerId, + actions: mpsc::Receiver, + cancel_token: CancellationToken, + send: FramedSend, +) -> JoinHandle<()> { + task::spawn(async move { + let source = Box::new(BlockSyncSource { + peer_id, + actions, + cancel_token, + }); + source.run(send).await; + }) +} + +#[cfg(test)] +#[derive(Debug)] +struct BlockSyncSource { + peer_id: ZakuraPeerId, + actions: mpsc::Receiver, + cancel_token: CancellationToken, +} + +#[cfg(test)] +impl Source for BlockSyncSource { + fn run(mut self: Box, send: FramedSend) -> BoxRunFuture<'static, ()> { + Box::pin(async move { + loop { + let action = tokio::select! { + _ = self.cancel_token.cancelled() => return, + action = self.actions.recv() => { + let Some(action) = action else { + return; + }; + action + } + }; + + let BlockSyncAction::SendMessage { peer, msg } = action else { + continue; + }; + if peer != self.peer_id { + continue; + } + + let frame = match msg.encode_frame() { + Ok(frame) => frame, + Err(error) => { + tracing::debug!( + ?error, + peer = ?self.peer_id, + "block-sync source refused to encode outbound message" + ); + continue; + } + }; + + if send.send(frame).await.is_err() { + return; + } + } + }) + } +} + +pub(super) fn block_sync_pipe( + peer_id: ZakuraPeerId, + events: mpsc::Sender, +) -> Pipe { + Pipe::new( + peer_id, + BsLocal, + BsEnv::new(events), + block_sync_guard(), + run_inbound, + &PIPE_SHAPE, + ) +} diff --git a/zebra-network/src/zakura/block_sync/state.rs b/zebra-network/src/zakura/block_sync/state.rs new file mode 100644 index 00000000000..aa0a60b2e4e --- /dev/null +++ b/zebra-network/src/zakura/block_sync/state.rs @@ -0,0 +1,436 @@ +use super::{config::*, events::BlockApplyToken, reorder::*, scheduler::*, *}; +use crate::zakura::{ + chain_frontier_from_parts, Frontier, FrontierUpdate, ServicePeerDirection, ServicePeerSnapshot, + ZakuraBlockSyncCandidateState, +}; + +/// Hard ceiling on outbound block-range requests kept in flight to one peer. +/// +/// A safety bound only; the binding per-peer concurrency is the peer's advertised +/// `max_inflight_requests` (config `max_inflight_requests`, clamped to +/// [`DEFAULT_BS_MAX_INFLIGHT`]). Set well above the configured cap so the byte +/// budget and per-peer byte cap, not this ceiling, govern in-flight depth. +pub(super) const EFFECTIVE_BS_OUTBOUND_INFLIGHT_PER_PEER: usize = 64; + +/// Cached chain frontiers used by the block-sync reactor. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct BlockSyncFrontiers { + /// Shared finalized height supplied by state. + pub finalized_height: block::Height, + /// Highest verified block-body height supplied by state. + pub verified_block_tip: block::Height, + /// Hash of [`verified_block_tip`](Self::verified_block_tip). + pub verified_block_hash: block::Hash, +} + +/// Startup inputs for the dependency-neutral block-sync reactor. +#[derive(Clone, Debug)] +pub struct BlockSyncStartup { + /// Cached state frontiers at startup. + pub frontiers: BlockSyncFrontiers, + /// Durable best header tip at startup. + pub best_header_tip: (block::Height, block::Hash), + /// Header-sync best-tip watch used as the moving body-download target. + pub header_tip: Option>, + /// Shared sync exchange frontier stream used as the moving body-download target. + pub frontier_updates: Option>, + /// Local stream-6 configuration. + pub config: ZakuraBlockSyncConfig, + /// Shared shutdown signal owned by the embedding endpoint or test harness. + pub shutdown: CancellationToken, + /// Enables query actions for state-backed metadata. + pub state_queries_enabled: bool, + /// JSONL trace emitter for block-sync scheduling, download, and commit rows. + pub trace: ZakuraTrace, +} + +impl BlockSyncStartup { + /// Build block-sync startup config from durable/frontier facts. + pub fn new( + frontiers: BlockSyncFrontiers, + best_header_tip: (block::Height, block::Hash), + header_tip: watch::Receiver<(block::Height, block::Hash)>, + config: ZakuraBlockSyncConfig, + ) -> Self { + Self { + frontiers, + best_header_tip, + header_tip: Some(header_tip), + frontier_updates: None, + config, + shutdown: CancellationToken::new(), + state_queries_enabled: true, + trace: ZakuraTrace::noop(), + } + } + + /// Build block-sync startup config from shared sync exchange frontiers. + pub fn new_with_exchange( + frontiers: BlockSyncFrontiers, + best_header_tip: (block::Height, block::Hash), + frontier_updates: watch::Receiver, + config: ZakuraBlockSyncConfig, + ) -> Self { + Self { + frontiers, + best_header_tip, + header_tip: None, + frontier_updates: Some(frontier_updates), + config, + shutdown: CancellationToken::new(), + state_queries_enabled: true, + trace: ZakuraTrace::noop(), + } + } + + /// Build a latest-value frontier update stream from legacy startup pieces. + pub fn frontier_update_from_parts( + frontiers: BlockSyncFrontiers, + best_header_tip: (block::Height, block::Hash), + ) -> FrontierUpdate { + FrontierUpdate { + frontier: chain_frontier_from_parts( + frontiers.finalized_height, + Frontier::new(frontiers.verified_block_tip, frontiers.verified_block_hash), + Frontier::new(best_header_tip.0, best_header_tip.1), + ), + change: crate::zakura::FrontierChange::Snapshot, + } + } + + pub(super) fn inert(config: ZakuraBlockSyncConfig) -> Self { + Self { + frontiers: BlockSyncFrontiers { + finalized_height: block::Height::MIN, + verified_block_tip: block::Height::MIN, + verified_block_hash: block::Hash([0; 32]), + }, + best_header_tip: (block::Height::MIN, block::Hash([0; 32])), + header_tip: None, + frontier_updates: None, + config, + shutdown: CancellationToken::new(), + state_queries_enabled: false, + trace: ZakuraTrace::noop(), + } + } +} + +/// Cheap cloneable handle used by services and drivers to inform block sync. +#[derive(Clone, Debug)] +pub struct BlockSyncHandle { + pub(super) events: mpsc::Sender, + pub(super) lifecycle: mpsc::UnboundedSender, + pub(super) peers: watch::Receiver, + pub(super) status: watch::Receiver, + pub(super) candidates: watch::Receiver, +} + +impl BlockSyncHandle { + /// Send a fact/event to the block-sync reactor. + pub async fn send( + &self, + event: BlockSyncEvent, + ) -> Result<(), mpsc::error::SendError> { + self.events.send(event).await + } + + /// Try to send a fact/event without awaiting. + pub fn try_send( + &self, + event: BlockSyncEvent, + ) -> Result<(), mpsc::error::TrySendError> { + self.events.try_send(event) + } + + /// Send a peer lifecycle event without sharing the bounded wire-event queue. + pub fn send_lifecycle( + &self, + event: BlockSyncEvent, + ) -> Result<(), mpsc::error::SendError> { + self.lifecycle + .send(event) + .map_err(|error| mpsc::error::SendError(error.0)) + } + + /// Return the currently cached peer slot snapshot. + pub fn peer_snapshot(&self) -> ServicePeerSnapshot { + *self.peers.borrow() + } + + /// Subscribe to local block-sync status advertisements. + pub fn subscribe_status(&self) -> watch::Receiver { + self.status.clone() + } + + /// Return the currently cached local status advertisement. + pub fn local_status(&self) -> BlockSyncStatus { + *self.status.borrow() + } + + /// Subscribe to block-sync candidate-selection hints. + pub fn subscribe_candidate_state(&self) -> watch::Receiver { + self.candidates.clone() + } + + /// Return the currently cached block-sync candidate-selection hints. + pub fn candidate_state(&self) -> ZakuraBlockSyncCandidateState { + self.candidates.borrow().clone() + } +} + +#[derive(Clone, Debug)] +pub(super) struct BlockSyncState { + pub(super) finalized_height: block::Height, + pub(super) verified_block_tip: block::Height, + pub(super) verified_block_hash: block::Hash, + pub(super) body_download_floor: block::Height, + pub(super) servable_high: block::Height, + pub(super) servable_hash: block::Hash, + pub(super) best_header_tip: block::Height, + pub(super) best_header_hash: block::Hash, + pub(super) peers: HashMap, + pub(super) parked_peers: HashSet, + pub(super) disconnected_peers: HashSet, + pub(super) schedule: BlockRangeScheduler, + pub(super) reorder: ReorderBuffer, + pub(super) applying: BTreeMap, + pub(super) next_apply_token: BlockApplyToken, + pub(super) budget: ByteBudget, + pub(super) needed_heights: Vec, + pub(super) status_refresh: RateMeter, + pub(super) pending_status_refresh: bool, + pub(super) last_advertised_status: BlockSyncStatus, +} + +impl BlockSyncState { + pub(super) fn new(startup: &BlockSyncStartup) -> Self { + let last_advertised_status = BlockSyncStatus { + servable_low: block::Height::MIN, + servable_high: startup.frontiers.verified_block_tip, + tip_hash: startup.frontiers.verified_block_hash, + max_blocks_per_response: startup.config.advertised_max_blocks_per_response(), + max_inflight_requests: startup.config.advertised_max_inflight_requests(), + max_response_bytes: startup.config.advertised_max_response_bytes(), + }; + + Self { + finalized_height: startup.frontiers.finalized_height, + verified_block_tip: startup.frontiers.verified_block_tip, + verified_block_hash: startup.frontiers.verified_block_hash, + body_download_floor: startup.frontiers.verified_block_tip, + servable_high: startup.frontiers.verified_block_tip, + servable_hash: startup.frontiers.verified_block_hash, + best_header_tip: startup.best_header_tip.0, + best_header_hash: startup.best_header_tip.1, + peers: HashMap::new(), + parked_peers: HashSet::new(), + disconnected_peers: HashSet::new(), + schedule: BlockRangeScheduler::new(startup.config.fanout), + reorder: ReorderBuffer::new(), + applying: BTreeMap::new(), + next_apply_token: 1, + budget: ByteBudget::new(startup.config.max_inflight_block_bytes), + needed_heights: Vec::new(), + status_refresh: RateMeter::new(startup.config.status_refresh_interval), + pending_status_refresh: false, + last_advertised_status, + } + } + + pub(super) fn peer_snapshot(&self, limits: ServicePeerLimits) -> ServicePeerSnapshot { + let inbound = self + .peers + .values() + .filter(|peer| peer.direction == ServicePeerDirection::Inbound) + .count(); + let outbound = self + .peers + .values() + .filter(|peer| peer.direction == ServicePeerDirection::Outbound) + .count(); + ServicePeerSnapshot::new(inbound, outbound, limits) + } +} + +#[derive(Clone, Debug)] +pub(super) struct ApplyingBlock { + pub(super) token: BlockApplyToken, + pub(super) hash: block::Hash, + pub(super) block: Arc, + pub(super) bytes: u64, + pub(super) submitted: bool, +} + +#[derive(Clone, Debug)] +pub(super) struct PeerBlockState { + pub(super) session: BlockSyncPeerSession, + pub(super) direction: ServicePeerDirection, + pub(super) servable_low: block::Height, + pub(super) servable_high: block::Height, + pub(super) max_blocks_per_response: u32, + pub(super) max_inflight_requests: u16, + pub(super) max_response_bytes: u32, + pub(super) received_status: bool, + pub(super) outstanding: Vec, + pub(super) inbound_status: RateMeter, + pub(super) unsolicited: RateMeter, + pub(super) served_blocks_inflight: u16, + pub(super) misbehavior: u32, +} + +impl PeerBlockState { + pub(super) fn new(session: BlockSyncPeerSession, config: &ZakuraBlockSyncConfig) -> Self { + Self { + direction: session.direction(), + session, + servable_low: block::Height::MIN, + servable_high: block::Height::MIN, + max_blocks_per_response: config.advertised_max_blocks_per_response(), + max_inflight_requests: config.advertised_max_inflight_requests(), + max_response_bytes: config.advertised_max_response_bytes(), + received_status: false, + outstanding: Vec::new(), + inbound_status: RateMeter::new( + config.status_refresh_interval.min(Duration::from_secs(1)), + ), + unsolicited: RateMeter::new(config.status_refresh_interval), + served_blocks_inflight: 0, + misbehavior: 0, + } + } + + pub(super) fn available_slots(&self) -> usize { + usize::from(self.max_inflight_requests) + .min(EFFECTIVE_BS_OUTBOUND_INFLIGHT_PER_PEER) + .saturating_sub(self.outstanding.len()) + } + + pub(super) fn can_serve_any(&self, heights: &[block::Height]) -> bool { + self.received_status + && heights + .iter() + .any(|height| self.servable_low <= *height && *height <= self.servable_high) + } + + pub(super) fn outstanding_index_for_height(&self, height: block::Height) -> Option { + self.outstanding + .iter() + .position(|outstanding| outstanding.request.contains(height)) + } + + pub(super) fn outstanding_index_for_start(&self, start_height: block::Height) -> Option { + self.outstanding + .iter() + .position(|outstanding| outstanding.request.start_height == start_height) + } + + pub(super) fn try_start_serving_blocks(&mut self, local_inflight_cap: u16) -> bool { + if self.served_blocks_inflight >= local_inflight_cap { + return false; + } + self.served_blocks_inflight = self.served_blocks_inflight.saturating_add(1); + true + } + + pub(super) fn finish_serving_blocks(&mut self) { + self.served_blocks_inflight = self.served_blocks_inflight.saturating_sub(1); + } +} + +#[derive(Clone, Debug)] +pub(super) struct OutstandingBlockRange { + pub(super) request: BlockRangeRequest, + pub(super) deadline: Instant, + pub(super) received: HashSet, +} + +impl OutstandingBlockRange { + pub(super) fn reserved_bytes(&self) -> u64 { + self.request + .expected_bytes + .iter() + .filter(|(height, _)| !self.received.contains(height)) + .map(|(_, bytes)| *bytes) + .sum() + } + + pub(super) fn estimated_bytes_for_height(&self, height: block::Height) -> Option { + self.request.estimated_bytes_for_height(height) + } + + pub(super) fn has_received(&self, height: block::Height) -> bool { + self.received.contains(&height) + } + + pub(super) fn mark_received(&mut self, height: block::Height) { + self.received.insert(height); + } + + pub(super) fn mark_received_through(&mut self, tip: block::Height) -> u64 { + self.request + .expected_bytes + .iter() + .filter_map(|(height, bytes)| { + (*height <= tip && self.received.insert(*height)).then_some(*bytes) + }) + .sum() + } + + pub(super) fn is_complete(&self) -> bool { + self.received.len() == self.request.expected_hashes.len() + } + + pub(super) fn missing_retry_requests(&self) -> Vec { + self.request + .expected_hashes + .iter() + .filter_map(|(height, _)| { + (!self.received.contains(height)) + .then(|| self.request.single_height_retry(*height)) + .flatten() + }) + .collect() + } +} + +#[derive(Clone, Debug)] +pub(super) struct RateMeter { + pub(super) next_allowed: Instant, + pub(super) interval: Duration, +} + +impl RateMeter { + pub(super) fn new(interval: Duration) -> Self { + Self { + next_allowed: Instant::now(), + interval, + } + } + + pub(super) fn try_take(&mut self, now: Instant) -> bool { + if now < self.next_allowed { + return false; + } + self.next_allowed = now + self.interval; + true + } +} + +// `ByteBudget` was promoted to `transport/guard.rs` so byte-rate protection is +// reusable across services. Re-exported here so existing block_sync call sites +// (`reorder.rs`, `scheduler.rs`, `tests.rs`, and the field on this module's +// state) keep resolving unchanged. +pub(crate) use crate::zakura::transport::ByteBudget; + +pub(super) fn next_height(height: block::Height) -> Option { + height.0.checked_add(1).map(block::Height) +} + +pub(super) fn previous_height(height: block::Height) -> Option { + height.0.checked_sub(1).map(block::Height) +} + +pub(super) fn height_after_count(start: block::Height, count: u32) -> Option { + start.0.checked_add(count).map(block::Height) +} diff --git a/zebra-network/src/zakura/block_sync/tests.rs b/zebra-network/src/zakura/block_sync/tests.rs new file mode 100644 index 00000000000..875d789e795 --- /dev/null +++ b/zebra-network/src/zakura/block_sync/tests.rs @@ -0,0 +1,6389 @@ +use std::{collections::HashMap, future}; + +use super::*; +use super::{ + config::{DEFAULT_BS_MAX_INFLIGHT, MAX_BS_RESPONSE_BYTES}, + reactor::node_id_from_block_peer_id, + reorder::*, + scheduler::*, + state::*, +}; +use crate::zakura::{ + framed_channel, ChainFrontier, FramedRecv, FramedSend, Frontier, FrontierChange, + FrontierUpdate, Peer, PeerStreamSession, Service, ServicePeerSnapshot, ServiceRegistry, + StreamMode, ZakuraBlockSyncCandidateState, ZakuraSyncExchange, +}; +use zebra_chain::{ + fmt::HexDebug, + serialization::{ZcashDeserializeInto, ZcashSerialize}, + transaction::Transaction, + transparent, +}; +use zebra_test::vectors::{BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES, BLOCK_MAINNET_3_BYTES}; + +fn peer(byte: u8) -> ZakuraPeerId { + ZakuraPeerId::new(vec![byte; 32]).expect("test peer id is within bounds") +} + +fn mainnet_block(bytes: &[u8]) -> Arc { + Arc::new(bytes.zcash_deserialize_into().expect("block vector parses")) +} + +fn mainnet_blocks_1_to_3() -> Vec> { + vec![ + mainnet_block(&BLOCK_MAINNET_1_BYTES), + mainnet_block(&BLOCK_MAINNET_2_BYTES), + mainnet_block(&BLOCK_MAINNET_3_BYTES), + ] +} + +fn forked_block(block: &Arc, nonce_tag: u8) -> Arc { + let mut fork = block.as_ref().clone(); + let mut header = *fork.header; + header.nonce = HexDebug([nonce_tag; 32]); + fork.header = Arc::new(header); + Arc::new(fork) +} + +fn block_with_bad_merkle_root( + block: &Arc, + extra_tx: &Arc, +) -> Arc { + let mut bad_block = block.as_ref().clone(); + bad_block + .transactions + .push(extra_tx.transactions[0].clone()); + + assert_eq!(bad_block.hash(), block.hash()); + assert_eq!(bad_block.coinbase_height(), block.coinbase_height()); + assert_ne!( + bad_block + .transactions + .iter() + .collect::(), + bad_block.header.merkle_root + ); + + Arc::new(bad_block) +} + +/// Build `count` internally-consistent blocks at the sequential heights +/// `1..=count`. +/// +/// Each block is mainnet block 1 with its coinbase height rewritten and its +/// header merkle root recomputed, so it has a distinct hash and passes the +/// reactor's `block_merkle_root_matches_header` check. The real test vectors +/// only cover a handful of contiguous heights, which is too few to flood the +/// per-peer wire queue, so the body-flood test synthesizes its own chain. +fn fake_sequential_blocks(count: u32) -> Vec> { + let template = mainnet_block(&BLOCK_MAINNET_1_BYTES); + (1..=count) + .map(|height| fake_block_at_height(&template, block::Height(height))) + .collect() +} + +fn fake_blocks_in_range(start: u32, end: u32) -> Vec> { + let template = mainnet_block(&BLOCK_MAINNET_1_BYTES); + (start..=end) + .map(|height| fake_block_at_height(&template, block::Height(height))) + .collect() +} + +fn fake_block_at_height(template: &Arc, height: block::Height) -> Arc { + let mut block = template.as_ref().clone(); + + let mut coinbase = block.transactions[0].clone(); + let input = match Arc::make_mut(&mut coinbase) { + Transaction::V1 { inputs, .. } + | Transaction::V2 { inputs, .. } + | Transaction::V3 { inputs, .. } + | Transaction::V4 { inputs, .. } + | Transaction::V5 { inputs, .. } => &mut inputs[0], + }; + match input { + transparent::Input::Coinbase { + height: coinbase_height, + .. + } => *coinbase_height = height, + _ => panic!("template block must start with a coinbase input"), + } + block.transactions[0] = coinbase; + + // Rewriting the coinbase changes the merkle root, so recompute it; otherwise + // the reactor's merkle check rejects the body before it can be buffered. + let merkle_root = block.transactions.iter().collect::(); + let mut header = *block.header; + header.merkle_root = merkle_root; + block.header = Arc::new(header); + + Arc::new(block) +} + +fn block_size(block: &block::Block) -> u32 { + u32::try_from( + block + .zcash_serialize_to_vec() + .expect("test block serializes") + .len(), + ) + .expect("test block size fits u32") +} + +fn status() -> BlockSyncStatus { + BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(42), + tip_hash: block::Hash([7; 32]), + max_blocks_per_response: 16, + max_inflight_requests: 4, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + } +} + +fn immediate_body_download_config() -> ZakuraBlockSyncConfig { + ZakuraBlockSyncConfig { + near_tip_body_download_pause_blocks: 0, + ..ZakuraBlockSyncConfig::default() + } +} + +fn test_frontier(height: u32) -> Frontier { + let hash_byte = u8::try_from(height % 251).expect("height modulo 251 fits in u8"); + Frontier::new(block::Height(height), block::Hash([hash_byte; 32])) +} + +fn test_frontier_update( + finalized: u32, + verified_body: u32, + best_header: u32, + change: FrontierChange, +) -> FrontierUpdate { + FrontierUpdate { + frontier: ChainFrontier { + finalized: test_frontier(finalized), + verified_body: test_frontier(verified_body), + best_header: test_frontier(best_header), + }, + change, + } +} + +fn exchange_block_sync_startup( + initial: FrontierUpdate, + config: ZakuraBlockSyncConfig, +) -> (ZakuraSyncExchange, BlockSyncStartup) { + let exchange = ZakuraSyncExchange::new(initial, ZakuraTrace::noop()); + let frontier = initial.frontier; + let startup = BlockSyncStartup::new_with_exchange( + BlockSyncFrontiers { + finalized_height: frontier.finalized.height, + verified_block_tip: frontier.verified_body.height, + verified_block_hash: frontier.verified_body.hash, + }, + (frontier.best_header.height, frontier.best_header.hash), + exchange.subscribe_frontier(), + config, + ); + + (exchange, startup) +} + +fn round_trip(message: BlockSyncMessage) { + let encoded = message.encode().expect("message encodes"); + let decoded = BlockSyncMessage::decode(&encoded).expect("message decodes"); + + assert_eq!(decoded, message); +} + +async fn next_event(events: &mut mpsc::Receiver) -> BlockSyncEvent { + tokio::time::timeout(Duration::from_secs(1), events.recv()) + .await + .expect("block-sync event should arrive") + .expect("block-sync event channel should stay open") +} + +async fn next_action(actions: &mut mpsc::Receiver) -> BlockSyncAction { + tokio::time::timeout(Duration::from_secs(1), actions.recv()) + .await + .expect("block-sync action should arrive") + .expect("block-sync action channel should stay open") +} + +async fn wait_for_query_needed_blocks( + actions: &mut mpsc::Receiver, + verified_block_tip: block::Height, + best_header_tip: block::Height, +) { + loop { + match next_action(actions).await { + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: actual_verified, + best_header_tip: actual_best, + } if actual_verified == verified_block_tip && actual_best == best_header_tip => return, + BlockSyncAction::QueryNeededBlocks { .. } | BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before target QueryNeededBlocks: {action:?}"), + } + } +} + +async fn wait_for_getblocks( + actions: &mut mpsc::Receiver, +) -> (ZakuraPeerId, block::Height, u32) { + loop { + match next_action(actions).await { + BlockSyncAction::SendMessage { + peer, + msg: + BlockSyncMessage::GetBlocks { + start_height, + count, + }, + } => return (peer, start_height, count), + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::QueryNeededBlocks { .. } => {} + BlockSyncAction::Misbehavior { + reason: BlockSyncMisbehavior::UnsolicitedBlock, + .. + } => {} + action => panic!("unexpected action before GetBlocks: {action:?}"), + } + } +} + +async fn wait_for_connect_status(actions: &mut mpsc::Receiver) -> ZakuraPeerId { + loop { + match next_action(actions).await { + BlockSyncAction::SendMessage { + peer, + msg: BlockSyncMessage::Status(_), + } => return peer, + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::QueryNeededBlocks { .. } => {} + BlockSyncAction::Misbehavior { .. } => {} + action => panic!("unexpected action before connect status: {action:?}"), + } + } +} + +async fn next_outbound_message(outbound: &mut FramedRecv) -> BlockSyncMessage { + let frame = tokio::time::timeout(Duration::from_secs(1), outbound.recv()) + .await + .expect("outbound frame arrives") + .expect("outbound channel is live"); + BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") +} + +async fn wait_for_outbound_block(outbound: &mut FramedRecv) -> Arc { + loop { + match next_outbound_message(outbound).await { + BlockSyncMessage::Block(block) => return block, + BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {} + msg => panic!("unexpected outbound message before block: {msg:?}"), + } + } +} + +async fn wait_for_outbound_blocks_done(outbound: &mut FramedRecv) -> (block::Height, u32) { + loop { + match next_outbound_message(outbound).await { + BlockSyncMessage::BlocksDone { + start_height, + returned, + } => return (start_height, returned), + BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {} + msg => panic!("unexpected outbound message before BlocksDone: {msg:?}"), + } + } +} + +async fn wait_for_outbound_range_unavailable(outbound: &mut FramedRecv) -> (block::Height, u32) { + loop { + match next_outbound_message(outbound).await { + BlockSyncMessage::RangeUnavailable { + start_height, + count, + } => return (start_height, count), + BlockSyncMessage::Status(_) | BlockSyncMessage::GetBlocks { .. } => {} + msg => panic!("unexpected outbound message before RangeUnavailable: {msg:?}"), + } + } +} + +async fn drain_parent_first_actions( + actions: &mut mpsc::Receiver, + verified_tip: &mut block::Height, + expected_new_fork: Option<&[Arc]>, +) { + while let Ok(Some(action)) = + tokio::time::timeout(Duration::from_millis(25), actions.recv()).await + { + match action { + BlockSyncAction::SubmitBlock { block, .. } => { + let height = block + .coinbase_height() + .expect("submitted test block has height"); + assert_eq!( + Some(height), + next_height(*verified_tip), + "block sync must submit only the contiguous parent-first prefix" + ); + if let Some(new_fork) = expected_new_fork { + let expected_hash = match height.0 { + 2 => new_fork[1].hash(), + 3 => new_fork[2].hash(), + _ => panic!("unexpected post-reset submitted height: {height:?}"), + }; + assert_eq!( + block.hash(), + expected_hash, + "post-reset submissions must follow the re-derived fork" + ); + } + *verified_tip = height; + } + BlockSyncAction::Misbehavior { + reason: BlockSyncMisbehavior::InvalidBlock | BlockSyncMisbehavior::UnsolicitedBlock, + .. + } => {} + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action while draining body responses: {action:?}"), + } + } +} + +fn peer_state(byte: u8) -> (ZakuraPeerId, PeerBlockState) { + let peer = peer(byte); + let (_inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, _outbound_rx) = framed_channel(4); + let session = PeerStreamSession::new( + peer.clone(), + ZAKURA_STREAM_BLOCK_SYNC, + inbound_rx, + outbound_tx, + CancellationToken::new(), + ); + let mut state = PeerBlockState::new( + BlockSyncPeerSession::new(&session, ServicePeerDirection::Outbound), + &ZakuraBlockSyncConfig::default(), + ); + state.received_status = true; + state.servable_high = block::Height(100); + (peer, state) +} + +async fn connect_peer_with_status( + service: &BlockSyncService, + actions: &mut mpsc::Receiver, + byte: u8, + servable_high: block::Height, + tip_hash: block::Hash, + max_inflight_requests: u16, + max_response_bytes: u32, +) -> (ZakuraPeerId, FramedSend, FramedRecv) { + connect_peer_with_status_message( + service, + actions, + byte, + BlockSyncStatus { + servable_low: block::Height(1), + servable_high, + tip_hash, + max_blocks_per_response: 16, + max_inflight_requests, + max_response_bytes, + }, + ) + .await +} + +async fn connect_peer_with_status_message( + service: &BlockSyncService, + actions: &mut mpsc::Receiver, + byte: u8, + status: BlockSyncStatus, +) -> (ZakuraPeerId, FramedSend, FramedRecv) { + let peer = peer(byte); + let (inbound_tx, inbound_rx) = framed_channel(16); + let (outbound_tx, outbound_rx) = framed_channel(16); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + assert_eq!(wait_for_connect_status(actions).await, peer); + inbound_tx + .send( + BlockSyncMessage::Status(status) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + (peer, inbound_tx, outbound_rx) +} + +fn needed(height: u32, size: BlockSizeEstimate) -> NeededBlock { + NeededBlock { + height: block::Height(height), + hash: block::Hash([height as u8; 32]), + size, + } +} + +fn block_meta(block: &Arc) -> BlockSyncBlockMeta { + BlockSyncBlockMeta { + height: block.coinbase_height().expect("test block has height"), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_size(block)), + } +} + +#[test] +fn block_sync_near_tip_pause_config_defaults_and_round_trips() { + let default = ZakuraBlockSyncConfig::default(); + assert_eq!(default.near_tip_body_download_pause_blocks, 2); + + let encoded = toml::to_string(&default).expect("block-sync config serializes"); + let decoded: ZakuraBlockSyncConfig = + toml::from_str(&encoded).expect("block-sync config deserializes"); + assert_eq!(decoded, default); + + let config: crate::Config = toml::from_str( + r#" + [zakura.block_sync] + near_tip_body_download_pause_blocks = 7 + "#, + ) + .expect("nested Zakura block-sync config deserializes"); + assert_eq!( + config.zakura.block_sync.near_tip_body_download_pause_blocks, + 7 + ); +} + +#[test] +fn codec_round_trips_every_message_variant() { + round_trip(BlockSyncMessage::Status(status())); + round_trip(BlockSyncMessage::GetBlocks { + start_height: block::Height(10), + count: 3, + }); + round_trip(BlockSyncMessage::Block(mainnet_block( + &BLOCK_MAINNET_1_BYTES, + ))); + round_trip(BlockSyncMessage::BlocksDone { + start_height: block::Height(10), + returned: 3, + }); + round_trip(BlockSyncMessage::RangeUnavailable { + start_height: block::Height(10), + count: 3, + }); +} + +#[test] +fn codec_round_trips_block_near_max_block_bytes() { + let block = Arc::new(zebra_chain::block::tests::generate::large_multi_transaction_block()); + let serialized_len = block + .zcash_serialize_to_vec() + .expect("large test block serializes") + .len(); + let max_block_bytes = + usize::try_from(block::MAX_BLOCK_BYTES).expect("max block size fits in usize"); + + assert!( + serialized_len <= max_block_bytes && serialized_len > max_block_bytes - 1000, + "test block should be close to the consensus cap, got {serialized_len}" + ); + round_trip(BlockSyncMessage::Block(block)); +} + +#[test] +fn codec_rejects_malformed_discriminator_and_truncated_payload() { + assert!(matches!( + BlockSyncMessage::decode(&[99]), + Err(BlockSyncWireError::UnknownMessageType(99)) + )); + + assert!(matches!( + BlockSyncMessage::decode(&[MSG_BS_GET_BLOCKS, 1, 0]), + Err(BlockSyncWireError::Io(_)) + )); +} + +#[test] +fn codec_classifies_payloads_above_old_raw_stream6_cap() { + let old_max_bs_message_bytes = + usize::try_from(block::MAX_BLOCK_BYTES).expect("max block bytes fits in usize") + 1; + let payload = vec![99; old_max_bs_message_bytes + 1]; + + assert!(payload.len() <= MAX_BS_MESSAGE_BYTES); + assert!(matches!( + BlockSyncMessage::decode(&payload), + Err(BlockSyncWireError::UnknownMessageType(99)) + )); +} + +#[test] +fn codec_rejects_oversized_frame_and_oversized_block() { + let oversized_payload = vec![0; MAX_BS_MESSAGE_BYTES + 1]; + assert!(matches!( + BlockSyncMessage::decode(&oversized_payload), + Err(BlockSyncWireError::OversizedPayload { .. }) + )); + + let oversized_block = + Arc::new(zebra_chain::block::tests::generate::oversized_multi_transaction_block()); + assert!(matches!( + BlockSyncMessage::Block(oversized_block).encode(), + Err(BlockSyncWireError::OversizedBlock { .. }) + | Err(BlockSyncWireError::OversizedPayload { .. }) + )); +} + +#[test] +fn codec_rejects_count_and_returned_over_cap() { + let over_cap = MAX_BS_BLOCKS_PER_REQUEST + 1; + + assert!(matches!( + BlockSyncMessage::BlocksDone { + start_height: block::Height(1), + returned: 0, + } + .encode(), + Err(BlockSyncWireError::ZeroBlockCount) + )); + + let mut zero_count_get_blocks = vec![MSG_BS_GET_BLOCKS]; + zero_count_get_blocks.extend_from_slice(&1u32.to_le_bytes()); + zero_count_get_blocks.extend_from_slice(&0u32.to_le_bytes()); + assert!(matches!( + BlockSyncMessage::decode(&zero_count_get_blocks), + Err(BlockSyncWireError::ZeroBlockCount) + )); + + let mut zero_count_range_unavailable = vec![MSG_BS_RANGE_UNAVAILABLE]; + zero_count_range_unavailable.extend_from_slice(&1u32.to_le_bytes()); + zero_count_range_unavailable.extend_from_slice(&0u32.to_le_bytes()); + assert!(matches!( + BlockSyncMessage::decode(&zero_count_range_unavailable), + Err(BlockSyncWireError::ZeroBlockCount) + )); + + assert!(matches!( + BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: over_cap, + } + .encode(), + Err(BlockSyncWireError::BlockCountLimit { .. }) + )); + + assert!(matches!( + BlockSyncMessage::BlocksDone { + start_height: block::Height(1), + returned: over_cap, + } + .encode(), + Err(BlockSyncWireError::BlockCountLimit { .. }) + )); + + assert!(matches!( + BlockSyncMessage::RangeUnavailable { + start_height: block::Height(1), + count: over_cap, + } + .encode(), + Err(BlockSyncWireError::BlockCountLimit { .. }) + )); +} + +#[test] +fn frame_decode_rejects_mismatched_unknown_flags_and_trailing_payload() { + let frame = Frame { + message_type: u16::from(MSG_BS_GET_BLOCKS), + flags: 1, + payload: BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 1, + } + .encode() + .expect("message encodes"), + }; + assert!(matches!( + BlockSyncMessage::decode_frame(frame), + Err(BlockSyncWireError::UnsupportedFlags(1)) + )); + + let mut payload = BlockSyncMessage::Status(status()) + .encode() + .expect("message encodes"); + payload.push(0); + assert!(matches!( + BlockSyncMessage::decode(&payload), + Err(BlockSyncWireError::TrailingBytes) + )); + + let frame = Frame { + message_type: u16::from(MSG_BS_BLOCK), + flags: 0, + payload: BlockSyncMessage::Status(status()) + .encode() + .expect("message encodes"), + }; + assert!(matches!( + BlockSyncMessage::decode_frame(frame), + Err(BlockSyncWireError::MismatchedFrameMessageType { .. }) + )); +} + +#[test] +fn status_decode_clamps_peer_capacity_advertisements() { + let mut payload = Vec::new(); + payload.push(MSG_BS_STATUS); + payload.extend_from_slice(&block::Height(1).0.to_le_bytes()); + payload.extend_from_slice(&block::Height(2).0.to_le_bytes()); + block::Hash([9; 32]) + .zcash_serialize(&mut payload) + .expect("hash serializes"); + payload.extend_from_slice(&u32::MAX.to_le_bytes()); + payload.extend_from_slice(&u16::MAX.to_le_bytes()); + payload.extend_from_slice(&u32::MAX.to_le_bytes()); + + let BlockSyncMessage::Status(status) = + BlockSyncMessage::decode(&payload).expect("status decodes") + else { + panic!("expected status message"); + }; + + assert_eq!(status.max_blocks_per_response, MAX_BS_BLOCKS_PER_REQUEST); + assert_eq!(status.max_inflight_requests, DEFAULT_BS_MAX_INFLIGHT); + assert_eq!(status.max_response_bytes, MAX_BS_RESPONSE_BYTES); +} + +#[test] +fn scheduler_assigns_needed_ranges_with_fanout_slots_and_dedup() { + let mut scheduler = BlockRangeScheduler::new(2); + scheduler.refresh_needed(vec![ + needed(1, BlockSizeEstimate::Advertised(10_000)), + needed(2, BlockSizeEstimate::Advertised(10_000)), + needed(3, BlockSizeEstimate::Advertised(10_000)), + ]); + let mut budget = ByteBudget::new(1_000_000); + let (peer1, state1) = peer_state(31); + let (peer2, state2) = peer_state(32); + let (peer3, state3) = peer_state(33); + + let first = scheduler + .next_for_peer(&peer1, &state1, &mut budget, u64::MAX) + .expect("first peer gets the range"); + assert_eq!(first.start_height, block::Height(1)); + assert_eq!(first.count, 3); + + let second = scheduler + .next_for_peer(&peer2, &state2, &mut budget, u64::MAX) + .expect("fanout allows a second peer"); + assert_eq!(second.start_height, block::Height(1)); + assert_eq!(second.count, 3); + + assert!( + scheduler + .next_for_peer(&peer1, &state1, &mut budget, u64::MAX) + .is_none(), + "same peer must not receive overlapping duplicate assignment" + ); + assert!( + scheduler + .next_for_peer(&peer3, &state3, &mut budget, u64::MAX) + .is_none(), + "fanout cap must deduplicate the covered range" + ); +} + +#[test] +fn scheduler_byte_budget_sizing_shrinks_or_defers_requests() { + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(10, BlockSizeEstimate::Advertised(100)), + needed(11, BlockSizeEstimate::Advertised(100)), + needed(12, BlockSizeEstimate::Advertised(100)), + ]); + let (peer, mut state) = peer_state(34); + state.max_response_bytes = 180; + state.max_blocks_per_response = 10; + let mut budget = ByteBudget::new(1_000); + + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("one block fits the peer response-byte cap"); + assert_eq!(request.start_height, block::Height(10)); + assert_eq!(request.count, 1); + assert_eq!(budget.reserved(), 100); + + scheduler.complete(&request, &mut budget); + assert_eq!(budget.reserved(), 0); + + let mut small_budget = ByteBudget::new(50); + assert!( + scheduler + .next_for_peer(&peer, &state, &mut small_budget, u64::MAX) + .is_none(), + "a first block that does not fit is deferred" + ); + assert_eq!(small_budget.reserved(), 0); +} + +#[test] +fn scheduler_per_peer_byte_cap_limits_request_below_global_budget() { + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(1, BlockSizeEstimate::Advertised(100)), + needed(2, BlockSizeEstimate::Advertised(100)), + needed(3, BlockSizeEstimate::Advertised(100)), + ]); + let (peer, mut state) = peer_state(50); + state.max_response_bytes = 10_000; // peer response cap not binding + state.max_blocks_per_response = 10; // count cap not binding + let mut budget = ByteBudget::new(1_000_000); // global budget ample + + // A 150-byte per-peer cap admits only the first 100-byte block, even though + // the count cap, peer response cap, and global budget all have ample room. + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, 150) + .expect("first block fits the per-peer byte cap"); + assert_eq!(request.count, 1); + assert_eq!(request.estimated_bytes, 100); + assert_eq!(budget.reserved(), 100); + + // A fresh peer with a 1_000-byte cap batches the whole three-block range. + let mut scheduler2 = BlockRangeScheduler::new(1); + scheduler2.set_estimator_for_tests(750, 1); + scheduler2.refresh_needed(vec![ + needed(1, BlockSizeEstimate::Advertised(100)), + needed(2, BlockSizeEstimate::Advertised(100)), + needed(3, BlockSizeEstimate::Advertised(100)), + ]); + let (peer2, mut state2) = peer_state(51); + state2.max_response_bytes = 10_000; + state2.max_blocks_per_response = 10; + let mut budget2 = ByteBudget::new(1_000_000); + let request2 = scheduler2 + .next_for_peer(&peer2, &state2, &mut budget2, 1_000) + .expect("higher per-peer cap admits the whole range"); + assert_eq!(request2.count, 3); + assert_eq!(request2.estimated_bytes, 300); +} + +#[test] +fn block_sync_per_peer_byte_cap_shares_budget_and_floors_at_one_response() { + // Ample budget: each of `expected_peers` gets an even share of the budget. + let config = ZakuraBlockSyncConfig::default(); + assert_eq!( + config.per_peer_byte_cap(), + config.max_inflight_block_bytes / config.expected_peers as u64, + ); + + // Tiny budget: the per-peer share would starve peers, so the cap floors at + // one advertised response so a peer can always make progress. + let tiny = ZakuraBlockSyncConfig { + max_inflight_block_bytes: 4_096, + ..ZakuraBlockSyncConfig::default() + }; + assert_eq!( + tiny.per_peer_byte_cap(), + u64::from(tiny.advertised_max_response_bytes()), + ); + + // `expected_peers == 0` disables per-peer byte fairness entirely. + let disabled = ZakuraBlockSyncConfig { + expected_peers: 0, + ..ZakuraBlockSyncConfig::default() + }; + assert_eq!(disabled.per_peer_byte_cap(), u64::MAX); +} + +#[tokio::test] +async fn reactor_fill_loop_saturates_multiple_slots_in_one_pass() { + let config = immediate_body_download_config(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + + // Peer serves heights 1..=4 and accepts four concurrent single-block requests. + let (peer_id, _inbound, _outbound) = connect_peer_with_status_message( + &service, + &mut actions, + 41, + BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(4), + tip_hash: block::Hash([4; 32]), + max_blocks_per_response: 1, + max_inflight_requests: 4, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }, + ) + .await; + + tip_tx + .send((block::Height(4), block::Hash([4; 32]))) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(1), + hash: block::Hash([1; 32]), + size: BlockSizeEstimate::Advertised(1_000), + }, + BlockSyncBlockMeta { + height: block::Height(2), + hash: block::Hash([2; 32]), + size: BlockSizeEstimate::Advertised(1_000), + }, + BlockSyncBlockMeta { + height: block::Height(3), + hash: block::Hash([3; 32]), + size: BlockSizeEstimate::Advertised(1_000), + }, + BlockSyncBlockMeta { + height: block::Height(4), + hash: block::Hash([4; 32]), + size: BlockSizeEstimate::Advertised(1_000), + }, + ])) + .await + .expect("needed metadata queues"); + + // The fill-loop opens all four slots from the single NeededBlocks event. + // Pre-fill-loop scheduling issued only one GetBlocks per scheduling event, + // so this would time out on the second request. + let mut heights = Vec::new(); + for _ in 0..4 { + let (peer, start_height, count) = wait_for_getblocks(&mut actions).await; + assert_eq!(peer, peer_id); + assert_eq!(count, 1); + heights.push(start_height.0); + } + heights.sort_unstable(); + assert_eq!(heights, vec![1, 2, 3, 4]); + + reactor_task.abort(); +} + +#[test] +fn scheduler_partial_requests_clear_the_issued_assignment_key() { + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(10, BlockSizeEstimate::Advertised(100)), + needed(11, BlockSizeEstimate::Advertised(100)), + needed(12, BlockSizeEstimate::Advertised(100)), + ]); + let (peer, mut state) = peer_state(38); + state.max_response_bytes = 200; + state.max_blocks_per_response = 10; + let mut budget = ByteBudget::new(1_000); + + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("response-byte cap drains a prefix of the queued range"); + assert_eq!(request.start_height, block::Height(10)); + assert_eq!(request.count, 2); + assert_eq!(scheduler.assigned_range_count(), 1); + + scheduler.complete(&request, &mut budget); + assert_eq!( + scheduler.assigned_range_count(), + 0, + "completing a partial request must clear the same range key it assigned" + ); +} + +#[test] +fn scheduler_drops_verified_prefix_from_queued_ranges() { + let (peer1, state1) = peer_state(39); + let (peer2, state2) = peer_state(40); + let mut scheduler = BlockRangeScheduler::new(2); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(10, BlockSizeEstimate::Advertised(100)), + needed(11, BlockSizeEstimate::Advertised(100)), + needed(12, BlockSizeEstimate::Advertised(100)), + ]); + let mut budget = ByteBudget::new(1_000); + + let first = scheduler + .next_for_peer(&peer1, &state1, &mut budget, u64::MAX) + .expect("first fanout assignment queues"); + assert_eq!(first.start_height, block::Height(10)); + + scheduler.drop_through(block::Height(10)); + assert!( + scheduler + .next_for_peer(&peer1, &state1, &mut budget, u64::MAX) + .is_none(), + "verified-prefix trimming must not let the same peer bypass its overlapping assignment", + ); + + let second = scheduler + .next_for_peer(&peer2, &state2, &mut budget, u64::MAX) + .expect("queued suffix remains requestable by another fanout peer"); + assert_eq!(second.start_height, block::Height(11)); + assert_eq!(second.count, 2); +} + +#[test] +fn scheduler_drops_covered_prefix_from_partially_queued_range() { + let mut scheduler = BlockRangeScheduler::new(2); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(10, BlockSizeEstimate::Advertised(100)), + needed(11, BlockSizeEstimate::Advertised(100)), + needed(12, BlockSizeEstimate::Advertised(100)), + ]); + let (peer1, state1) = peer_state(40); + let (peer2, state2) = peer_state(41); + let mut budget = ByteBudget::new(1_000); + + let first = scheduler + .next_for_peer(&peer1, &state1, &mut budget, u64::MAX) + .expect("first fanout assignment leaves the queued range for another peer"); + assert_eq!(first.start_height, block::Height(10)); + assert_eq!(first.count, 3); + + scheduler.mark_height_covered(block::Height(10)); + + assert!( + scheduler + .next_for_peer(&peer1, &state1, &mut budget, u64::MAX) + .is_none(), + "trimming a covered prefix must not let the same peer bypass its overlapping assignment", + ); + + let second = scheduler + .next_for_peer(&peer2, &state2, &mut budget, u64::MAX) + .expect("uncovered suffix remains requestable"); + assert_eq!( + second.start_height, + block::Height(11), + "covered prefix must not remain as the next request anchor", + ); + assert_eq!(second.count, 2); +} + +#[test] +fn scheduler_splits_queued_range_around_covered_heights() { + let (peer, mut state) = peer_state(42); + state.max_blocks_per_response = 10; + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(10, BlockSizeEstimate::Advertised(100)), + needed(11, BlockSizeEstimate::Advertised(100)), + needed(12, BlockSizeEstimate::Advertised(100)), + needed(13, BlockSizeEstimate::Advertised(100)), + needed(14, BlockSizeEstimate::Advertised(100)), + ]); + let mut budget = ByteBudget::new(1_000); + + scheduler.mark_height_covered(block::Height(12)); + + let first = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("uncovered prefix remains requestable"); + assert_eq!(first.start_height, block::Height(10)); + assert_eq!(first.count, 2); + + let second = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("uncovered suffix remains requestable"); + assert_eq!(second.start_height, block::Height(13)); + assert_eq!(second.count, 2); +} + +#[test] +fn scheduler_retries_only_uncovered_suffix() { + let (peer, state) = peer_state(43); + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(20, BlockSizeEstimate::Advertised(100)), + needed(21, BlockSizeEstimate::Advertised(100)), + needed(22, BlockSizeEstimate::Advertised(100)), + ]); + let mut budget = ByteBudget::new(1_000); + + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("range fits"); + assert_eq!(request.start_height, block::Height(20)); + assert_eq!(request.count, 3); + + scheduler.mark_height_covered(block::Height(20)); + scheduler.timeout(request, &mut budget); + + let retry = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("uncovered retry suffix remains requestable"); + assert_eq!( + retry.start_height, + block::Height(21), + "covered retry prefix must not be requested again", + ); + assert_eq!(retry.count, 2); +} + +#[test] +fn scheduler_releases_budget_on_completion_timeout_and_cancel() { + let (peer, state) = peer_state(35); + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![needed(20, BlockSizeEstimate::Advertised(1_000))]); + let mut budget = ByteBudget::new(10_000); + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("range fits"); + assert_eq!(budget.reserved(), 1_000); + scheduler.complete(&request, &mut budget); + assert_eq!(budget.reserved(), 0); + + scheduler.refresh_needed(vec![needed(21, BlockSizeEstimate::Advertised(2_000))]); + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("range fits"); + scheduler.timeout(request, &mut budget); + assert_eq!(budget.reserved(), 0); + + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("retried range fits"); + assert_eq!(budget.reserved(), 2_000); + assert_eq!(request.count, 1); + scheduler.release_cancelled(&mut budget); + assert_eq!(budget.reserved(), 0); +} + +#[test] +fn scheduler_drops_queued_ranges_whose_anchor_left_current_header_spine() { + let (peer, state) = peer_state(37); + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1); + scheduler.refresh_needed(vec![ + needed(40, BlockSizeEstimate::Advertised(1_000)), + needed(41, BlockSizeEstimate::Advertised(1_000)), + ]); + + let current = HashMap::from([ + (block::Height(40), block::Hash([90; 32])), + (block::Height(41), block::Hash([91; 32])), + ]); + scheduler.retain_matching_needed(¤t); + + let mut budget = ByteBudget::new(10_000); + assert!( + scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .is_none(), + "stale queued anchors must not survive a re-derived needed set" + ); + assert_eq!(budget.reserved(), 0); +} + +#[test] +fn scheduler_uses_ewma_for_unknown_and_confirmed_size_values() { + let (peer, mut state) = peer_state(36); + state.max_response_bytes = 100_000; + let mut scheduler = BlockRangeScheduler::new(1); + scheduler.set_estimator_for_tests(750, 1_000); + scheduler.refresh_needed(vec![ + needed(30, BlockSizeEstimate::Unknown), + needed(31, BlockSizeEstimate::Advertised(500)), + needed(32, BlockSizeEstimate::Confirmed(50_000)), + ]); + let mut budget = ByteBudget::new(100_000); + + let request = scheduler + .next_for_peer(&peer, &state, &mut budget, u64::MAX) + .expect("range fits"); + assert_eq!(request.count, 3); + assert_eq!(request.estimated_bytes, 52_000); +} + +#[test] +fn reorder_drains_only_contiguous_prefix_without_releasing_budget() { + let mut reorder = ReorderBuffer::new(); + let mut budget = ByteBudget::new(10_000); + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + + assert_eq!( + reorder.insert(block::Height(3), block.clone(), 300, &mut budget), + ReorderInsertResult::Inserted + ); + assert!(reorder.drain_contiguous_prefix(block::Height(0)).is_empty()); + assert_eq!(reorder.buffered_bytes(), 300); + assert_eq!(budget.reserved(), 300); + + assert_eq!( + reorder.insert(block::Height(1), block.clone(), 100, &mut budget), + ReorderInsertResult::Inserted + ); + let released = reorder.drain_contiguous_prefix(block::Height(0)); + assert_eq!( + released + .iter() + .map(|(height, _, bytes)| (*height, *bytes)) + .collect::>(), + vec![(block::Height(1), 100)] + ); + assert_eq!(reorder.buffered_bytes(), 300); + assert_eq!(budget.reserved(), 400); + budget.release(100); + + assert_eq!( + reorder.insert(block::Height(2), block.clone(), 200, &mut budget), + ReorderInsertResult::Inserted + ); + let released = reorder.drain_contiguous_prefix(block::Height(1)); + assert_eq!( + released + .iter() + .map(|(height, _, bytes)| (*height, *bytes)) + .collect::>(), + vec![(block::Height(2), 200), (block::Height(3), 300)] + ); + assert_eq!(budget.reserved(), 500); + budget.release(500); + + assert_eq!( + reorder.insert(block::Height(2), block.clone(), 200, &mut budget), + ReorderInsertResult::Inserted + ); + assert_eq!( + reorder.insert(block::Height(3), block, 300, &mut budget), + ReorderInsertResult::Inserted + ); + reorder.drop_from(block::Height(3), &mut budget); + assert_eq!(reorder.buffered_bytes(), 200); + assert_eq!(budget.reserved(), 200); + reorder.drop_through(block::Height(2), &mut budget); + assert_eq!(reorder.buffered_bytes(), 0); + assert_eq!(budget.reserved(), 0); + assert_eq!( + reorder.insert( + block::Height(3), + mainnet_block(&BLOCK_MAINNET_1_BYTES), + 300, + &mut budget + ), + ReorderInsertResult::Inserted + ); + reorder.clear(&mut budget); + assert_eq!(reorder.buffered_bytes(), 0); + assert_eq!(budget.reserved(), 0); +} + +#[test] +fn reorder_fuzzes_arrival_order_as_parent_first() { + let orders = [ + [1, 2, 3, 4], + [4, 3, 2, 1], + [2, 4, 1, 3], + [3, 1, 4, 2], + [2, 1, 4, 3], + ]; + + for order in orders { + let mut reorder = ReorderBuffer::new(); + let mut budget = ByteBudget::new(10_000); + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let mut tip = block::Height(0); + let mut released_all = Vec::new(); + + for height in order { + assert_eq!( + reorder.insert(block::Height(height), block.clone(), 100, &mut budget), + ReorderInsertResult::Inserted + ); + for (released, _, bytes) in reorder.drain_contiguous_prefix(tip) { + assert_eq!(released, block::Height(tip.0 + 1)); + tip = released; + released_all.push(released); + budget.release(bytes); + } + } + + assert_eq!( + released_all, + vec![ + block::Height(1), + block::Height(2), + block::Height(3), + block::Height(4) + ] + ); + assert_eq!(budget.reserved(), 0); + } +} + +#[test] +fn block_sync_stream_declares_kind_capability_version_and_frame_cap() { + let stream = block_sync_streams() + .first() + .copied() + .expect("block sync declares one stream"); + + assert_eq!(stream.kind, ZAKURA_STREAM_BLOCK_SYNC); + assert_eq!(stream.version, ZAKURA_BLOCK_SYNC_STREAM_VERSION); + assert_eq!(stream.capability, ZAKURA_CAP_BLOCK_SYNC); + assert_eq!(stream.mode, StreamMode::Ordered); + assert_eq!(stream.frame_cap, MAX_BS_FRAME_BYTES); +} + +#[tokio::test] +async fn service_registry_routes_block_sync_by_exact_capability_and_version() { + let service = Arc::new(BlockSyncService::new(ZakuraBlockSyncConfig::default())); + let registry = + ServiceRegistry::new(vec![service]).expect("block-sync service declares unique kind"); + let peer = peer(1); + + assert_eq!( + registry.capability_for_stream(ZAKURA_STREAM_BLOCK_SYNC, ZAKURA_BLOCK_SYNC_STREAM_VERSION), + Some(ZAKURA_CAP_BLOCK_SYNC) + ); + assert!(registry + .capability_for_stream( + ZAKURA_STREAM_BLOCK_SYNC, + ZAKURA_BLOCK_SYNC_STREAM_VERSION + 1 + ) + .is_none()); + assert_eq!( + registry + .ordered_streams_for_negotiated(ZAKURA_CAP_BLOCK_SYNC) + .iter() + .map(|stream| stream.kind) + .collect::>(), + vec![ZAKURA_STREAM_BLOCK_SYNC] + ); + assert!(registry.ordered_streams_for_negotiated(0).is_empty()); + assert!(registry.wants_ordered_stream( + ZAKURA_STREAM_BLOCK_SYNC, + ZAKURA_CAP_BLOCK_SYNC, + &peer, + ServicePeerDirection::Inbound, + )); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn inert_reactor_parks_after_header_tip_watch_closes() { + let _service = BlockSyncService::new(ZakuraBlockSyncConfig::default()); + + let elapsed = tokio::time::timeout(Duration::from_secs(1), future::pending::<()>()).await; + + assert!( + elapsed.is_err(), + "paused-time timeout only elapses if the inert reactor has no always-ready branch" + ); +} + +#[cfg(debug_assertions)] +#[test] +#[should_panic(expected = "state-backed block sync must have exactly one frontier source")] +fn state_backed_reactor_panics_with_two_frontier_sources() { + let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let (_frontier_tx, frontier_rx) = + watch::channel(test_frontier_update(0, 0, 0, FrontierChange::Snapshot)); + let mut startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + ZakuraBlockSyncConfig::default(), + ); + startup.frontier_updates = Some(frontier_rx); + + let (_handle, _actions, _task) = spawn_block_sync_reactor(startup); +} + +#[tokio::test] +async fn add_peer_emits_events_and_round_trips_status_over_framed_path() { + let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default()); + let peer = peer(2); + let cancel_token = CancellationToken::new(); + let (inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, mut outbound_rx) = framed_channel(4); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + streams, + cancel_token, + )); + + let session = match next_event(&mut events).await { + BlockSyncEvent::PeerConnected(session) => session, + event => panic!("expected PeerConnected, got {event:?}"), + }; + assert_eq!(session.peer_id(), &peer); + assert_eq!(service.peer_count(), 1); + + service + .send_action(BlockSyncAction::SendMessage { + peer: peer.clone(), + msg: BlockSyncMessage::Status(status()), + }) + .await + .expect("action queues to source"); + let frame = tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv()) + .await + .expect("action status frame should be sent") + .expect("outbound frame receiver stays open"); + assert_eq!( + BlockSyncMessage::decode_frame(frame).expect("status frame decodes"), + BlockSyncMessage::Status(status()) + ); + + inbound_tx + .send( + BlockSyncMessage::Status(status()) + .encode_frame() + .expect("status frame encodes"), + ) + .await + .expect("inbound status queues"); + assert!(matches!( + next_event(&mut events).await, + BlockSyncEvent::WireMessage { + msg: BlockSyncMessage::Status(_), + .. + } + )); + + service.remove_peer(&peer); + assert_eq!(service.peer_count(), 0); + assert!(session.cancel_token().is_cancelled()); +} + +#[tokio::test] +async fn lifecycle_events_bypass_full_bounded_wire_queue() { + let mut config = ZakuraBlockSyncConfig::default(); + config.peer_limits.inbound_queue_depth = 1; + let (events, _event_rx) = mpsc::channel(config.peer_limits.inbound_queue_depth); + events + .try_send(BlockSyncEvent::WireMessage { + peer: peer(90), + msg: BlockSyncMessage::Status(status()), + }) + .expect("test fills bounded wire queue"); + let (lifecycle, mut lifecycle_rx) = mpsc::unbounded_channel(); + let (_peers_tx, peers) = watch::channel(ServicePeerSnapshot::new(0, 0, config.peer_limits)); + let (_status_tx, status) = watch::channel(config.initial_status()); + let (_candidates_tx, candidates) = watch::channel(ZakuraBlockSyncCandidateState::default()); + let handle = BlockSyncHandle { + events, + lifecycle, + peers, + status, + candidates, + }; + let service = BlockSyncService::new_with_handle_for_test(config, handle); + let peer = peer(91); + let (inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, _outbound_rx) = framed_channel(4); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + let _inbound_tx = inbound_tx; + + assert!(matches!( + tokio::time::timeout(Duration::from_secs(1), lifecycle_rx.recv()) + .await + .expect("lifecycle event arrives") + .expect("lifecycle channel stays open"), + BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer + )); + + service.remove_peer(&peer); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(1), lifecycle_rx.recv()) + .await + .expect("lifecycle event arrives") + .expect("lifecycle channel stays open"), + BlockSyncEvent::PeerDisconnected(disconnected) if disconnected == peer + )); +} + +#[tokio::test] +async fn add_peer_decode_failure_emits_wire_decode_failed() { + let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default()); + let peer = peer(3); + let (inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, _outbound_rx) = framed_channel(4); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + let cancel_token = CancellationToken::new(); + + service.add_peer(Peer::new( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + streams, + cancel_token.clone(), + )); + let _ = next_event(&mut events).await; + + inbound_tx + .send(Frame { + message_type: u16::from(MSG_BS_STATUS), + flags: 0, + payload: Vec::new(), + }) + .await + .expect("malformed inbound frame queues"); + + assert!(matches!( + next_event(&mut events).await, + BlockSyncEvent::WireDecodeFailed { .. } + )); + assert!(cancel_token.is_cancelled()); +} + +#[tokio::test] +async fn registry_add_peer_requires_negotiated_block_sync_capability() { + let (service, mut events) = BlockSyncService::new_for_test(ZakuraBlockSyncConfig::default()); + let registry = ServiceRegistry::new(vec![Arc::new(service)]) + .expect("block-sync service declares unique kind"); + let peer = peer(4); + let (inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, _outbound_rx) = framed_channel(4); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + registry.add_peer(Peer::new(peer, None, 0, streams, CancellationToken::new())); + drop(inbound_tx); + + assert!( + tokio::time::timeout(Duration::from_millis(100), events.recv()) + .await + .is_err(), + "without cap 1<<3 the registry must not deliver kind-6 streams" + ); +} + +#[tokio::test] +async fn wants_peer_rejects_when_configured_slot_cap_is_reached() { + let config = ZakuraBlockSyncConfig { + peer_limits: ServicePeerLimits { + max_inbound_peers: 0, + max_outbound_peers: 2, + ..ServicePeerLimits::default() + }, + ..ZakuraBlockSyncConfig::default() + }; + let (service, mut events) = BlockSyncService::new_for_test(config); + let inbound_peer = peer(5); + + assert!(!service.wants_peer( + &inbound_peer, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Inbound + )); + assert!(service.wants_peer( + &inbound_peer, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound + )); + + let mut inbound_senders = Vec::new(); + for byte in 6..=7 { + let peer_id = peer(byte); + let (inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, _outbound_rx) = framed_channel(4); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + assert!(matches!( + next_event(&mut events).await, + BlockSyncEvent::PeerConnected(session) if session.peer_id() == &peer_id + )); + inbound_senders.push(inbound_tx); + } + + assert_eq!(service.peer_count(), 2); + assert!(!service.wants_peer( + &peer(8), + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound + )); + + let (_inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, _outbound_rx) = framed_channel(4); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer(8), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + assert_eq!(service.peer_count(), 2); +} + +#[tokio::test] +async fn reactor_drives_tip_to_getblocks_to_submit_over_framed_path() { + let config = immediate_body_download_config(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer = peer(40); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, mut outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(1), + tip_hash: block::Hash([1; 32]), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block_hash = block.hash(); + let block_size = u32::try_from( + block + .zcash_serialize_to_vec() + .expect("block serializes") + .len(), + ) + .expect("test block size fits u32"); + tip_tx + .send((block::Height(1), block_hash)) + .expect("tip watch is live"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + assert_eq!(verified_block_tip, block::Height(0)); + assert_eq!(best_header_tip, block::Height(1)); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before query: {action:?}"), + } + } + + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block_hash, + size: BlockSizeEstimate::Advertised(block_size), + }])) + .await + .expect("needed metadata queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::SendMessage { + peer: action_peer, + msg: + BlockSyncMessage::GetBlocks { + start_height, + count, + }, + } => { + assert_eq!(action_peer, peer); + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 1); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before getblocks: {action:?}"), + } + } + + loop { + let frame = tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv()) + .await + .expect("outbound frame arrives") + .expect("outbound channel is live"); + if let BlockSyncMessage::GetBlocks { + start_height, + count, + } = BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes") + { + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 1); + break; + } + } + + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block frame encodes"), + ) + .await + .expect("block frame queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { + block: submitted, .. + } => { + assert_eq!(submitted.hash(), block_hash); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before submit: {action:?}"), + } + } + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_keeps_submitted_body_budget_until_apply_finishes() { + let blocks = mainnet_blocks_1_to_3(); + let block1_size = block_size(&blocks[0]); + let mut config = immediate_body_download_config(); + config.max_inflight_block_bytes = u64::from(block1_size); + + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(41); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, mut outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(2), + tip_hash: blocks[1].hash(), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + tip_tx + .send((block::Height(2), blocks[1].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(1), + hash: blocks[0].hash(), + size: BlockSizeEstimate::Advertised(block1_size), + }, + BlockSyncBlockMeta { + height: block::Height(2), + hash: blocks[1].hash(), + size: BlockSizeEstimate::Advertised(block1_size), + }, + ])) + .await + .expect("needed metadata queues"); + + let (request_peer, start_height, count) = wait_for_getblocks(&mut actions).await; + assert_eq!(request_peer, peer_id); + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 1); + while !matches!( + BlockSyncMessage::decode_frame( + tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv()) + .await + .expect("outbound frame arrives") + .expect("outbound channel is live") + ) + .expect("frame decodes"), + BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 1, + } + ) {} + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[0].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + + let submit_token = loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { token, block } => { + assert_eq!(block.hash(), blocks[0].hash()); + break token; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before submit: {action:?}"), + } + }; + assert_eq!(handle.local_status().servable_high, block::Height(0)); + + let quiet = tokio::time::timeout(Duration::from_millis(100), actions.recv()).await; + assert!( + quiet.is_err(), + "submitted-but-not-applied block should keep body budget full", + ); + + handle + .send(BlockSyncEvent::BlockApplyFinished { + token: submit_token, + height: block::Height(1), + hash: blocks[0].hash(), + result: BlockApplyResult::Committed, + local_frontier: Some(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }), + }) + .await + .expect("apply-finished event queues"); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if handle.local_status().servable_high == block::Height(1) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("apply completion frontier advances advertised status"); + + let (_request_peer, start_height, count) = wait_for_getblocks(&mut actions).await; + assert_eq!(start_height, block::Height(2)); + assert_eq!(count, 1); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_keeps_active_response_when_needed_snapshot_omits_inflight_height() { + let blocks = fake_sequential_blocks(3); + let config = immediate_body_download_config(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(42); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(3), + tip_hash: blocks[2].hash(), + max_blocks_per_response: 16, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks( + blocks.iter().map(block_meta).collect(), + )) + .await + .expect("needed metadata queues"); + + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 3) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[0].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("first block queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } if block.hash() == blocks[0].hash() => break, + BlockSyncAction::SendMessage { .. } | BlockSyncAction::QueryNeededBlocks { .. } => {} + action => panic!("unexpected action before first submit: {action:?}"), + } + } + + handle + .send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])])) + .await + .expect("newer needed metadata queues"); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[1].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("second block queues"); + + let mut submitted_second = false; + while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await + { + match action { + BlockSyncAction::SubmitBlock { block, .. } if block.hash() == blocks[1].hash() => { + submitted_second = true; + break; + } + BlockSyncAction::Misbehavior { + peer, + reason: BlockSyncMisbehavior::UnsolicitedBlock, + } if peer == peer_id => panic!("in-flight body was misclassified as unsolicited"), + BlockSyncAction::SendMessage { .. } + | BlockSyncAction::QueryNeededBlocks { .. } + | BlockSyncAction::SubmitBlock { .. } => {} + action => panic!("unexpected action after second block: {action:?}"), + } + } + + assert!( + submitted_second, + "second body from the original active response should remain correlated", + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_ignores_unmatched_body_for_currently_needed_height() { + let blocks = mainnet_blocks_1_to_3(); + let config = immediate_body_download_config(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(142); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + assert_eq!(wait_for_connect_status(&mut actions).await, peer_id); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(2), + servable_high: block::Height(3), + tip_hash: blocks[2].hash(), + max_blocks_per_response: 16, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + tip_tx + .send((block::Height(1), blocks[0].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])])) + .await + .expect("needed metadata queues"); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[0].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("unmatched needed block queues"); + + let quiet = tokio::time::timeout(Duration::from_millis(200), async { + loop { + if let BlockSyncAction::Misbehavior { reason, .. } = next_action(&mut actions).await { + return reason; + } + } + }) + .await; + assert!( + quiet.is_err(), + "unmatched body for a currently needed height should not be hard misbehavior", + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_queries_needed_blocks_above_submitted_floor() { + let blocks = mainnet_blocks_1_to_3(); + let block1_size = block_size(&blocks[0]); + let block2_size = block_size(&blocks[1]); + let mut config = immediate_body_download_config(); + config.max_inflight_block_bytes = u64::from(block1_size) + u64::from(block2_size); + + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(43); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(3), + tip_hash: blocks[2].hash(), + max_blocks_per_response: 2, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(1), + hash: blocks[0].hash(), + size: BlockSizeEstimate::Advertised(block1_size), + }, + BlockSyncBlockMeta { + height: block::Height(2), + hash: blocks[1].hash(), + size: BlockSizeEstimate::Advertised(block2_size), + }, + ])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(1), 2) + ); + + for block in blocks.iter().take(2) { + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + } + + let mut submitted = Vec::new(); + while submitted.len() < 2 { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { token, block } => { + submitted.push(( + block.coinbase_height().expect("test block has height"), + token, + )); + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before checkpoint submissions: {action:?}"), + } + } + assert_eq!( + submitted + .iter() + .map(|(height, _token)| *height) + .collect::>(), + vec![block::Height(1), block::Height(2)] + ); + + handle + .send(BlockSyncEvent::BlockApplyFinished { + token: submitted[0].1, + height: block::Height(1), + hash: blocks[0].hash(), + result: BlockApplyResult::Committed, + local_frontier: None, + }) + .await + .expect("apply-finished event queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + assert_eq!( + verified_block_tip, + block::Height(2), + "missing-body query must skip already submitted contiguous bodies", + ); + assert_eq!(best_header_tip, block::Height(3)); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before needed-block query: {action:?}"), + } + } + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_retries_submitted_body_after_apply_rejection() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block_bytes = block_size(&block); + let mut config = immediate_body_download_config(); + config.max_inflight_block_bytes = u64::from(block_bytes); + + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(42); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(1), + tip_hash: block.hash(), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + tip_tx + .send((block::Height(1), block.hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_bytes), + }])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 1) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + let submit_token = loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { + token, + block: submitted, + } => { + assert_eq!(submitted.hash(), block.hash()); + break token; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before submit: {action:?}"), + } + }; + + handle + .send(BlockSyncEvent::BlockApplyFinished { + token: submit_token, + height: block::Height(1), + hash: block.hash(), + result: BlockApplyResult::Rejected, + local_frontier: None, + }) + .await + .expect("apply-finished event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_bytes), + }])) + .await + .expect("needed metadata queues after rejection"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(1), 1), + "apply rejection must release capacity and clear submitted coverage" + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_pauses_new_body_downloads_near_tip_by_default() { + let config = ZakuraBlockSyncConfig::default(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(70); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + assert_eq!(wait_for_connect_status(&mut actions).await, peer_id); + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(3), + tip_hash: block::Hash([3; 32]), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status queues"); + + for height in [block::Height(1), block::Height(2)] { + let hash_byte = u8::try_from(height.0).expect("test height fits in u8"); + handle + .send(BlockSyncEvent::HeaderTipChanged { + height, + hash: block::Hash([hash_byte; 32]), + }) + .await + .expect("header-tip event queues"); + let quiet = tokio::time::timeout(Duration::from_millis(100), actions.recv()).await; + assert!( + quiet.is_err(), + "lag {height:?} is within the default pause window and must not query needed blocks", + ); + + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height, + hash: block::Hash([hash_byte; 32]), + size: BlockSizeEstimate::Unknown, + }])) + .await + .expect("stale needed-block event queues"); + let quiet = tokio::time::timeout(Duration::from_millis(100), actions.recv()).await; + assert!( + quiet.is_err(), + "stale needed metadata inside the pause window must not schedule GetBlocks", + ); + } + + handle + .send(BlockSyncEvent::HeaderTipChanged { + height: block::Height(3), + hash: block::Hash([3; 32]), + }) + .await + .expect("header-tip event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(3), + } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(1), + hash: block::Hash([1; 32]), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(2), + hash: block::Hash([2; 32]), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(3), + hash: block::Hash([3; 32]), + size: BlockSizeEstimate::Unknown, + }, + ])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(1), 3) + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_zero_pause_threshold_preserves_lag_one_downloads() { + let config = immediate_body_download_config(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config, + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + + handle + .send(BlockSyncEvent::HeaderTipChanged { + height: block::Height(1), + hash: block::Hash([1; 32]), + }) + .await + .expect("header-tip event queues"); + + assert!(matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(1), + } + )); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_keeps_block_sync_peer_after_catch_up_and_reuses_later() { + let mut config = ZakuraBlockSyncConfig::default(); + config.peer_limits.max_outbound_peers = 1; + let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(4), block::Hash([4; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 71, + block::Height(6), + block::Hash([6; 32]), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + handle + .send(BlockSyncEvent::HeaderTipChanged { + height: block::Height(3), + hash: block::Hash([3; 32]), + }) + .await + .expect("header-tip event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(1), + hash: block::Hash([1; 32]), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(2), + hash: block::Hash([2; 32]), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(3), + hash: block::Hash([3; 32]), + size: BlockSizeEstimate::Unknown, + }, + ])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 3) + ); + + for height in [block::Height(1), block::Height(2)] { + let hash_byte = u8::try_from(height.0).expect("test height fits in u8"); + handle + .send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: height, + verified_block_hash: block::Hash([hash_byte; 32]), + })) + .await + .expect("frontier event queues"); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(handle.peer_snapshot().outbound_peers, 1); + assert_eq!(service.peer_count(), 1); + } + + inbound_tx + .send( + BlockSyncMessage::BlocksDone { + start_height: block::Height(1), + returned: 3, + } + .encode_frame() + .expect("BlocksDone encodes"), + ) + .await + .expect("BlocksDone queues"); + + handle + .send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(3), + verified_block_hash: block::Hash([3; 32]), + })) + .await + .expect("caught-up frontier event queues"); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + handle.peer_snapshot().outbound_peers, + 1, + "caught-up nodes must keep block-sync peers so they can serve fresh nodes", + ); + assert_eq!(service.peer_count(), 1); + + handle + .send(BlockSyncEvent::HeaderTipChanged { + height: block::Height(6), + hash: block::Hash([6; 32]), + }) + .await + .expect("later header-tip event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(3), + best_header_tip: block::Height(6), + } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(4), + hash: block::Hash([4; 32]), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(5), + hash: block::Hash([5; 32]), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(6), + hash: block::Hash([6; 32]), + size: BlockSizeEstimate::Unknown, + }, + ])) + .await + .expect("new needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(4), 3), + "the retained block-sync peer should be reused after later header growth", + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_accepts_multi_block_range_and_submits_parent_first() { + let config = ZakuraBlockSyncConfig::default(); + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer = peer(43); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, mut outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(3), + tip_hash: blocks[2].hash(), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks( + blocks + .iter() + .map(|block| BlockSyncBlockMeta { + height: block.coinbase_height().expect("test block has height"), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_size(block)), + }) + .collect(), + )) + .await + .expect("needed metadata queues"); + + let (action_peer, start_height, count) = wait_for_getblocks(&mut actions).await; + assert_eq!(action_peer, peer); + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 3); + + while !matches!( + BlockSyncMessage::decode_frame( + tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv()) + .await + .expect("outbound frame arrives") + .expect("outbound channel is live") + ) + .expect("frame decodes"), + BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 3, + } + ) {} + + for index in [1usize, 2, 0] { + inbound_tx + .send( + BlockSyncMessage::Block(blocks[index].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + } + + let mut submitted = Vec::new(); + while submitted.len() < 3 { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } => submitted.push( + block + .coinbase_height() + .expect("submitted test block has height"), + ), + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::Misbehavior { reason, .. } => { + panic!("honest multi-block response was misclassified: {reason:?}") + } + action => panic!("unexpected action before all submits: {action:?}"), + } + } + assert_eq!( + submitted, + vec![block::Height(1), block::Height(2), block::Height(3)] + ); + + reactor_task.abort(); +} + +/// Every solicited body in a burst must reach the apply stage — the inbound +/// peer->reactor path must never silently drop a block body under load. +/// +/// This is the regression guard for the production "drop-through" stall. The +/// per-peer wire queue is bounded; the old inbound pump forwarded decoded +/// messages with a non-blocking `try_send` and dropped solicited block bodies +/// once that queue filled during a body flood. A single dropped body wedges +/// `body_download_floor`, and because checkpoint-range commits wait +/// indefinitely for a contiguous range, the wedge never clears — every block +/// above it sits in `applying` forever and sync stalls at a checkpoint. The fix +/// makes the pump backpressure instead of dropping, so the flood always drains. +/// +/// The harness makes a drop *fatal* rather than self-healing, which is what our +/// earlier tests missed: a one-slot wire queue forces the drop, a single +/// in-flight request stops the reactor from working around the gap, and a very +/// long request timeout means a dropped body is never re-requested within the +/// deadline. So a regression hangs here (outer timeout) instead of slowly +/// recovering and passing. +#[tokio::test] +async fn reactor_backpressures_inbound_body_flood_without_dropping_bodies() { + const FLOOD: u32 = 64; + let blocks = fake_sequential_blocks(FLOOD); + let tip = blocks.last().expect("flood is non-empty").clone(); + let tip_height = tip.coinbase_height().expect("tip has height"); + + let mut config = immediate_body_download_config(); + // One-slot wire queue: a pump that outruns the reactor by more than one + // frame must backpressure, never drop. + config.peer_limits.inbound_queue_depth = 1; + // Hold the whole flood in flight at once so nothing pauses on the byte + // budget; the inbound flood, not the budget, is what this test exercises. + config.max_inflight_block_bytes = u64::MAX; + // A dropped body must not be quietly re-requested and healed before the + // deadline — that would hide the very regression this test guards. + config.request_timeout = Duration::from_secs(300); + + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer = peer(64); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + + service.add_peer(Peer::new_with_direction( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: tip_height, + tip_hash: tip.hash(), + max_blocks_per_response: FLOOD, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + tip_tx + .send((tip_height, tip.hash())) + .expect("tip watch is live"); + + // Feed solicited bodies from a dedicated task so the drive loop never blocks + // on inbound backpressure while it still needs to drain reactor actions. + let feed_blocks = blocks.clone(); + let (feed_tx, mut feed_rx) = mpsc::unbounded_channel::(); + let feeder = tokio::spawn(async move { + while let Some(height) = feed_rx.recv().await { + let frame = BlockSyncMessage::Block(feed_blocks[(height - 1) as usize].clone()) + .encode_frame() + .expect("block frame encodes"); + if inbound_tx.send(frame).await.is_err() { + break; + } + } + }); + + let metas: Vec<_> = blocks + .iter() + .map(|block| BlockSyncBlockMeta { + height: block.coinbase_height().expect("test block has height"), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_size(block)), + }) + .collect(); + + let drive = async { + let mut submitted = std::collections::HashSet::new(); + while submitted.len() < FLOOD as usize { + // Wait on the raw channel (no per-action timeout): if a regression + // drops a body the reactor simply stops producing actions, and the + // outer deadline below reports the failure. + let Some(action) = actions.recv().await else { + break; + }; + match action { + BlockSyncAction::QueryNeededBlocks { .. } => { + handle + .send(BlockSyncEvent::NeededBlocks(metas.clone())) + .await + .expect("needed metadata queues"); + } + BlockSyncAction::SendMessage { + msg: + BlockSyncMessage::GetBlocks { + start_height, + count, + }, + .. + } => { + for height in start_height.0..start_height.0 + count { + feed_tx.send(height).expect("feeder task stays open"); + } + } + BlockSyncAction::SubmitBlock { block, .. } => { + submitted.insert(block.coinbase_height().expect("submitted block has height")); + } + _ => {} + } + } + submitted + }; + + let submitted = tokio::time::timeout(Duration::from_secs(20), drive) + .await + .expect( + "flooded bodies must all reach the apply stage; a dropped inbound body wedges block sync", + ); + + let expected: std::collections::HashSet<_> = (1..=FLOOD).map(block::Height).collect(); + assert_eq!( + submitted, expected, + "every solicited body in the flood must be submitted exactly once, with no drops" + ); + + feeder.abort(); + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_restarted_at_genesis_queries_and_schedules_without_tip_change() { + let config = immediate_body_download_config(); + let blocks = mainnet_blocks_1_to_3(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(3), blocks[2].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + + match next_action(&mut actions).await { + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + assert_eq!(verified_block_tip, block::Height(0)); + assert_eq!(best_header_tip, block::Height(3)); + } + action => panic!("restart from genesis must query missing bodies, got {action:?}"), + } + + let (peer, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 67, + block::Height(3), + blocks[2].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + handle + .send(BlockSyncEvent::NeededBlocks( + blocks.iter().map(block_meta).collect(), + )) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer, block::Height(1), 3), + "restart from genesis must schedule scratch body sync from height 1" + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[0].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } => { + assert_eq!(block.hash(), blocks[0].hash()); + assert_eq!(block.coinbase_height(), Some(block::Height(1))); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before first scratch submit: {action:?}"), + } + } + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_accepts_blocks_done_after_completed_range() { + let config = immediate_body_download_config(); + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 68, + block::Height(2), + blocks[1].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + tip_tx + .send((block::Height(2), blocks[1].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer.clone(), block::Height(1), 1) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[0].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } => { + assert_eq!(block.hash(), blocks[0].hash()); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before submit: {action:?}"), + } + } + + inbound_tx + .send( + BlockSyncMessage::BlocksDone { + start_height: block::Height(1), + returned: 1, + } + .encode_frame() + .expect("BlocksDone encodes"), + ) + .await + .expect("BlocksDone queues"); + + while let Ok(Some(action)) = + tokio::time::timeout(Duration::from_millis(200), actions.recv()).await + { + if let BlockSyncAction::Misbehavior { + peer: action_peer, + reason, + } = action + { + assert_ne!( + (action_peer, reason), + (peer.clone(), BlockSyncMisbehavior::UnsolicitedDone), + "a valid terminator after a completed block response must not be scored" + ); + } + } + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_retries_missing_heights_after_partial_blocks_done() { + let config = immediate_body_download_config(); + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 69, + block::Height(3), + blocks[2].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks( + blocks.iter().map(block_meta).collect(), + )) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer.clone(), block::Height(1), 3) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[0].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } => { + assert_eq!(block.hash(), blocks[0].hash()); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before first submit: {action:?}"), + } + } + + inbound_tx + .send( + BlockSyncMessage::BlocksDone { + start_height: block::Height(1), + returned: 1, + } + .encode_frame() + .expect("BlocksDone encodes"), + ) + .await + .expect("BlocksDone queues"); + + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer, block::Height(2), 1), + "partial responses must retry the first missing height" + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn checkpoint_hole_disconnect_retries_first_missing_height_with_fresh_peer() { + const FIRST_NEEDED: u32 = 801; + const PREFIX_END: u32 = 1072; + const HOLE_START: u32 = 1073; + const HOLE_END: u32 = 1080; + const LAST_METADATA: u32 = 1621; + const BEST_HEADER_TIP: u32 = 10_400; + + let blocks = fake_blocks_in_range(FIRST_NEEDED, LAST_METADATA); + let block_at = |height: u32| -> Arc { + let index = + usize::try_from(height - FIRST_NEEDED).expect("test height is inside block vector"); + blocks[index].clone() + }; + let metas: Vec<_> = blocks.iter().map(block_meta).collect(); + let prefix: std::collections::HashSet<_> = + (FIRST_NEEDED..=PREFIX_END).map(block::Height).collect(); + let sparse_above_hole: std::collections::HashSet<_> = [1081, 1096, 1200, 1300] + .into_iter() + .map(block::Height) + .collect(); + + let mut config = immediate_body_download_config(); + config.fanout = 1; + config.max_inflight_block_bytes = u64::MAX; + config.request_timeout = Duration::from_secs(300); + config.peer_limits.max_outbound_peers = 1; + config.peer_limits.inbound_queue_depth = 32; + config.peer_limits.outbound_queue_depth = 32; + + let (_tip_tx, tip_rx) = watch::channel((block::Height(BEST_HEADER_TIP), block::Hash([10; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(800), + verified_block_tip: block::Height(800), + verified_block_hash: block::Hash([8; 32]), + }, + (block::Height(BEST_HEADER_TIP), block::Hash([10; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + + let (old_peer, old_inbound, _old_outbound) = connect_peer_with_status_message( + &service, + &mut actions, + 70, + BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(LAST_METADATA), + tip_hash: block_at(LAST_METADATA).hash(), + max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST, + max_inflight_requests: 4, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }, + ) + .await; + handle + .send(BlockSyncEvent::NeededBlocks(metas.clone())) + .await + .expect("checkpoint metadata queues after peer connection"); + + let (feed_tx, mut feed_rx) = mpsc::unbounded_channel::(); + let blocks_for_feeder = blocks.clone(); + let feeder = tokio::spawn(async move { + while let Some(height) = feed_rx.recv().await { + let index = usize::try_from(height - FIRST_NEEDED) + .expect("fed test height is inside block vector"); + let frame = BlockSyncMessage::Block(blocks_for_feeder[index].clone()) + .encode_frame() + .expect("block frame encodes"); + if old_inbound.send(frame).await.is_err() { + break; + } + } + }); + + let mut requests = Vec::new(); + let mut submitted = std::collections::HashSet::new(); + let primed = tokio::time::timeout(Duration::from_secs(20), async { + while requests.len() < 4 || !prefix.is_subset(&submitted) { + let action = actions + .recv() + .await + .expect("block-sync action channel should stay open"); + match action { + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + assert_eq!(verified_block_tip, block::Height(800)); + assert_eq!(best_header_tip, block::Height(BEST_HEADER_TIP)); + handle + .send(BlockSyncEvent::NeededBlocks(metas.clone())) + .await + .expect("checkpoint metadata queues"); + } + BlockSyncAction::SendMessage { + peer, + msg: + BlockSyncMessage::GetBlocks { + start_height, + count, + }, + } if peer == old_peer => { + requests.push((start_height, count)); + let end_height = start_height + .0 + .checked_add(count) + .expect("test request height range fits u32"); + for height in start_height.0..end_height { + let height = block::Height(height); + if height.0 <= PREFIX_END || sparse_above_hole.contains(&height) { + feed_tx.send(height.0).expect("feeder task stays open"); + } + } + } + BlockSyncAction::SubmitBlock { block, .. } => { + let height = block.coinbase_height().expect("submitted block has height"); + assert!( + submitted.insert(height), + "height {height:?} was submitted more than once before the hole retry" + ); + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action while priming checkpoint hole: {action:?}"), + } + } + }) + .await; + assert!( + primed.is_ok(), + "checkpoint-hole priming timed out: requests={requests:?}, submitted={} of {}", + submitted.len(), + prefix.len() + ); + + assert!( + requests + .iter() + .any(|(start, count)| *start <= block::Height(HOLE_START) + && start.0.saturating_add(*count) > HOLE_END), + "the initial in-flight requests must cover the missing checkpoint hole" + ); + + service.remove_peer(&old_peer); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if handle.peer_snapshot().outbound_peers == 0 && service.peer_count() == 0 { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("disconnect releases the old block-sync peer slot"); + + let (new_peer, _new_inbound, _new_outbound) = connect_peer_with_status_message( + &service, + &mut actions, + 71, + BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(LAST_METADATA), + tip_hash: block_at(LAST_METADATA).hash(), + max_blocks_per_response: MAX_BS_BLOCKS_PER_REQUEST, + max_inflight_requests: 4, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }, + ) + .await; + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let action = actions + .recv() + .await + .expect("block-sync action channel should stay open"); + match action { + BlockSyncAction::QueryNeededBlocks { .. } => { + handle + .send(BlockSyncEvent::NeededBlocks(metas.clone())) + .await + .expect("post-disconnect metadata queues"); + } + BlockSyncAction::SendMessage { + peer, + msg: + BlockSyncMessage::GetBlocks { + start_height, + count, + }, + } if peer == new_peer => { + assert_eq!( + start_height, + block::Height(HOLE_START), + "after a partial response disconnect, the first fresh request must fill the \ + checkpoint hole instead of jumping above it or reusing a stale assignment" + ); + assert!(count >= 1); + break; + } + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::SubmitBlock { block, .. } => { + let height = block.coinbase_height().expect("submitted block has height"); + assert!( + submitted.insert(height), + "height {height:?} was resubmitted before the checkpoint hole was retried" + ); + } + action => { + panic!("unexpected action while waiting for checkpoint-hole retry: {action:?}") + } + } + } + }) + .await + .expect("fresh peer should request the checkpoint hole after disconnect"); + + feeder.abort(); + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_reset_mid_download_drops_stale_anchors_and_releases_budget() { + let mut config = ZakuraBlockSyncConfig { + max_inflight_block_bytes: 20_000, + ..immediate_body_download_config() + }; + config.peer_limits.outbound_queue_depth = 16; + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(1), blocks[0].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 47, + block::Height(3), + blocks[2].hash(), + 1, + 20_000, + ) + .await; + + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(3), + } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(2), + hash: blocks[1].hash(), + size: BlockSizeEstimate::Advertised(10_000), + }, + BlockSyncBlockMeta { + height: block::Height(3), + hash: blocks[2].hash(), + size: BlockSizeEstimate::Advertised(10_000), + }, + ])) + .await + .expect("old-fork needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(2), 2) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[2].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("out-of-order old-fork block queues"); + + handle + .send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + })) + .await + .expect("reset event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(3), + } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(2), + hash: block::Hash([92; 32]), + size: BlockSizeEstimate::Advertised(20_000), + }])) + .await + .expect("new-fork needed metadata queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::SendMessage { + peer, + msg: + BlockSyncMessage::GetBlocks { + start_height, + count, + }, + } => { + assert_eq!( + (peer, start_height, count), + (peer_id, block::Height(2), 1), + "a full-budget new fork request can only be scheduled if stale bytes were released" + ); + break; + } + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::Misbehavior { + reason: BlockSyncMisbehavior::UnsolicitedBlock, + .. + } => {} + action => panic!("unexpected action before new fork request: {action:?}"), + } + } + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_forward_reset_preserves_submitted_successor_body() { + let mut config = ZakuraBlockSyncConfig { + max_inflight_block_bytes: 20_000, + ..immediate_body_download_config() + }; + config.peer_limits.outbound_queue_depth = 16; + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(1), blocks[0].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 49, + block::Height(3), + blocks[2].hash(), + 1, + 20_000, + ) + .await; + + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(3), + } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + block_meta(&blocks[1]), + block_meta(&blocks[2]), + ])) + .await + .expect("initial needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(2), 2) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[1].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("contiguous body queues"); + loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } => { + assert_eq!(block.hash(), blocks[1].hash()); + break; + } + BlockSyncAction::SendMessage { .. } | BlockSyncAction::QueryNeededBlocks { .. } => {} + action => panic!("unexpected action before first submit: {action:?}"), + } + } + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[2].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("successor body queues"); + let successor_token = loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { token, block } => { + assert_eq!(block.hash(), blocks[2].hash()); + break token; + } + BlockSyncAction::SendMessage { .. } | BlockSyncAction::QueryNeededBlocks { .. } => {} + action => panic!("unexpected action before successor submit: {action:?}"), + } + }; + + handle + .send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(2), + verified_block_hash: blocks[1].hash(), + })) + .await + .expect("forward reset event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(3), + best_header_tip: block::Height(3), + } + ) {} + + handle + .send(BlockSyncEvent::BlockApplyFinished { + token: successor_token, + height: block::Height(3), + hash: blocks[2].hash(), + result: BlockApplyResult::Committed, + local_frontier: None, + }) + .await + .expect("successor apply result queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(3), + best_header_tip: block::Height(3), + } + ) {} + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_ignores_stale_apply_completion_after_resubmit() { + let config = immediate_body_download_config(); + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block_hash = block.hash(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(61); + let (inbound_tx, inbound_rx) = framed_channel(16); + let (outbound_tx, _outbound_rx) = framed_channel(16); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + assert_eq!(wait_for_connect_status(&mut actions).await, peer_id); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(1), + tip_hash: block_hash, + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + tip_tx + .send((block::Height(1), block_hash)) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block_hash, + size: BlockSizeEstimate::Advertised(block_size(&block)), + }])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 1) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("first body frame queues"); + let stale_token = loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { token, block } => { + assert_eq!(block.hash(), block_hash); + break token; + } + BlockSyncAction::SendMessage { .. } | BlockSyncAction::QueryNeededBlocks { .. } => {} + action => panic!("unexpected action before first submit: {action:?}"), + } + }; + + handle + .send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + })) + .await + .expect("reset event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block_hash, + size: BlockSizeEstimate::Advertised(block_size(&block)), + }])) + .await + .expect("needed metadata queues after reset"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(1), 1) + ); + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("second body frame queues"); + let current_token = loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { token, block } => { + assert_eq!(block.hash(), block_hash); + break token; + } + BlockSyncAction::SendMessage { .. } | BlockSyncAction::QueryNeededBlocks { .. } => {} + action => panic!("unexpected action before second submit: {action:?}"), + } + }; + assert_ne!(stale_token, current_token); + + handle + .send(BlockSyncEvent::BlockApplyFinished { + token: stale_token, + height: block::Height(1), + hash: block_hash, + result: BlockApplyResult::Duplicate, + local_frontier: None, + }) + .await + .expect("stale apply-finished event queues"); + let no_query_from_stale_completion = tokio::time::timeout(Duration::from_millis(100), async { + while let Some(action) = actions.recv().await { + if matches!(action, BlockSyncAction::QueryNeededBlocks { .. }) { + panic!("stale apply completion released the current submission"); + } + } + }) + .await; + assert!( + no_query_from_stale_completion.is_err(), + "reactor should keep the current submission after a stale completion" + ); + + handle + .send(BlockSyncEvent::BlockApplyFinished { + token: current_token, + height: block::Height(1), + hash: block_hash, + result: BlockApplyResult::Committed, + local_frontier: None, + }) + .await + .expect("current apply-finished event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_fast_forward_reset_clears_buffered_bodies_and_releases_budget() { + let mut config = ZakuraBlockSyncConfig { + max_inflight_block_bytes: 20_000, + ..immediate_body_download_config() + }; + config.peer_limits.outbound_queue_depth = 16; + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(1), blocks[0].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 50, + block::Height(4), + block::Hash([4; 32]), + 1, + 20_000, + ) + .await; + + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(3), + } + ) {} + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(2), + hash: blocks[1].hash(), + size: BlockSizeEstimate::Advertised(10_000), + }, + BlockSyncBlockMeta { + height: block::Height(3), + hash: blocks[2].hash(), + size: BlockSizeEstimate::Advertised(10_000), + }, + ])) + .await + .expect("initial needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(2), 2) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[2].clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("out-of-order body queues"); + + handle + .send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(3), + verified_block_hash: blocks[2].hash(), + })) + .await + .expect("fast-forward reset event queues"); + + tip_tx + .send((block::Height(4), block::Hash([4; 32]))) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(3), + best_header_tip: block::Height(4), + } + ) {} + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + handle.peer_snapshot().outbound_peers, + 1, + "caught-up reset keeps the previous block-sync peer available for serving", + ); + assert_eq!(service.peer_count(), 1); + + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(4), + hash: block::Hash([4; 32]), + size: BlockSizeEstimate::Advertised(20_000), + }])) + .await + .expect("post-reset needed metadata queues"); + + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(4), 1), + "a full-budget request after fast-forward Reset requires releasing buffered bytes" + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_fuzzes_arrival_order_across_fork_parent_first() { + #[derive(Copy, Clone)] + enum ForkBody { + Old(usize), + New(usize), + } + + let cases = vec![ + ( + "stale-high-before-reset", + vec![3], + vec![], + vec![ForkBody::Old(2), ForkBody::New(3), ForkBody::New(2)], + ), + ( + "old-prefix-before-reset", + vec![2, 1], + vec![], + vec![ForkBody::Old(3), ForkBody::New(2), ForkBody::New(3)], + ), + ( + "stale-before-new-needed", + vec![2], + vec![3], + vec![ForkBody::New(2), ForkBody::Old(2), ForkBody::New(3)], + ), + ( + "new-out-of-order-with-stale-tail", + vec![], + vec![2], + vec![ForkBody::New(3), ForkBody::Old(3), ForkBody::New(2)], + ), + ]; + + for (case, old_before_reset, old_before_new_needed, after_new_needed) in cases { + let mut config = ZakuraBlockSyncConfig { + max_inflight_block_bytes: 60_000, + ..immediate_body_download_config() + }; + config.peer_limits.outbound_queue_depth = 16; + let old_blocks = mainnet_blocks_1_to_3(); + let new_blocks = vec![ + forked_block(&old_blocks[0], 101), + forked_block(&old_blocks[1], 102), + forked_block(&old_blocks[2], 103), + ]; + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 51, + block::Height(4), + old_blocks[2].hash(), + 1, + 60_000, + ) + .await; + + tip_tx + .send((block::Height(3), old_blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(3), + } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks( + old_blocks + .iter() + .map(|block| BlockSyncBlockMeta { + height: block.coinbase_height().expect("test block has height"), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_size(block)), + }) + .collect(), + )) + .await + .expect("old-fork needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 3), + "{case}: old fork request schedules" + ); + + let mut submitted_tip = block::Height(0); + for height in old_before_reset { + inbound_tx + .send( + BlockSyncMessage::Block(old_blocks[height - 1].clone()) + .encode_frame() + .expect("old-fork block encodes"), + ) + .await + .expect("old-fork block queues"); + drain_parent_first_actions(&mut actions, &mut submitted_tip, None).await; + } + + handle + .send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: new_blocks[0].hash(), + })) + .await + .expect("reset event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(3), + } + ) {} + submitted_tip = block::Height(1); + + tip_tx + .send((block::Height(3), new_blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(3), + } + ) {} + + for height in old_before_new_needed { + inbound_tx + .send( + BlockSyncMessage::Block(old_blocks[height - 1].clone()) + .encode_frame() + .expect("stale old-fork block encodes"), + ) + .await + .expect("stale old-fork block queues"); + drain_parent_first_actions(&mut actions, &mut submitted_tip, Some(&new_blocks)).await; + } + + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(2), + hash: new_blocks[1].hash(), + size: BlockSizeEstimate::Advertised(block_size(&new_blocks[1])), + }, + BlockSyncBlockMeta { + height: block::Height(3), + hash: new_blocks[2].hash(), + size: BlockSizeEstimate::Advertised(block_size(&new_blocks[2])), + }, + ])) + .await + .expect("new-fork needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(2), 2), + "{case}: new fork request schedules after reset" + ); + + for body in after_new_needed { + let block = match body { + ForkBody::Old(height) => old_blocks[height - 1].clone(), + ForkBody::New(height) => new_blocks[height - 1].clone(), + }; + inbound_tx + .send( + BlockSyncMessage::Block(block) + .encode_frame() + .expect("fork body encodes"), + ) + .await + .expect("fork body queues"); + drain_parent_first_actions(&mut actions, &mut submitted_tip, Some(&new_blocks)).await; + } + assert_eq!( + submitted_tip, + block::Height(3), + "{case}: new fork bodies submit parent-first through height 3" + ); + + handle + .send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(3), + verified_block_hash: new_blocks[2].hash(), + })) + .await + .expect("post-submit grow event queues"); + tip_tx + .send((block::Height(4), block::Hash([4; 32]))) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(3), + best_header_tip: block::Height(4), + } + ) {} + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + handle.peer_snapshot().outbound_peers, + 1, + "{case}: caught-up fork handling keeps the old block-sync peer available for serving", + ); + assert_eq!(service.peer_count(), 1); + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(4), + hash: block::Hash([4; 32]), + size: BlockSizeEstimate::Advertised(60_000), + }])) + .await + .expect("post-fuzz needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(4), 1), + "{case}: byte budget returns to baseline after reset and submissions" + ); + + reactor_task.abort(); + } +} + +#[tokio::test] +async fn reactor_competing_fork_download_switches_to_current_header_hashes() { + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(1), blocks[0].hash()), + tip_rx, + immediate_body_download_config(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test( + immediate_body_download_config(), + handle.clone(), + ); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 48, + block::Height(3), + blocks[2].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + tip_tx + .send((block::Height(3), blocks[2].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + block_meta(&blocks[1]), + block_meta(&blocks[2]), + ])) + .await + .expect("old fork metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(2), 2) + ); + + handle + .send(BlockSyncEvent::ChainTipReset(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + })) + .await + .expect("reset event queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(2), + hash: block::Hash([222; 32]), + size: BlockSizeEstimate::Advertised(block_size(&blocks[1])), + }])) + .await + .expect("new fork metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(2), 1) + ); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[1].clone()) + .encode_frame() + .expect("old-fork body encodes"), + ) + .await + .expect("old-fork body queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, BlockSyncMisbehavior::InvalidBlock); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before stale body rejection: {action:?}"), + } + } + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_legacy_commit_dedups_inflight_request_and_reuses_budget() { + let mut config = ZakuraBlockSyncConfig { + max_inflight_block_bytes: 10_000, + ..immediate_body_download_config() + }; + config.peer_limits.outbound_queue_depth = 16; + let blocks = mainnet_blocks_1_to_3(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, _inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 49, + block::Height(2), + blocks[1].hash(), + 1, + 10_000, + ) + .await; + + tip_tx + .send((block::Height(2), blocks[1].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: blocks[0].hash(), + size: BlockSizeEstimate::Advertised(10_000), + }])) + .await + .expect("first needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 1) + ); + + handle + .send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + })) + .await + .expect("legacy commit grow event queues"); + + tip_tx + .send((block::Height(2), blocks[1].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(2), + } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(2), + hash: blocks[1].hash(), + size: BlockSizeEstimate::Advertised(10_000), + }])) + .await + .expect("second needed metadata queues"); + + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id, block::Height(2), 1), + "legacy commit must release the duplicate in-flight reservation" + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_treats_duplicate_buffered_blocks_as_benign() { + let config = immediate_body_download_config(); + let blocks = [ + mainnet_block(&BLOCK_MAINNET_1_BYTES), + mainnet_block(&BLOCK_MAINNET_2_BYTES), + ]; + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(44); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + assert_eq!(wait_for_connect_status(&mut actions).await, peer_id); + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(2), + tip_hash: blocks[1].hash(), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + + tip_tx + .send((block::Height(2), blocks[1].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks( + blocks + .iter() + .map(|block| BlockSyncBlockMeta { + height: block.coinbase_height().expect("test block has height"), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_size(block)), + }) + .collect(), + )) + .await + .expect("needed metadata queues"); + + let (action_peer, start_height, count) = wait_for_getblocks(&mut actions).await; + assert_eq!(action_peer, peer_id); + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 2); + + for block in [&blocks[1], &blocks[1], &blocks[0]] { + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + } + + let mut submitted = Vec::new(); + while submitted.len() < 2 { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } => submitted.push( + block + .coinbase_height() + .expect("submitted test block has height"), + ), + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::Misbehavior { reason, .. } => { + panic!("duplicate buffered body was misclassified: {reason:?}") + } + action => panic!("unexpected action before submit: {action:?}"), + } + } + assert_eq!(submitted, vec![block::Height(1), block::Height(2)]); + + let quiet = tokio::time::timeout(Duration::from_millis(100), async { + while let Some(action) = actions.recv().await { + if let BlockSyncAction::Misbehavior { reason, .. } = action { + panic!("duplicate buffered body was misclassified after submit: {reason:?}"); + } + } + }) + .await; + assert!(quiet.is_err()); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_accepts_rapid_status_growth_without_spam_score() { + let config = ZakuraBlockSyncConfig::default(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + drop(tip_tx); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle); + let peer_id = peer(46); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + assert_eq!(wait_for_connect_status(&mut actions).await, peer_id); + + for servable_high in [block::Height(1), block::Height(2)] { + let hash_byte = u8::try_from(servable_high.0).expect("test height fits in u8"); + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high, + tip_hash: block::Hash([hash_byte; 32]), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status frame queues"); + } + + let quiet = tokio::time::timeout(Duration::from_millis(100), async { + while let Some(action) = actions.recv().await { + if let BlockSyncAction::Misbehavior { reason, .. } = action { + panic!("rapid status growth was misclassified: {reason:?}"); + } + } + }) + .await; + assert!(quiet.is_err()); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_rejects_block_hash_mismatch_without_hard_drop_for_size_mismatch() { + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + immediate_body_download_config(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test( + immediate_body_download_config(), + handle.clone(), + ); + let peer = peer(41); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, mut outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(1), + tip_hash: block::Hash([1; 32]), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status queues"); + tip_tx + .send((block::Height(1), block::Hash([9; 32]))) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block::Hash([9; 32]), + size: BlockSizeEstimate::Advertised(1), + }])) + .await + .expect("needed metadata queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::SendMessage { + msg: BlockSyncMessage::GetBlocks { .. }, + .. + } + ) {} + while !matches!( + BlockSyncMessage::decode_frame( + tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv()) + .await + .expect("outbound frame arrives") + .expect("outbound channel is live") + ) + .expect("frame decodes"), + BlockSyncMessage::GetBlocks { .. } + ) {} + + inbound_tx + .send( + BlockSyncMessage::Block(mainnet_block(&BLOCK_MAINNET_1_BYTES)) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::Misbehavior { reason, .. } => { + assert_eq!(reason, BlockSyncMisbehavior::InvalidBlock); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before invalid-block report: {action:?}"), + } + } + + reactor_task.abort(); +} + +// SECURITY AUDIT (candidate claude-block-sync-source-task-unwired / +// trace-block-sync-source-task-unwired-source-task / +// subset-panic-runtime-containment-block-sync-idle-source-task): SR-4 +// cleanup/anti-drift for the outbound block-sync send path. +// +// Production block-sync scheduling sends outbound `GetBlocks` *directly* through +// `BlockSyncPeerSession` (`reactor::schedule` -> `try_send_get_blocks`). The +// per-peer `BlockSyncSource` action pump (`BlockSyncPeerRecord::actions`) is +// test-only scaffolding with no production producer, and +// `drive_block_sync_actions` deliberately ignores the reactor's duplicate +// `SendMessage` action to avoid double-sending. The audit flagged the risk that +// the per-peer source is dead production scaffolding (per-peer overhead) and a +// latent double-send footgun if it were ever wired naively. +// +// This production-shaped scheduling test locks in the single-sourced outbound +// contract: one scheduled request yields EXACTLY ONE outbound `GetBlocks` frame +// (the authoritative direct session send), never a second copy from a per-peer +// source pump. It proves the outbound behavior is unchanged by the source task +// being idle/test-only, and guards against a future double-send regression. +#[tokio::test] +async fn scheduled_get_blocks_is_sent_once_via_session_not_duplicated_by_source() { + let config = immediate_body_download_config(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + + // Admit a peer through the production `add_peer` path with an observable + // outbound transport channel. + let peer = peer(57); + let (inbound_tx, inbound_rx) = framed_channel(16); + let (outbound_tx, mut outbound_rx) = framed_channel(16); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + // The peer can serve exactly one block above our tip; publish the header tip + // and the needed metadata so the reactor schedules exactly one request. + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(1), + tip_hash: block::Hash([9; 32]), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status queues"); + tip_tx + .send((block::Height(1), block::Hash([9; 32]))) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block::Hash([9; 32]), + size: BlockSizeEstimate::Advertised(1), + }])) + .await + .expect("needed metadata queues"); + // The reactor emits the duplicate `SendMessage` action on the global channel + // (which the production driver ignores). Wait for it so we know the + // authoritative direct send through `BlockSyncPeerSession` already happened. + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::SendMessage { + msg: BlockSyncMessage::GetBlocks { .. }, + .. + } + ) {} + + // Drain the outbound stream for a bounded idle window and count `GetBlocks` + // frames. A single scheduled request must produce exactly one outbound + // `GetBlocks` (the direct session send); a second copy would mean the + // per-peer source action pump is also (double-)sending on the wire. + let mut get_blocks = 0usize; + let mut frames = 0usize; + while frames < 16 { + match tokio::time::timeout(Duration::from_millis(300), outbound_rx.recv()).await { + Ok(Some(frame)) => { + frames += 1; + if matches!( + BlockSyncMessage::decode_frame(frame).expect("outbound frame decodes"), + BlockSyncMessage::GetBlocks { .. } + ) { + get_blocks += 1; + } + } + _ => break, + } + } + assert_eq!( + get_blocks, 1, + "one scheduled request must produce exactly one outbound GetBlocks via \ + BlockSyncPeerSession; a different count means the per-peer source pump is \ + (double-)sending in addition to the authoritative direct path", + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_scores_header_valid_merkle_invalid_body_and_accepts_clean_peer() { + let request_bytes: u32 = 10_000; + let mut config = ZakuraBlockSyncConfig { + max_inflight_block_bytes: u64::from(request_bytes) * 2, + fanout: 2, + ..immediate_body_download_config() + }; + config.peer_limits.max_outbound_peers = 2; + config.peer_limits.outbound_queue_depth = 16; + + let blocks = mainnet_blocks_1_to_3(); + let bad_body = block_with_bad_merkle_root(&blocks[0], &blocks[1]); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (bad_peer, bad_inbound, _bad_outbound) = connect_peer_with_status( + &service, + &mut actions, + 40, + block::Height(2), + blocks[1].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + let (good_peer, good_inbound, _good_outbound) = connect_peer_with_status( + &service, + &mut actions, + 41, + block::Height(2), + blocks[1].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + tip_tx + .send((block::Height(2), blocks[1].hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: blocks[0].hash(), + size: BlockSizeEstimate::Advertised(request_bytes), + }])) + .await + .expect("needed metadata queues"); + + let mut requested_peers = Vec::new(); + while requested_peers.len() < 2 { + let (peer, start_height, count) = wait_for_getblocks(&mut actions).await; + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 1); + requested_peers.push(peer); + } + assert!(requested_peers.contains(&bad_peer)); + assert!(requested_peers.contains(&good_peer)); + + bad_inbound + .send( + BlockSyncMessage::Block(bad_body) + .encode_frame() + .expect("bad block frame encodes"), + ) + .await + .expect("bad block frame queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, bad_peer); + assert_eq!(reason, BlockSyncMisbehavior::InvalidBlock); + break; + } + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::SubmitBlock { block, .. } => { + panic!("merkle-invalid block was buffered and submitted: {block:?}"); + } + action => panic!("unexpected action before invalid body scoring: {action:?}"), + } + } + + good_inbound + .send( + BlockSyncMessage::Block(blocks[0].clone()) + .encode_frame() + .expect("good block frame encodes"), + ) + .await + .expect("good block frame queues"); + + let submit_token = loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { token, block } => { + assert_eq!(block.hash(), blocks[0].hash()); + assert_eq!(block.coinbase_height(), Some(block::Height(1))); + break token; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before clean body submit: {action:?}"), + } + }; + + handle + .send(BlockSyncEvent::BlockApplyFinished { + token: submit_token, + height: block::Height(1), + hash: blocks[0].hash(), + result: BlockApplyResult::Committed, + local_frontier: None, + }) + .await + .expect("apply-finished event queues"); + + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(2), + hash: blocks[1].hash(), + size: BlockSizeEstimate::Advertised(request_bytes * 2), + }])) + .await + .expect("follow-up needed metadata queues"); + let (_peer, start_height, count) = wait_for_getblocks(&mut actions).await; + assert_eq!(start_height, block::Height(2)); + assert_eq!(count, 1); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_serves_committed_blocks_with_count_and_byte_clamps() { + let blocks = mainnet_blocks_1_to_3(); + let block1_size = block_size(&blocks[0]); + let mut config = ZakuraBlockSyncConfig { + max_blocks_per_response: 2, + max_response_bytes: block1_size, + ..ZakuraBlockSyncConfig::default() + }; + config.peer_limits.outbound_queue_depth = 16; + let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(3), + verified_block_hash: blocks[2].hash(), + }, + (block::Height(4), block::Hash([4; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 60, + block::Height(3), + blocks[2].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + inbound_tx + .send( + BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 10, + } + .encode_frame() + .expect("GetBlocks frame encodes"), + ) + .await + .expect("GetBlocks frame queues"); + + loop { + match next_action(&mut actions).await { + BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => { + assert_eq!(peer, peer_id); + assert_eq!(start, block::Height(1)); + assert_eq!(count, 2); + break; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before block range query: {action:?}"), + } + } + + handle + .send(BlockSyncEvent::BlockRangeResponseReady { + peer: peer_id.clone(), + start_height: block::Height(1), + requested_count: 2, + blocks: vec![ + ( + block::Height(1), + blocks[0].clone(), + usize::try_from(block1_size).expect("block size fits usize"), + ), + ( + block::Height(2), + blocks[1].clone(), + usize::try_from(block_size(&blocks[1])).expect("block size fits usize"), + ), + ], + }) + .await + .expect("served block response queues"); + + assert_eq!( + wait_for_outbound_block(&mut outbound_rx).await.hash(), + blocks[0].hash() + ); + assert_eq!( + wait_for_outbound_blocks_done(&mut outbound_rx).await, + (block::Height(1), 1), + "max_response_bytes clamps the served response to one body" + ); + + inbound_tx + .send( + BlockSyncMessage::GetBlocks { + start_height: block::Height(4), + count: 1, + } + .encode_frame() + .expect("GetBlocks frame encodes"), + ) + .await + .expect("above-tip GetBlocks frame queues"); + + assert_eq!( + wait_for_outbound_range_unavailable(&mut outbound_rx).await, + (block::Height(4), 1) + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_never_serves_reorder_buffer_bodies() { + let blocks = mainnet_blocks_1_to_3(); + let mut config = immediate_body_download_config(); + config.peer_limits.outbound_queue_depth = 16; + let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(3), blocks[2].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 61, + block::Height(3), + blocks[2].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + handle + .send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(3), 1) + ); + inbound_tx + .send( + BlockSyncMessage::Block(blocks[2].clone()) + .encode_frame() + .expect("block frame encodes"), + ) + .await + .expect("block frame queues"); + + let quiet = tokio::time::timeout(Duration::from_millis(50), async { + while let Some(action) = actions.recv().await { + if matches!(action, BlockSyncAction::SubmitBlock { .. }) { + panic!("height 3 must stay buffered behind the height 2 gap"); + } + } + }) + .await; + assert!(quiet.is_err()); + + inbound_tx + .send( + BlockSyncMessage::GetBlocks { + start_height: block::Height(3), + count: 1, + } + .encode_frame() + .expect("GetBlocks frame encodes"), + ) + .await + .expect("GetBlocks frame queues"); + + assert_eq!( + wait_for_outbound_range_unavailable(&mut outbound_rx).await, + (block::Height(3), 1), + "uncommitted reorder-buffer body must not be served" + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_schedules_gap_below_buffered_reorder_run() { + // Regression for the mainnet stuck-at-0 deadlock: a body run received above + // an open gap must not starve the gap below it. The state reports every + // header-known, body-missing height (it cannot see our in-memory reorder + // buffer), so a re-query returns already-buffered heights too. With the + // default fanout > 1 the held range lingers in the scheduler queue. Because + // `refresh_needed` builds one maximal contiguous range and `ensure` rejects + // any range overlapping a queued one, the gap below the held run would never + // be scheduled and `body_download_floor` would freeze forever while we + // re-requested the already-held blocks. The reactor must drop already-held + // heights from the needed set so the gap gets scheduled. + let blocks = mainnet_blocks_1_to_3(); + let mut config = immediate_body_download_config(); + config.peer_limits.outbound_queue_depth = 16; + assert!( + config.fanout > 1, + "held range must linger across fanout peers to reproduce the deadlock", + ); + let (_tip_tx, tip_rx) = watch::channel((block::Height(3), blocks[2].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(3), blocks[2].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 63, + block::Height(3), + blocks[2].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + // Height 2 is momentarily not offered, so we fetch and buffer height 3 in + // the reorder buffer above the open height-2 gap. + handle + .send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[2])])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(3), 1) + ); + inbound_tx + .send( + BlockSyncMessage::Block(blocks[2].clone()) + .encode_frame() + .expect("block frame encodes"), + ) + .await + .expect("block frame queues"); + + // Height 3 is now buffered in the reorder buffer and marked covered. Drain + // to quiescence: this both lets the reactor finish processing the body and + // asserts the core fix — a buffered (covered) height is NEVER re-requested. + // The production deadlock re-requested the held run thousands of times via + // the retry path (which bypasses the `needed`-set filter), pinning the queue + // and every peer slot so the gap below the run never got a request. + while let Ok(Some(action)) = + tokio::time::timeout(Duration::from_millis(50), actions.recv()).await + { + match action { + BlockSyncAction::SendMessage { + msg: BlockSyncMessage::GetBlocks { start_height, .. }, + .. + } => panic!("buffered (covered) height {start_height:?} must not be re-requested"), + BlockSyncAction::SendMessage { .. } | BlockSyncAction::QueryNeededBlocks { .. } => {} + other => panic!("unexpected action after buffering block: {other:?}"), + } + } + + // The state still reports both 2 and 3 as body-missing because the reorder + // buffer is invisible to it. With height 3 covered, the reactor must schedule + // the gap at height 2 rather than staying trapped on the buffered run. + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + block_meta(&blocks[1]), + block_meta(&blocks[2]), + ])) + .await + .expect("needed metadata queues"); + + let (got_peer, got_start, _count) = wait_for_getblocks(&mut actions).await; + assert_eq!(got_peer, peer_id); + assert_eq!( + got_start, + block::Height(2), + "reactor must schedule the gap at height 2 below the buffered reorder run", + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_debounces_status_advertisements_on_serving_tip_change() { + let mut config = ZakuraBlockSyncConfig { + status_refresh_interval: Duration::from_secs(60), + ..immediate_body_download_config() + }; + config.peer_limits.outbound_queue_depth = 16; + let (_tip_tx, tip_rx) = watch::channel((block::Height(4), block::Hash([4; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(4), block::Hash([4; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (_peer_id, _inbound_tx, mut outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 62, + block::Height(4), + block::Hash([4; 32]), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + assert!(matches!( + next_outbound_message(&mut outbound_rx).await, + BlockSyncMessage::Status(_) + )); + + handle + .send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + })) + .await + .expect("unchanged frontier queues"); + assert!( + tokio::time::timeout(Duration::from_millis(50), outbound_rx.recv()) + .await + .is_err(), + "unchanged serving range must not advertise" + ); + + handle + .send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: block::Hash([1; 32]), + })) + .await + .expect("changed frontier queues"); + match next_outbound_message(&mut outbound_rx).await { + BlockSyncMessage::Status(status) => { + assert_eq!(status.servable_high, block::Height(1)); + assert_eq!(handle.local_status().servable_high, block::Height(1)); + } + msg => panic!("expected debounced Status after serving tip change, got {msg:?}"), + } + + for height in [2, 3] { + let hash_byte = u8::try_from(height).expect("test height fits in u8"); + handle + .send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(height), + verified_block_hash: block::Hash([hash_byte; 32]), + })) + .await + .expect("burst frontier queues"); + } + assert!( + tokio::time::timeout(Duration::from_millis(50), outbound_rx.recv()) + .await + .is_err(), + "rapid serving-tip changes must be debounced to one Status per window" + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_exchange_watch_converges_to_latest_valid_frontier() { + let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot); + let (exchange, startup) = + exchange_block_sync_startup(initial, immediate_body_download_config()); + + exchange.publish_frontier( + test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced), + "test", + ); + exchange.publish_frontier( + test_frontier_update(0, 0, 2, FrontierChange::HeaderAdvanced), + "test", + ); + exchange.publish_frontier( + test_frontier_update(0, 0, 5, FrontierChange::HeaderAdvanced), + "test", + ); + exchange.publish_frontier( + test_frontier_update(0, 0, 5, FrontierChange::HeaderAdvanced), + "test", + ); + + let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + + wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(5)).await; + assert_eq!( + exchange.current_frontier().frontier.best_header, + test_frontier(5) + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_exchange_progress_retries_after_empty_needed_blocks() { + let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot); + let (exchange, startup) = + exchange_block_sync_startup(initial, immediate_body_download_config()); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + + exchange.publish_frontier( + test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await; + + handle + .send(BlockSyncEvent::NeededBlocks(Vec::new())) + .await + .expect("empty needed-blocks event queues"); + + exchange.publish_frontier( + test_frontier_update(0, 0, 4, FrontierChange::HeaderAdvanced), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(4)).await; + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_exchange_body_progress_retries_after_header_tip_stops() { + let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot); + let (exchange, startup) = + exchange_block_sync_startup(initial, immediate_body_download_config()); + let (_handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + + exchange.publish_frontier( + test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await; + + exchange.publish_frontier( + test_frontier_update(0, 1, 0, FrontierChange::VerifiedGrow), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(1), block::Height(3)).await; + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_exchange_coalesced_header_advance_catches_body_frontier_up() { + let initial = test_frontier_update(0, 0, 0, FrontierChange::Snapshot); + let (exchange, startup) = + exchange_block_sync_startup(initial, immediate_body_download_config()); + + exchange.publish_frontier( + test_frontier_update(0, 3, 0, FrontierChange::VerifiedGrow), + "test", + ); + exchange.publish_frontier( + test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced), + "test", + ); + + let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup); + + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let status = handle.local_status(); + if status.servable_high == block::Height(3) { + assert_eq!(status.tip_hash, test_frontier(3).hash); + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("coalesced header update catches the body frontier up"); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_exchange_ignores_stale_grow_but_accepts_reset() { + let initial = test_frontier_update(0, 5, 10, FrontierChange::Snapshot); + let (exchange, startup) = + exchange_block_sync_startup(initial, immediate_body_download_config()); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + + wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(10)).await; + + exchange.publish_frontier( + test_frontier_update(0, 4, 10, FrontierChange::VerifiedGrow), + "test", + ); + assert!( + tokio::time::timeout(Duration::from_millis(50), actions.recv()) + .await + .is_err(), + "stale lower VerifiedGrow must not trigger a lower body query" + ); + assert_eq!(handle.local_status().servable_high, block::Height(5)); + + exchange.publish_frontier( + test_frontier_update(0, 4, 0, FrontierChange::VerifiedReset), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(4), block::Height(10)).await; + assert_eq!(handle.local_status().servable_high, block::Height(4)); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_exchange_reanchor_lowers_only_best_header_target() { + let initial = test_frontier_update(0, 5, 10, FrontierChange::Snapshot); + let (exchange, startup) = + exchange_block_sync_startup(initial, immediate_body_download_config()); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + + wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(10)).await; + + exchange.publish_frontier( + test_frontier_update(0, 1, 7, FrontierChange::HeaderReanchored), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(5), block::Height(7)).await; + assert_eq!(handle.local_status().servable_high, block::Height(5)); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_exchange_reanchor_releases_stale_submitted_bodies() { + let blocks = mainnet_blocks_1_to_3(); + let mut config = immediate_body_download_config(); + config.max_inflight_block_bytes = blocks + .iter() + .map(|block| u64::from(block_size(block))) + .sum(); + config.request_timeout = Duration::from_secs(300); + + let initial = test_frontier_update(0, 0, 3, FrontierChange::Snapshot); + let (exchange, startup) = exchange_block_sync_startup(initial, config.clone()); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 66, + block::Height(3), + blocks[2].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + handle + .send(BlockSyncEvent::NeededBlocks( + blocks.iter().map(block_meta).collect(), + )) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 3) + ); + + for block in &blocks { + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block frame encodes"), + ) + .await + .expect("block frame queues"); + } + + let mut submitted = Vec::new(); + while submitted.len() < blocks.len() { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { block, .. } => submitted.push( + block + .coinbase_height() + .expect("submitted test block has height"), + ), + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action before all submitted bodies: {action:?}"), + } + } + assert_eq!( + submitted, + vec![block::Height(1), block::Height(2), block::Height(3)] + ); + + exchange.publish_frontier( + test_frontier_update(0, 0, 1, FrontierChange::HeaderReanchored), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(1)).await; + + exchange.publish_frontier( + test_frontier_update(0, 0, 3, FrontierChange::HeaderAdvanced), + "test", + ); + wait_for_query_needed_blocks(&mut actions, block::Height(0), block::Height(3)).await; + + handle + .send(BlockSyncEvent::NeededBlocks( + blocks.iter().map(block_meta).collect(), + )) + .await + .expect("needed metadata after reanchor queues"); + let (got_peer, got_start, got_count) = wait_for_getblocks(&mut actions).await; + assert_eq!(got_peer, peer_id); + assert_eq!(got_start, block::Height(1)); + assert_eq!( + got_count, 3, + "reanchored headers must release old submitted bodies and request them again", + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_ignores_stale_non_reset_frontier_updates() { + let (_tip_tx, tip_rx) = watch::channel((block::Height(3600), block::Hash([36; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(3200), + verified_block_tip: block::Height(3200), + verified_block_hash: block::Hash([32; 32]), + }, + (block::Height(3600), block::Hash([36; 32])), + tip_rx, + immediate_body_download_config(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + + assert!(matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(3200), + best_header_tip: block::Height(3600), + } + )); + + handle + .send(BlockSyncEvent::ChainTipGrow(BlockSyncFrontiers { + finalized_height: block::Height(3200), + verified_block_tip: block::Height(2913), + verified_block_hash: block::Hash([29; 32]), + })) + .await + .expect("stale grow event queues"); + + assert!( + tokio::time::timeout(Duration::from_millis(50), actions.recv()) + .await + .is_err(), + "stale lower grow frontier must not query from the lower height" + ); + assert_eq!(handle.local_status().servable_high, block::Height(3200)); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_retries_matched_range_unavailable_without_scoring_peer() { + let blocks = mainnet_blocks_1_to_3(); + let mut config = immediate_body_download_config(); + config.peer_limits.outbound_queue_depth = 16; + let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(2), blocks[1].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 63, + block::Height(2), + blocks[1].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + handle + .send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[0])])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 1) + ); + + inbound_tx + .send( + BlockSyncMessage::RangeUnavailable { + start_height: block::Height(1), + count: 1, + } + .encode_frame() + .expect("RangeUnavailable frame encodes"), + ) + .await + .expect("RangeUnavailable frame queues"); + + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(1), 1), + "matched RangeUnavailable should retry without scoring the serving peer", + ); + assert_eq!(handle.peer_snapshot().outbound_peers, 1); + + inbound_tx + .send( + BlockSyncMessage::RangeUnavailable { + start_height: block::Height(2), + count: 1, + } + .encode_frame() + .expect("RangeUnavailable frame encodes"), + ) + .await + .expect("unmatched RangeUnavailable frame queues"); + + assert!( + tokio::time::timeout(Duration::from_millis(50), actions.recv()) + .await + .is_err(), + "unmatched RangeUnavailable should be treated as advisory backpressure" + ); + assert_eq!(handle.peer_snapshot().outbound_peers, 1); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_backpressures_serving_slots_without_scoring_peer() { + let mut config = ZakuraBlockSyncConfig { + max_inflight_requests: 1, + ..ZakuraBlockSyncConfig::default() + }; + config.peer_limits.outbound_queue_depth = 16; + let blocks = mainnet_blocks_1_to_3(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(2), blocks[1].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, mut outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 63, + block::Height(1), + blocks[0].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + tokio::time::sleep(Duration::from_millis(10)).await; + + for _ in 0..2 { + inbound_tx + .send( + BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 1, + } + .encode_frame() + .expect("GetBlocks frame encodes"), + ) + .await + .expect("GetBlocks frame queues"); + } + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryBlocksByHeightRange { .. } + ) {} + + assert_eq!( + wait_for_outbound_range_unavailable(&mut outbound_rx).await, + (block::Height(1), 1), + "serving-slot saturation should backpressure the requester, not score it as spam", + ); + assert_eq!(handle.peer_snapshot().outbound_peers, 1); + + handle + .send(BlockSyncEvent::BlockRangeResponseFinished { + peer: peer_id.clone(), + start_height: block::Height(1), + requested_count: 1, + returned_count: 1, + }) + .await + .expect("serving slot release queues"); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_publishes_block_sync_candidate_gap() { + let config = immediate_body_download_config(); + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer_id = peer(77); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, _outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer_id.clone(), + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + assert_eq!(wait_for_connect_status(&mut actions).await, peer_id); + + tip_tx + .send((block::Height(2), block::Hash([2; 32]))) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![ + BlockSyncBlockMeta { + height: block::Height(2), + hash: block::Hash([2; 32]), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(1), + hash: block::Hash([1; 32]), + size: BlockSizeEstimate::Unknown, + }, + ])) + .await + .expect("needed blocks event queues"); + + let mut candidates = handle.subscribe_candidate_state(); + let observed = tokio::time::timeout(Duration::from_secs(1), async { + loop { + candidates + .changed() + .await + .expect("candidate watch remains open"); + let state = candidates.borrow().clone(); + if state.missing_block_bodies == vec![block::Height(1), block::Height(2)] { + return state; + } + } + }) + .await + .unwrap_or_else(|_| handle.candidate_state()); + assert_eq!( + observed.missing_block_bodies, + vec![block::Height(1), block::Height(2)] + ); + assert!( + observed.admitted_node_ids.is_empty(), + "a peer without block-sync status must not satisfy body-sync demand" + ); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(2), + tip_hash: block::Hash([2; 32]), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status queues"); + + let peer_node_id = + node_id_from_block_peer_id(&peer_id).expect("test peer id is a valid node id"); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + candidates + .changed() + .await + .expect("candidate watch remains open"); + if candidates.borrow().admitted_node_ids == vec![peer_node_id] { + return; + } + } + }) + .await + .expect("status-bearing peer is published as an admitted candidate"); + + handle + .send(BlockSyncEvent::StateFrontiersChanged(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(2), + verified_block_hash: block::Hash([2; 32]), + })) + .await + .expect("frontier event queues"); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if handle.candidate_state().missing_block_bodies.is_empty() { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("candidate state clears after the gap is gone"); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_reports_size_mismatch_softly_and_still_submits_valid_block() { + let mut config = ZakuraBlockSyncConfig { + size_deviation_tolerance: 100, + ..immediate_body_download_config() + }; + config.peer_limits.outbound_queue_depth = 8; + let (tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let peer = peer(42); + let (inbound_tx, inbound_rx) = framed_channel(8); + let (outbound_tx, mut outbound_rx) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_BLOCK_SYNC, (inbound_rx, outbound_tx))]); + service.add_peer(Peer::new_with_direction( + peer, + None, + ZAKURA_CAP_BLOCK_SYNC, + ServicePeerDirection::Outbound, + streams, + CancellationToken::new(), + )); + + inbound_tx + .send( + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(1), + tip_hash: block::Hash([1; 32]), + max_blocks_per_response: 4, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame() + .expect("status encodes"), + ) + .await + .expect("status queues"); + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + tip_tx + .send((block::Height(1), block.hash())) + .expect("tip watch is live"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::QueryNeededBlocks { .. } + ) {} + handle + .send(BlockSyncEvent::NeededBlocks(vec![BlockSyncBlockMeta { + height: block::Height(1), + hash: block.hash(), + size: BlockSizeEstimate::Advertised(1), + }])) + .await + .expect("needed metadata queues"); + while !matches!( + next_action(&mut actions).await, + BlockSyncAction::SendMessage { + msg: BlockSyncMessage::GetBlocks { .. }, + .. + } + ) {} + while !matches!( + BlockSyncMessage::decode_frame( + tokio::time::timeout(Duration::from_secs(1), outbound_rx.recv()) + .await + .expect("outbound frame arrives") + .expect("outbound channel is live") + ) + .expect("frame decodes"), + BlockSyncMessage::GetBlocks { .. } + ) {} + + inbound_tx + .send( + BlockSyncMessage::Block(block.clone()) + .encode_frame() + .expect("block encodes"), + ) + .await + .expect("block queues"); + + let mut saw_size_mismatch = false; + let mut saw_submit = false; + while !(saw_size_mismatch && saw_submit) { + match next_action(&mut actions).await { + BlockSyncAction::Misbehavior { reason, .. } => { + assert_eq!(reason, BlockSyncMisbehavior::SizeMismatch); + saw_size_mismatch = true; + } + BlockSyncAction::SubmitBlock { + block: submitted, .. + } => { + assert_eq!(submitted.hash(), block.hash()); + saw_submit = true; + } + BlockSyncAction::SendMessage { .. } => {} + action => panic!("unexpected action during size mismatch test: {action:?}"), + } + } + + reactor_task.abort(); +} + +// SECURITY AUDIT (candidate claude-block-sync-unsolicited-blocksdone-not-rejected / +// codex-blocksync-unsolicited-blocksdone-not-rejected): SR-6/SR-7 response +// correlation + fail-closed. +// +// `handle_blocks_done` reports `UnsolicitedDone` only when the peer is *unknown*. +// For a known, active peer that sends a valid `BlocksDone` with no matching +// outstanding request, the `if let Some(index)` body is skipped and the reactor +// falls through to `schedule()` with no `else` reporting `UnsolicitedDone`. +// `UnsolicitedDone` is a *hard* block-sync misbehavior (`block_sync_misbehavior_is_hard` +// in zebrad start.rs), so the production driver `drive_block_sync_actions` +// disconnects on the first offense -- but this branch never emits it, so an +// admitted peer can stream uncorrelated response terminators forever and stay +// connected. +// +// This test asserts the SAFE behavior (the reactor must report `UnsolicitedDone`). +// It currently FAILS, which is the reproduction. Do not weaken it to pass; the +// fix is to add the missing `else` branch in `handle_blocks_done`. +#[tokio::test] +async fn reactor_known_peer_unsolicited_blocks_done_is_reported_as_misbehavior() { + let config = ZakuraBlockSyncConfig::default(); + let blocks = mainnet_blocks_1_to_3(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(1), blocks[0].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + + // Connect a peer that advertises no downloadable work (servable_high == our + // verified tip), so the reactor never schedules a GetBlocks and the peer has + // zero outstanding requests. The peer is known/active (received_status=true). + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 63, + block::Height(1), + blocks[0].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + // Surprising/hostile input: a valid `BlocksDone` terminator with a start + // height that matches no outstanding request (there are none). + inbound_tx + .send( + BlockSyncMessage::BlocksDone { + start_height: block::Height(7), + returned: 1, + } + .encode_frame() + .expect("BlocksDone frame encodes"), + ) + .await + .expect("BlocksDone frame queues"); + + // Expected safe behavior: the reactor reports `UnsolicitedDone` for this peer + // (which the production driver maps to a hard disconnect). Collect actions for + // a bounded window and assert it appears. + let mut saw_unsolicited_done = false; + while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await + { + if let BlockSyncAction::Misbehavior { peer, reason } = action { + if peer == peer_id && reason == BlockSyncMisbehavior::UnsolicitedDone { + saw_unsolicited_done = true; + break; + } + } + } + + assert!( + saw_unsolicited_done, + "a known peer's unsolicited BlocksDone with no matching outstanding request \ + must be reported as Misbehavior::UnsolicitedDone (SR-6/SR-7), but the reactor \ + silently tolerated it and kept the peer connected", + ); + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_ignores_duplicate_response_at_body_download_floor() { + let config = immediate_body_download_config(); + let blocks = mainnet_blocks_1_to_3(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(2), blocks[1].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(2), blocks[1].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config, handle.clone()); + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 65, + block::Height(2), + blocks[1].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + handle + .send(BlockSyncEvent::NeededBlocks(vec![block_meta(&blocks[1])])) + .await + .expect("needed metadata queues"); + assert_eq!( + wait_for_getblocks(&mut actions).await, + (peer_id.clone(), block::Height(2), 1) + ); + inbound_tx + .send( + BlockSyncMessage::Block(blocks[1].clone()) + .encode_frame() + .expect("block frame encodes"), + ) + .await + .expect("block frame queues"); + + let (token, hash) = loop { + match next_action(&mut actions).await { + BlockSyncAction::SubmitBlock { token, block } => break (token, block.hash()), + BlockSyncAction::SendMessage { .. } | BlockSyncAction::QueryNeededBlocks { .. } => {} + action => panic!("unexpected action while waiting for submit: {action:?}"), + } + }; + + // Simulate the verifier reporting success before the frontier mirror has + // delivered the matching verified-tip update. The applying entry is gone, + // but `body_download_floor` still proves this height was already accepted. + handle + .send(BlockSyncEvent::BlockApplyFinished { + token, + height: block::Height(2), + hash, + result: BlockApplyResult::Committed, + local_frontier: None, + }) + .await + .expect("apply result queues"); + + inbound_tx + .send( + BlockSyncMessage::Block(blocks[1].clone()) + .encode_frame() + .expect("duplicate block frame encodes"), + ) + .await + .expect("duplicate block frame queues"); + inbound_tx + .send( + BlockSyncMessage::BlocksDone { + start_height: block::Height(2), + returned: 1, + } + .encode_frame() + .expect("duplicate terminator frame encodes"), + ) + .await + .expect("duplicate terminator frame queues"); + + while let Ok(Some(action)) = + tokio::time::timeout(Duration::from_millis(200), actions.recv()).await + { + if let BlockSyncAction::Misbehavior { peer, reason } = action { + assert_ne!( + peer, peer_id, + "duplicate response at body_download_floor was reported as {reason:?}" + ); + } + } + + reactor_task.abort(); +} + +#[tokio::test] +async fn reactor_ignores_late_response_frames_after_peer_disconnect() { + let config = ZakuraBlockSyncConfig::default(); + let blocks = mainnet_blocks_1_to_3(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(1), blocks[0].hash())); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: blocks[0].hash(), + }, + (block::Height(1), blocks[0].hash()), + tip_rx, + config.clone(), + ); + let (handle, mut actions, reactor_task) = spawn_block_sync_reactor(startup); + let service = BlockSyncService::new_with_handle_for_test(config.clone(), handle.clone()); + + let (peer_id, _inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 64, + block::Height(1), + blocks[0].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + + service.remove_peer(&peer_id); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if handle.peer_snapshot().outbound_peers == 0 && service.peer_count() == 0 { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("disconnect releases the block-sync peer slot"); + + handle + .send(BlockSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: BlockSyncMessage::Block(blocks[1].clone()), + }) + .await + .expect("late body frame queues"); + handle + .send(BlockSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: BlockSyncMessage::BlocksDone { + start_height: block::Height(2), + returned: 1, + }, + }) + .await + .expect("late terminator frame queues"); + + while let Ok(Some(action)) = + tokio::time::timeout(Duration::from_millis(200), actions.recv()).await + { + if let BlockSyncAction::Misbehavior { peer, reason } = action { + assert_ne!( + peer, peer_id, + "late response from a disconnected peer was reported as {reason:?}" + ); + } + } + + let (peer_id, inbound_tx, _outbound_rx) = connect_peer_with_status( + &service, + &mut actions, + 64, + block::Height(1), + blocks[0].hash(), + 1, + MAX_BS_RESPONSE_BYTES, + ) + .await; + inbound_tx + .send( + BlockSyncMessage::BlocksDone { + start_height: block::Height(7), + returned: 1, + } + .encode_frame() + .expect("BlocksDone frame encodes"), + ) + .await + .expect("unsolicited terminator frame queues"); + + let mut saw_unsolicited_done = false; + while let Ok(Some(action)) = tokio::time::timeout(Duration::from_secs(1), actions.recv()).await + { + if let BlockSyncAction::Misbehavior { peer, reason } = action { + if peer == peer_id && reason == BlockSyncMisbehavior::UnsolicitedDone { + saw_unsolicited_done = true; + break; + } + } + } + assert!( + saw_unsolicited_done, + "reconnecting the same peer must restore hard unsolicited-terminator checks" + ); + + reactor_task.abort(); +} + +/// Regression for `claude-sync-reactor-action-backpressure-stalls-disconnect` in +/// the block-sync reactor: when the bounded 128-slot action channel is saturated +/// and the action driver is stalled, awaiting `actions.send` for `Misbehavior` +/// wedged the reactor, so it could no longer reach its own disconnect path. The +/// reactor must instead enqueue misbehavior non-blockingly and stay live — here, +/// live enough to still tear down a soft offender once it crosses the +/// disconnect threshold. +#[tokio::test] +async fn misbehaving_peer_is_disconnected_even_when_action_channel_is_saturated() { + let config = ZakuraBlockSyncConfig::default(); + let (_tip_tx, tip_rx) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + config, + ); + // `_actions` is held but never drained: the production action driver is + // "stalled". + let (handle, _actions, reactor_task) = spawn_block_sync_reactor(startup); + + // Connect the probe peer with a session whose cancellation token we keep, so + // we can observe the local disconnect. A bare session (no supervised pipe) + // means the cancel is observable without removing the peer from the reactor. + let probe = peer(7); + let probe_cancel = CancellationToken::new(); + let (_inbound_tx, inbound_rx) = framed_channel(4); + let (outbound_tx, _outbound_rx) = framed_channel(4); + let session = PeerStreamSession::new( + probe.clone(), + ZAKURA_STREAM_BLOCK_SYNC, + inbound_rx, + outbound_tx, + probe_cancel.clone(), + ); + handle + .send(BlockSyncEvent::PeerConnected(BlockSyncPeerSession::new( + &session, + ServicePeerDirection::Outbound, + ))) + .await + .expect("probe peer connects"); + + // Saturate the bounded 128-slot action channel. Malformed-frame events from + // an unknown filler peer enqueue `Misbehavior` actions until the channel is + // full. A per-send timeout keeps the test from hanging if the (unfixed) + // reactor wedges on a blocking `actions.send` and stops draining events. + let filler = peer(200); + let decode_error = Arc::new(BlockSyncWireError::TrailingBytes); + for _ in 0..400 { + let send = handle.send(BlockSyncEvent::WireDecodeFailed { + peer: filler.clone(), + error: decode_error.clone(), + }); + if tokio::time::timeout(Duration::from_millis(200), send) + .await + .is_err() + { + break; + } + } + + // Drive the connected probe peer past the soft-misbehavior disconnect + // threshold (3) while the action channel is saturated. A reactor wedged on a + // blocking misbehavior send can never process these events, so it never + // reaches the threshold cancel; a non-blocking reactor does. + for _ in 0..4 { + let _ = tokio::time::timeout( + Duration::from_millis(200), + handle.send(BlockSyncEvent::WireMessage { + peer: probe.clone(), + msg: BlockSyncMessage::GetBlocks { + start_height: block::Height(1), + count: 1, + }, + }), + ) + .await; + } + + tokio::time::timeout(Duration::from_secs(1), probe_cancel.cancelled()) + .await + .expect( + "a repeatedly-misbehaving block-sync peer must still be disconnected when the action \ + channel is saturated and the action driver is stalled", + ); + + reactor_task.abort(); +} diff --git a/zebra-network/src/zakura/block_sync/wire.rs b/zebra-network/src/zakura/block_sync/wire.rs new file mode 100644 index 00000000000..e42eae1a597 --- /dev/null +++ b/zebra-network/src/zakura/block_sync/wire.rs @@ -0,0 +1,246 @@ +use super::{config::*, error::*, *}; + +/// Zakura stream kind reserved for native block sync. +pub const ZAKURA_STREAM_BLOCK_SYNC: u16 = 6; +/// Capability bit for the native block-sync service. +pub const ZAKURA_CAP_BLOCK_SYNC: u64 = 1 << 3; +/// Version of the native block-sync stream. +pub const ZAKURA_BLOCK_SYNC_STREAM_VERSION: u16 = 1; + +/// Peer status advertisement. +pub const MSG_BS_STATUS: u8 = 1; +/// Request a contiguous range of block bodies by height. +pub const MSG_BS_GET_BLOCKS: u8 = 2; +/// Respond with one full block body. +pub const MSG_BS_BLOCK: u8 = 3; +/// Terminate a `GetBlocks` response. +pub const MSG_BS_BLOCKS_DONE: u8 = 4; +/// Report that a requested range is not servable. +pub const MSG_BS_RANGE_UNAVAILABLE: u8 = 5; + +/// Maximum block bodies ever requested or reported by stream 6. +pub const MAX_BS_BLOCKS_PER_REQUEST: u32 = 128; +/// Maximum encoded stream-6 message bytes. +/// +/// This cap is intentionally larger than Zebra's consensus block-size limit so +/// stream-6 can read and classify slightly oversized or future-expanded frames +/// in the block-sync codec instead of dropping them at the raw transport gate. +/// Decoded `Block` messages are still bounded by [`block::MAX_BLOCK_BYTES`]. +pub const MAX_BS_MESSAGE_BYTES: usize = 3 * 1024 * 1024; + +pub(super) const BLOCK_SYNC_MESSAGE_TYPE_BYTES: usize = 1; + +const _: () = assert!(MAX_BS_MESSAGE_BYTES < 4 * 1024 * 1024); +const _: () = assert!(MAX_BS_MESSAGE_BYTES > block::MAX_BLOCK_BYTES as usize); + +/// Native stream-6 block-sync message. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum BlockSyncMessage { + /// Servable range and serving capacity advertisement. + Status(BlockSyncStatus), + /// Request `count` block bodies starting at `start_height`. + GetBlocks { + /// First requested height. + start_height: block::Height, + /// Requested block count. + count: u32, + }, + /// One full block body. + Block(Arc), + /// End of a `GetBlocks` response. + BlocksDone { + /// First requested height. + start_height: block::Height, + /// Number of blocks returned. + returned: u32, + }, + /// The peer cannot serve this range. + RangeUnavailable { + /// First unavailable height. + start_height: block::Height, + /// Unavailable block count. + count: u32, + }, +} + +impl BlockSyncMessage { + /// Returns this message's stream-6 discriminator. + pub fn message_type(&self) -> u8 { + match self { + Self::Status(_) => MSG_BS_STATUS, + Self::GetBlocks { .. } => MSG_BS_GET_BLOCKS, + Self::Block(_) => MSG_BS_BLOCK, + Self::BlocksDone { .. } => MSG_BS_BLOCKS_DONE, + Self::RangeUnavailable { .. } => MSG_BS_RANGE_UNAVAILABLE, + } + } + + /// Encode this message as `[u8 message_type][bounded fields...]`. + pub fn encode(&self) -> Result, BlockSyncWireError> { + let mut bytes = Vec::new(); + bytes.write_u8(self.message_type())?; + match self { + Self::Status(status) => status.encode_to(&mut bytes)?, + Self::GetBlocks { + start_height, + count, + } + | Self::RangeUnavailable { + start_height, + count, + } => { + validate_block_count(*count)?; + write_height(&mut bytes, *start_height)?; + bytes.write_u32::(*count)?; + } + Self::Block(block) => { + block.zcash_serialize(&mut bytes)?; + validate_encoded_block_len( + bytes.len().saturating_sub(BLOCK_SYNC_MESSAGE_TYPE_BYTES), + )?; + } + Self::BlocksDone { + start_height, + returned, + } => { + validate_block_count(*returned)?; + write_height(&mut bytes, *start_height)?; + bytes.write_u32::(*returned)?; + } + } + validate_payload_len(bytes.len())?; + Ok(bytes) + } + + /// Decode a stream-6 message. + pub fn decode(bytes: &[u8]) -> Result { + validate_payload_len(bytes.len())?; + let mut reader = Cursor::new(bytes); + let message_type = reader.read_u8()?; + let message = match message_type { + MSG_BS_STATUS => Self::Status(BlockSyncStatus::decode_from(&mut reader)?), + MSG_BS_GET_BLOCKS => { + let start_height = read_height(&mut reader)?; + let count = reader.read_u32::()?; + validate_block_count(count)?; + Self::GetBlocks { + start_height, + count, + } + } + MSG_BS_BLOCK => { + let block_start = usize::try_from(reader.position()) + .map_err(|_| BlockSyncWireError::NumericOverflow("block payload offset"))?; + let block = Arc::new(block::Block::zcash_deserialize(&mut reader)?); + let block_end = usize::try_from(reader.position()) + .map_err(|_| BlockSyncWireError::NumericOverflow("block payload end"))?; + validate_encoded_block_len(block_end.saturating_sub(block_start))?; + Self::Block(block) + } + MSG_BS_BLOCKS_DONE => { + let start_height = read_height(&mut reader)?; + let returned = reader.read_u32::()?; + validate_block_count(returned)?; + Self::BlocksDone { + start_height, + returned, + } + } + MSG_BS_RANGE_UNAVAILABLE => { + let start_height = read_height(&mut reader)?; + let count = reader.read_u32::()?; + validate_block_count(count)?; + Self::RangeUnavailable { + start_height, + count, + } + } + value => return Err(BlockSyncWireError::UnknownMessageType(value)), + }; + reject_trailing(bytes, &reader)?; + Ok(message) + } + + /// Convert this message into a bounded Zakura frame. + pub fn encode_frame(&self) -> Result { + Ok(Frame { + message_type: u16::from(self.message_type()), + flags: 0, + payload: self.encode()?, + }) + } + + /// Decode this message from a Zakura frame after checking flags and type agreement. + pub fn decode_frame(frame: Frame) -> Result { + if frame.flags != 0 { + return Err(BlockSyncWireError::UnsupportedFlags(frame.flags)); + } + let message = Self::decode(&frame.payload)?; + let frame_message_type = u8::try_from(frame.message_type) + .map_err(|_| BlockSyncWireError::UnknownFrameMessageType(frame.message_type))?; + if frame_message_type != message.message_type() { + return Err(BlockSyncWireError::MismatchedFrameMessageType { + frame: frame.message_type, + payload: message.message_type(), + }); + } + Ok(message) + } +} + +pub(super) fn validate_block_count(count: u32) -> Result<(), BlockSyncWireError> { + if count == 0 { + return Err(BlockSyncWireError::ZeroBlockCount); + } + if count > MAX_BS_BLOCKS_PER_REQUEST { + return Err(BlockSyncWireError::BlockCountLimit { + actual: count, + max: MAX_BS_BLOCKS_PER_REQUEST, + }); + } + Ok(()) +} + +pub(super) fn validate_payload_len(len: usize) -> Result<(), BlockSyncWireError> { + if len > MAX_BS_MESSAGE_BYTES { + return Err(BlockSyncWireError::OversizedPayload { + actual: len, + max: MAX_BS_MESSAGE_BYTES, + }); + } + Ok(()) +} + +fn validate_encoded_block_len(len: usize) -> Result<(), BlockSyncWireError> { + let max = usize::try_from(block::MAX_BLOCK_BYTES) + .map_err(|_| BlockSyncWireError::NumericOverflow("max block bytes"))?; + if len > max { + return Err(BlockSyncWireError::OversizedBlock { actual: len, max }); + } + Ok(()) +} + +pub(super) fn write_height( + writer: &mut W, + height: block::Height, +) -> Result<(), BlockSyncWireError> { + writer.write_u32::(height.0)?; + Ok(()) +} + +pub(super) fn read_height(reader: &mut R) -> Result { + let raw = reader.read_u32::()?; + block::Height::try_from(raw).map_err(|_| BlockSyncWireError::HeightOutOfRange(raw)) +} + +pub(super) fn reject_trailing( + bytes: &[u8], + reader: &Cursor<&[u8]>, +) -> Result<(), BlockSyncWireError> { + let consumed = usize::try_from(reader.position()) + .map_err(|_| BlockSyncWireError::NumericOverflow("payload cursor"))?; + if consumed != bytes.len() { + return Err(BlockSyncWireError::TrailingBytes); + } + Ok(()) +} diff --git a/zebra-network/src/zakura/discovery/candidate_dialer.rs b/zebra-network/src/zakura/discovery/candidate_dialer.rs index 958e60bc79f..8ec75521d7c 100644 --- a/zebra-network/src/zakura/discovery/candidate_dialer.rs +++ b/zebra-network/src/zakura/discovery/candidate_dialer.rs @@ -38,12 +38,15 @@ struct DiscoveryDialWorkerResult { } /// Spawn the long-lived discovery candidate dialer for `endpoint`. +/// +/// Returns the task handle so the caller can track it under the endpoint +/// shutdown owner; the loop also observes the endpoint shutdown token directly. pub(crate) fn spawn_native_discovery_dialer( endpoint: ZakuraEndpoint, discovery: ZakuraDiscoveryHandle, limits: ZakuraLocalLimits, -) { - tokio::spawn(run_native_discovery_dialer(endpoint, discovery, limits)); +) -> tokio::task::JoinHandle<()> { + tokio::spawn(run_native_discovery_dialer(endpoint, discovery, limits)) } /// Seed the discovery book with the configured bootstrap peers as trusted static @@ -72,12 +75,17 @@ pub(crate) async fn run_native_discovery_dialer( discovery: ZakuraDiscoveryHandle, limits: ZakuraLocalLimits, ) { + let shutdown = endpoint.background_shutdown_token(); let mut registered = endpoint.supervisor().subscribe(); let mut in_flight = HashSet::new(); let mut in_flight_by_ip = HashMap::new(); let mut workers = JoinSet::new(); loop { + if shutdown.is_cancelled() { + return; + } + spawn_discovery_dial_candidates( &endpoint, &discovery, @@ -89,6 +97,11 @@ pub(crate) async fn run_native_discovery_dialer( .await; tokio::select! { + biased; + // Endpoint shutdown cancels this token. The dialer holds an endpoint + // clone, so the supervisor registration watch never closes on its + // own; this is the only reliable teardown signal. + _ = shutdown.cancelled() => return, joined = workers.join_next(), if !workers.is_empty() => { match joined { Some(Ok(worker_result)) => { diff --git a/zebra-network/src/zakura/discovery/dialer.rs b/zebra-network/src/zakura/discovery/dialer.rs index 461c78cc829..6904d6fed65 100644 --- a/zebra-network/src/zakura/discovery/dialer.rs +++ b/zebra-network/src/zakura/discovery/dialer.rs @@ -11,13 +11,17 @@ use crate::zakura::{ }; /// Spawn supervised dials for configured native bootstrap peers. +/// +/// Returns one task handle per configured peer so the caller can track them +/// under the endpoint shutdown owner; each maintained dial also observes the +/// endpoint shutdown token directly via [`native_dial_supervised`]. pub(crate) fn spawn_native_bootstrap_dialer( endpoint: ZakuraEndpoint, bootstrap_peers: Vec, limits: ZakuraLocalLimits, -) { +) -> Vec> { if bootstrap_peers.is_empty() { - return; + return Vec::new(); } // Configured bootstrap peers are maintained: keep re-dialing forever so a @@ -30,16 +34,18 @@ pub(crate) fn spawn_native_bootstrap_dialer( DEFAULT_ZAKURA_REDIAL_MAX_BACKOFF, ); + let mut tasks = Vec::with_capacity(bootstrap_peers.len()); for entry in bootstrap_peers { let endpoint = endpoint.clone(); let limits = limits.clone(); - tokio::spawn(async move { + tasks.push(tokio::spawn(async move { match parse_bootstrap_peer(&entry) { Ok(node_addr) => native_dial_supervised(endpoint, node_addr, limits, policy).await, Err(error) => tracing::warn!(?error, ?entry, "invalid Zakura bootstrap peer"), } - }); + })); } + tasks } pub(crate) async fn native_bootstrap_dial( diff --git a/zebra-network/src/zakura/discovery/mod.rs b/zebra-network/src/zakura/discovery/mod.rs index 2a099758c3f..30e0d16c369 100644 --- a/zebra-network/src/zakura/discovery/mod.rs +++ b/zebra-network/src/zakura/discovery/mod.rs @@ -2,6 +2,7 @@ mod candidate_dialer; mod dialer; +mod pipe; mod protocol; mod redial; mod runtime; diff --git a/zebra-network/src/zakura/discovery/pipe.rs b/zebra-network/src/zakura/discovery/pipe.rs new file mode 100644 index 00000000000..cbb8af84815 --- /dev/null +++ b/zebra-network/src/zakura/discovery/pipe.rs @@ -0,0 +1,284 @@ +//! discovery/pipe.rs — the per-peer discovery pipe (stream 4). +//! +//! THE PHASE-3B DAG SLICE IS THIS DIAGRAM. The code below is a mechanical +//! transcription; the [`PIPE_SHAPE`] const is the inspectable, drift-checked +//! copy of it. +//! +//! recv ─▶ guard ─▶ decode ─▶ branch(msg) +//! ├─ Hello ─▶ handoff_async +//! ├─ GetPeers ─▶ handoff_async +//! ├─ Peers ─▶ handoff_async +//! ├─ GetServices ─▶ handoff_async +//! └─ Services ─▶ handoff_async +//! +//! Discovery's semantic handlers perform real async runtime work and may send +//! responses on this peer's stream. Phase 3b therefore keeps those handlers in +//! `service.rs`: this pipe owns the shared guard/decode/branch traversal, then +//! stores the decoded message in peer-local state for the async runner to take +//! and handle immediately after `Pipe::run_one`. + +use super::protocol::{DiscoveryMessage, MAX_DISCOVERY_MESSAGE_BYTES}; +use crate::zakura::{ + Edge, Flow, Frame, Node, NodeKind, Pipe, PipeCx, PipeShape, SessionGuard, SinkReject, + ZakuraPeerId, +}; + +/// Frame message type carrying a discovery payload (matches the native wire). +pub(super) const DISCOVERY_FRAME_MESSAGE_TYPE: u16 = 1; + +const DISCOVERY_ALLOWED_FRAME_TYPES: &[u8] = &[1]; + +/// Per-peer discovery local state. +pub(super) struct DsLocal { + decoded: Option, +} + +impl DsLocal { + fn new() -> Self { + Self { decoded: None } + } + + fn set_decoded(&mut self, msg: DiscoveryMessage) { + self.decoded = Some(msg); + } + + /// Take the decoded message handed off by the synchronous pipe traversal. + pub(super) fn take_decoded(&mut self) -> Option { + self.decoded.take() + } +} + +/// Shared discovery pipe environment. +#[derive(Clone)] +pub(super) struct DsEnv; + +/// The Phase-3b discovery pipe DAG slice, as checked documentation. +pub(super) const PIPE_SHAPE: PipeShape = PipeShape { + service: "discovery", + nodes: &[ + Node { + id: "guard", + kind: NodeKind::Guard, + }, + Node { + id: "decode", + kind: NodeKind::Decode, + }, + Node { + id: "branch", + kind: NodeKind::Branch, + }, + Node { + id: "handoff_async", + kind: NodeKind::Emit, + }, + ], + edges: &[ + Edge { + from: "guard", + to: "decode", + on: "Pass", + }, + Edge { + from: "decode", + to: "branch", + on: "Ok", + }, + Edge { + from: "branch", + to: "handoff_async", + on: "Hello", + }, + Edge { + from: "branch", + to: "handoff_async", + on: "GetPeers", + }, + Edge { + from: "branch", + to: "handoff_async", + on: "Peers", + }, + Edge { + from: "branch", + to: "handoff_async", + on: "GetServices", + }, + Edge { + from: "branch", + to: "handoff_async", + on: "Services", + }, + ], +}; + +/// Build one peer-owned discovery pipe. +pub(super) fn discovery_pipe(peer_id: ZakuraPeerId) -> Pipe { + Pipe::new( + peer_id, + DsLocal::new(), + DsEnv, + // The transport already applies the connection-global count bucket. + // Discovery's service guard owns the stream-4 frame type boundary and + // the payload cap; protocol-message variants remain payload-level + // decode decisions because discovery uses one frame envelope type. + SessionGuard::new( + DISCOVERY_ALLOWED_FRAME_TYPES, + // `MAX_DISCOVERY_MESSAGE_BYTES` is 16 KiB, so it fits in `u32`. + MAX_DISCOVERY_MESSAGE_BYTES as u32, + None, + ), + run_inbound, + &PIPE_SHAPE, + ) +} + +/// Executable transcription of [`PIPE_SHAPE`] — the synchronous pipe entry. +pub(super) fn run_inbound(cx: &mut PipeCx<'_, DsLocal, DsEnv>, frame: Frame) -> Flow<()> { + let msg = match decode(frame) { + Flow::Continue(msg) => msg, + Flow::Done => return Flow::Done, + Flow::Reject(reject) => return Flow::Reject(reject), + }; + + match &msg { + DiscoveryMessage::Hello { .. } + | DiscoveryMessage::GetPeers { .. } + | DiscoveryMessage::Peers { .. } + | DiscoveryMessage::GetServices(_) + | DiscoveryMessage::Services(_) => { + cx.local.set_decoded(msg); + Flow::Continue(()) + } + } +} + +fn decode(frame: Frame) -> Flow { + match decode_discovery_frame(&frame) { + Ok(msg) => Flow::Continue(msg), + Err(error) => Flow::Reject(SinkReject::protocol(error)), + } +} + +/// Decodes a discovery message from a transport frame, rejecting a frame whose +/// envelope is not a discovery payload. +pub(super) fn decode_discovery_frame(frame: &Frame) -> Result { + if frame.message_type != DISCOVERY_FRAME_MESSAGE_TYPE || frame.flags != 0 { + return Err(format!( + "unexpected discovery frame envelope (message_type={}, flags={})", + frame.message_type, frame.flags + ) + .into()); + } + DiscoveryMessage::decode(&frame.payload).map_err(Into::into) +} + +#[cfg(test)] +mod tests { + use super::*; + + const MESSAGE_BRANCHES: [&str; 5] = ["Hello", "GetPeers", "Peers", "GetServices", "Services"]; + + fn peer() -> ZakuraPeerId { + ZakuraPeerId::new(vec![4; 32]).expect("test peer id is within bounds") + } + + /// A valid discovery frame decodes and is handed off via peer-local state for + /// the async runner to take. + #[test] + fn run_one_hands_off_decoded_message() { + let mut pipe = discovery_pipe(peer()); + let payload = DiscoveryMessage::GetPeers { + limit: 8, + wanted_services: Vec::new(), + exclude_node_ids: Vec::new(), + } + .encode() + .expect("message encodes"); + + let flow = pipe.run_one(Frame { + message_type: DISCOVERY_FRAME_MESSAGE_TYPE, + flags: 0, + payload, + }); + + assert!(matches!(flow, Flow::Continue(()))); + assert!( + matches!( + pipe.local_mut().take_decoded(), + Some(DiscoveryMessage::GetPeers { .. }) + ), + "the decoded message is handed off to the async runner" + ); + } + + /// A frame whose envelope type is not the discovery type is rejected by the + /// guard before decode, and nothing is handed off. + #[test] + fn run_one_rejects_non_discovery_frame_type() { + let mut pipe = discovery_pipe(peer()); + + let flow = pipe.run_one(Frame { + message_type: DISCOVERY_FRAME_MESSAGE_TYPE + 1, + flags: 0, + payload: Vec::new(), + }); + + assert!(matches!(flow, Flow::Reject(SinkReject::Protocol(_)))); + assert!(pipe.local_mut().take_decoded().is_none()); + } + + /// A discovery-typed frame with non-zero flags passes the guard but is + /// rejected at decode, and nothing is handed off. + #[test] + fn run_one_rejects_unexpected_frame_flags() { + let mut pipe = discovery_pipe(peer()); + + let flow = pipe.run_one(Frame { + message_type: DISCOVERY_FRAME_MESSAGE_TYPE, + flags: 1, + payload: Vec::new(), + }); + + assert!(matches!(flow, Flow::Reject(SinkReject::Protocol(_)))); + assert!(pipe.local_mut().take_decoded().is_none()); + } + + #[test] + fn pipe_shape_matches_runtime() { + PIPE_SHAPE + .validate() + .expect("discovery PIPE_SHAPE edges name only real nodes"); + + // The runtime branch is the decoded discovery message variant; the + // exhaustive non-wildcard `match &msg` in `run_inbound` is what fails the + // build if a variant is added without a branch, so the shape only needs to + // carry one edge per current variant. + let branches: Vec<&str> = PIPE_SHAPE + .edges + .iter() + .filter(|edge| edge.from == "branch") + .map(|edge| edge.on) + .collect(); + + assert_eq!( + branches.len(), + MESSAGE_BRANCHES.len(), + "branch has exactly one edge per decoded discovery message variant" + ); + for branch in MESSAGE_BRANCHES { + assert!( + branches.contains(&branch), + "branch edge missing for runtime message variant {branch}" + ); + } + + assert!( + PIPE_SHAPE + .edges + .iter() + .any(|edge| edge.from == "guard" && edge.to == "decode"), + "admitted frames decode before branch" + ); + } +} diff --git a/zebra-network/src/zakura/discovery/protocol.rs b/zebra-network/src/zakura/discovery/protocol.rs index 4b318d7af92..ba853a73a1c 100644 --- a/zebra-network/src/zakura/discovery/protocol.rs +++ b/zebra-network/src/zakura/discovery/protocol.rs @@ -21,8 +21,9 @@ use zebra_chain::{ }; use crate::zakura::{ - ServiceAdmissionDecision, ServicePeerDirection, ServicePeerLimits, ServicePeerSnapshot, - ZakuraNetworkId, ZakuraPeerId, + BlockSyncStatus, ServiceAdmissionDecision, ServicePeerDirection, ServicePeerLimits, + ServicePeerSnapshot, ZakuraNetworkId, ZakuraPeerId, MAX_BS_BLOCKS_PER_REQUEST, + MAX_BS_RESPONSE_BYTES, }; /// Native discovery stream kind. @@ -88,6 +89,8 @@ pub const DEFAULT_LIVE_SERVICE_SUMMARY_TTL: Duration = Duration::from_secs(30); pub const SERVICE_ID_DISCOVERY: &str = "zakura.discovery.v1"; /// Native header-sync service id. pub const SERVICE_ID_HEADER_SYNC: &str = "zakura.header_sync.v1"; +/// Native block-sync service id. +pub const SERVICE_ID_BLOCK_SYNC: &str = "zakura.block_sync.v1"; /// Native legacy gossip service id. pub const SERVICE_ID_LEGACY_GOSSIP: &str = "zakura.legacy_gossip.v1"; /// Native legacy requests service id. @@ -98,11 +101,8 @@ pub const SERVICE_ID_SERVICE_DISCOVERY: &str = "zakura.service_discovery.v1"; pub const SUMMARY_TAG_HEADER_SYNC_V1: u16 = 1; /// Summary tag for [`DiscoveryServiceSummary`]. pub const SUMMARY_TAG_DISCOVERY_V1: u16 = 2; -/// Reserved summary tag for a future block-sync summary. -/// -/// The block-sync reactor does not exist yet, so this tag is intentionally -/// skippable and has no typed payload in this version. -pub const SUMMARY_TAG_BLOCK_SYNC_V1_RESERVED: u16 = 3; +/// Summary tag for [`BlockSyncServiceSummary`]. +pub const SUMMARY_TAG_BLOCK_SYNC_V1: u16 = 3; const SIGNATURE_BYTES: usize = 64; const NODE_ID_BYTES: usize = 32; @@ -140,6 +140,12 @@ impl ZakuraServiceId { .expect("built-in Zakura header-sync service id is non-empty bounded ASCII") } + /// Returns the native block-sync service id. + pub fn block_sync() -> Self { + Self::new(SERVICE_ID_BLOCK_SYNC) + .expect("built-in Zakura block-sync service id is non-empty bounded ASCII") + } + /// Returns the native legacy gossip service id. pub fn legacy_gossip() -> Self { Self::new(SERVICE_ID_LEGACY_GOSSIP) @@ -336,6 +342,17 @@ impl ServiceSummaryEnvelope { }) } + /// Build a block-sync summary envelope. + pub fn block_sync(summary: &BlockSyncServiceSummary) -> Result { + let mut summary_bytes = Vec::new(); + encode_block_sync_summary(summary, &mut summary_bytes)?; + Ok(Self { + service_id: ZakuraServiceId::block_sync(), + summary_tag: SUMMARY_TAG_BLOCK_SYNC_V1, + summary_bytes, + }) + } + /// Decode this envelope as a header-sync summary when its tag matches. pub fn decode_header_sync( &self, @@ -355,6 +372,15 @@ impl ServiceSummaryEnvelope { Ok(None) } } + + /// Decode this envelope as a block-sync summary when its tag matches. + pub fn decode_block_sync(&self) -> Result, DiscoveryWireError> { + if self.summary_tag == SUMMARY_TAG_BLOCK_SYNC_V1 { + decode_block_sync_summary(&self.summary_bytes).map(Some) + } else { + Ok(None) + } + } } /// Header-sync live summary used only as a dial/admission hint. @@ -434,6 +460,70 @@ impl DiscoveryServiceSummary { } } +/// Block-sync live summary used only as a dial/admission hint. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct BlockSyncServiceSummary { + /// Earliest committed body height this node can serve. + pub servable_low: block::Height, + /// Highest committed body height this node can serve. + pub servable_high: block::Height, + /// Hash of [`servable_high`](Self::servable_high). + pub tip_hash: block::Hash, + /// Free serving slots for inbound `GetBlocks` requests. + pub free_slots: u16, + /// Maximum blocks this node will return in one block-sync response. + pub max_blocks_per_response: u32, + /// Maximum encoded bytes this node will return in one block-sync response. + pub max_response_bytes: u32, +} + +impl BlockSyncServiceSummary { + /// Build a summary from the block-sync reactor's cheap local status and peer snapshots. + pub fn from_status_and_snapshot(status: BlockSyncStatus, peers: ServicePeerSnapshot) -> Self { + Self { + servable_low: status.servable_low, + servable_high: status.servable_high, + tip_hash: status.tip_hash, + free_slots: saturating_u16(peers.inbound_slots_free), + max_blocks_per_response: status.max_blocks_per_response, + max_response_bytes: status.max_response_bytes, + } + } + + /// Returns true if this summary can serve every height in `gap`. + pub fn covers_gap(&self, gap: &[block::Height]) -> bool { + self.gap_coverage(gap) == BlockSyncGapCoverage::Whole + } + + /// Returns how much of `gap` this summary can serve. + fn gap_coverage(&self, gap: &[block::Height]) -> BlockSyncGapCoverage { + if self.free_slots == 0 { + return BlockSyncGapCoverage::None; + } + + let Some(first) = gap.iter().min().copied() else { + return BlockSyncGapCoverage::None; + }; + let Some(last) = gap.iter().max().copied() else { + return BlockSyncGapCoverage::None; + }; + if self.servable_low <= first && last <= self.servable_high { + BlockSyncGapCoverage::Whole + } else if self.servable_low <= first && first <= self.servable_high { + BlockSyncGapCoverage::LowPrefix + } else { + BlockSyncGapCoverage::None + } + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +enum BlockSyncGapCoverage { + None, + LowPrefix, + Whole, +} + /// Native discovery protocol messages. #[derive(Clone, Debug, Eq, PartialEq)] pub enum DiscoveryMessage { @@ -791,9 +881,9 @@ pub struct ZakuraDiscoveryPersistedEntry { /// A locally dialable discovery candidate. /// -/// Candidates may come from signed discovery records or from trusted static bootstrap -/// configuration. Only signed records are eligible for peer samples; unsigned static candidates are -/// local dial hints. +/// Candidates may come from first-party confirmed signed discovery records or from trusted static +/// bootstrap configuration. Only signed records are eligible for peer samples; unsigned static +/// candidates are local dial hints. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ZakuraDiscoveryDialCandidate { /// Candidate iroh node id. @@ -823,7 +913,7 @@ pub struct ZakuraServiceCandidates { } /// Header-sync candidate selection hints owned by the header-sync reactor. -#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct ZakuraHeaderSyncCandidateState { /// Lowest header height that would make a new peer useful. pub target_height: block::Height, @@ -833,6 +923,25 @@ pub struct ZakuraHeaderSyncCandidateState { pub backed_off_node_ids: Vec, } +impl Default for ZakuraHeaderSyncCandidateState { + fn default() -> Self { + Self { + target_height: block::Height::MIN, + admitted_node_ids: Vec::new(), + backed_off_node_ids: Vec::new(), + } + } +} + +/// Block-sync candidate selection hints owned by the block-sync reactor. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ZakuraBlockSyncCandidateState { + /// Header-known body heights currently missing from local state. + pub missing_block_bodies: Vec, + /// Peers already admitted by block sync; advisory "full" summaries do not remove them. + pub admitted_node_ids: Vec, +} + /// The result of importing one signed discovery record. #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum ImportOutcome { @@ -1024,6 +1133,39 @@ pub struct ZakuraDiscoveryHandle { self_record_tx: watch::Sender>, peer_snapshot: watch::Receiver, peer_snapshot_tx: watch::Sender, + import_validation: DiscoveryImportValidation, +} + +/// Immutable inputs needed to validate and bound a peer record batch without holding the +/// global discovery mutex. +/// +/// These mirror the fields [`ZakuraDiscoveryInner::validation_context`] reads, all of which +/// are fixed at construction (local identity/network parameters and import limits never +/// change at runtime). Snapshotting them on the handle lets [`ZakuraDiscoveryHandle::import_peer_records`] +/// run CPU-heavy Ed25519 verification outside the lock. +#[derive(Clone, Debug)] +struct DiscoveryImportValidation { + expected_network_id: ZakuraNetworkId, + expected_chain_id: [u8; 32], + supported_protocol_min: u16, + supported_protocol_max: u16, + max_record_ttl: Duration, + clock_skew_tolerance: Duration, + max_imported_records_per_response: usize, +} + +impl DiscoveryImportValidation { + fn context(&self, now: u64) -> DiscoveryRecordValidationContext { + DiscoveryRecordValidationContext { + expected_network_id: self.expected_network_id, + expected_chain_id: self.expected_chain_id, + current_unix_secs: now, + supported_protocol_min: self.supported_protocol_min, + supported_protocol_max: self.supported_protocol_max, + max_record_ttl: self.max_record_ttl, + clock_skew_tolerance: self.clock_skew_tolerance, + } + } } impl fmt::Debug for ZakuraDiscoveryHandle { @@ -1135,7 +1277,25 @@ impl ZakuraActiveServiceEntry { } match summary.summary { ZakuraLiveServiceSummary::HeaderSync(header_sync) => Some(header_sync), - ZakuraLiveServiceSummary::Discovery(_) | ZakuraLiveServiceSummary::Unknown => None, + ZakuraLiveServiceSummary::BlockSync(_) + | ZakuraLiveServiceSummary::Discovery(_) + | ZakuraLiveServiceSummary::Unknown => None, + } + }) + } + + fn fresh_block_sync_summary(&self, now_unix_secs: u64) -> Option { + self.live_summaries.iter().find_map(|summary| { + if summary.service_id != ZakuraServiceId::block_sync() + || summary.expires_at_unix_secs <= now_unix_secs + { + return None; + } + match summary.summary { + ZakuraLiveServiceSummary::BlockSync(block_sync) => Some(block_sync), + ZakuraLiveServiceSummary::HeaderSync(_) + | ZakuraLiveServiceSummary::Discovery(_) + | ZakuraLiveServiceSummary::Unknown => None, } }) } @@ -1166,6 +1326,8 @@ impl ZakuraCachedLiveServiceSummary { ZakuraLiveServiceSummary::HeaderSync(summary) } else if let Some(summary) = envelope.decode_discovery()? { ZakuraLiveServiceSummary::Discovery(summary) + } else if let Some(summary) = envelope.decode_block_sync()? { + ZakuraLiveServiceSummary::BlockSync(summary) } else { ZakuraLiveServiceSummary::Unknown }; @@ -1190,6 +1352,12 @@ impl ZakuraCachedLiveServiceSummary { ZakuraLiveServiceSummary::Discovery(summary) => { u32::from(summary.peer_exchange_slots_free) } + ZakuraLiveServiceSummary::BlockSync(summary) => u32::from(summary.free_slots) + .saturating_add( + summary + .max_blocks_per_response + .min(MAX_BS_BLOCKS_PER_REQUEST), + ), ZakuraLiveServiceSummary::Unknown => 1, } } @@ -1202,6 +1370,8 @@ pub enum ZakuraLiveServiceSummary { HeaderSync(HeaderSyncServiceSummary), /// Discovery exchange hint. Discovery(DiscoveryServiceSummary), + /// Block-sync dial/admission hint. + BlockSync(BlockSyncServiceSummary), /// Unknown or reserved summary tag, kept only as a freshness hint. Unknown, } @@ -1268,6 +1438,15 @@ impl ZakuraDiscoveryHandle { let (peer_snapshot_tx, peer_snapshot_rx) = watch::channel(ServicePeerSnapshot::new(0, 0, config.peer_limits)); let book = ZakuraDiscoveryBook::with_local_node_id(config.book_limits, local.node_id); + let import_validation = DiscoveryImportValidation { + expected_network_id: local.network_id, + expected_chain_id: local.chain_id, + supported_protocol_min: local.zakura_protocol_min, + supported_protocol_max: local.zakura_protocol_max, + max_record_ttl: config.max_record_ttl, + clock_skew_tolerance: config.clock_skew_tolerance, + max_imported_records_per_response: config.book_limits.max_imported_records_per_response, + }; Ok(Self { inner: Arc::new(Mutex::new(ZakuraDiscoveryInner { book, @@ -1282,6 +1461,7 @@ impl ZakuraDiscoveryHandle { self_record_tx, peer_snapshot: peer_snapshot_rx, peer_snapshot_tx, + import_validation, }) } @@ -1444,15 +1624,47 @@ impl ZakuraDiscoveryHandle { } /// Imports peer-supplied signed records through the discovery book validation path. + /// + /// Record signatures are verified OUTSIDE the global discovery mutex. Ed25519 + /// verification and signature-domain re-encoding are CPU-heavy and fully + /// attacker-driven (an authenticated discovery peer can send a full `Peers` batch of + /// records with valid or invalid signatures, repeatedly); running them while holding + /// the shared async lock lets that peer stall unrelated discovery admission, + /// sampling, and dial selection. Only the cheap book mutation (storage-limit recheck, + /// address policy, map insert, eviction) runs under the lock, and the lock is skipped + /// entirely when nothing survives verification. See finding + /// `claude-discovery-expensive-work-under-global-mutex` (SR-2). pub async fn import_peer_records( &self, records: impl IntoIterator, source: Option, ) -> ImportBatchOutcome { let now = current_unix_secs(); - let mut inner = self.inner.lock().await; - let context = inner.validation_context(now); - inner.book.import_records(records, source, now, &context) + let context = self.import_validation.context(now); + let max_records = self.import_validation.max_imported_records_per_response; + + let mut outcome = ImportBatchOutcome::default(); + let mut verified = Vec::new(); + for record in records { + if outcome.attempted >= max_records { + outcome.dropped_for_limit += 1; + continue; + } + outcome.attempted += 1; + match record.verify(&context) { + Ok(()) => verified.push(record), + Err(_) => outcome.rejected += 1, + } + } + + if !verified.is_empty() { + let mut inner = self.inner.lock().await; + inner + .book + .import_pre_verified_records(verified, source, now, &context, &mut outcome); + } + + outcome } /// Imports one peer-supplied signed record through the discovery book validation path. @@ -1877,6 +2089,114 @@ impl ZakuraDiscoveryHandle { } } + /// Returns block-sync candidates filtered and ordered by first-party advisory summaries. + /// + /// The returned peers are still advisory only: block-sync `Status`, body hash validation, and + /// consensus verification remain authoritative after admission. + pub async fn block_sync_candidates( + &self, + block_sync: &ZakuraBlockSyncCandidateState, + allow_fallback: bool, + in_flight_node_ids: &[NodeId], + ) -> ZakuraServiceCandidates { + let connected = self.connected.borrow().clone(); + let connected_node_ids = connected_peer_node_ids(&connected); + let (limit, dial_backoff_base, dial_backoff_max, book_limits) = { + let inner = self.inner.lock().await; + ( + discovery_dial_slot_limit( + connected.len(), + in_flight_node_ids.len(), + inner.config.max_zakura_connections, + inner.config.discovery_connection_headroom, + inner.config.max_concurrent_discovery_dials, + ), + inner.config.dial_backoff_base, + inner.config.dial_backoff_max, + inner.config.book_limits, + ) + }; + + let now = current_unix_secs(); + let mut inner = self.inner.lock().await; + inner.sync_active_services(&connected_node_ids, now); + + let admitted_node_ids: HashSet<_> = block_sync.admitted_node_ids.iter().copied().collect(); + let block_sync_service = ZakuraServiceId::block_sync(); + let mut connected_entries: Vec<_> = inner + .active_services + .iter() + .filter_map(|(node_id, entry)| { + if !connected_node_ids.contains(node_id) + || !entry.services.contains(&block_sync_service) + { + return None; + } + + if block_sync.missing_block_bodies.is_empty() { + return Some(( + *node_id, + entry.live_summary_preference(&block_sync_service, now), + node_id_sort_key(node_id), + )); + } + + let is_admitted = admitted_node_ids.contains(node_id); + let summary = entry.fresh_block_sync_summary(now); + if summary.is_some_and(|summary| summary.free_slots == 0 && !is_admitted) { + return None; + } + + Some(( + *node_id, + block_sync_candidate_preference(summary, &block_sync.missing_block_bodies), + node_id_sort_key(node_id), + )) + }) + .collect(); + connected_entries + .sort_by_key(|(_, preference, sort_key)| (Reverse(*preference), *sort_key)); + let connected: Vec = connected_entries + .into_iter() + .map(|(node_id, _, _)| node_id) + .collect(); + + let wanted_services = + bounded_services(std::slice::from_ref(&block_sync_service), book_limits); + let mut rng = rand::thread_rng(); + let mut discovered = inner.book.dial_candidates( + limit, + &wanted_services, + DialCandidateExclusions { + connected_node_ids: &connected_node_ids, + in_flight_node_ids, + }, + now, + (dial_backoff_base, dial_backoff_max), + &mut rng, + ); + let used_fallback = allow_fallback && connected.is_empty() && discovered.is_empty(); + if used_fallback { + discovered = inner.book.dial_candidates( + limit, + &[], + DialCandidateExclusions { + connected_node_ids: &connected_node_ids, + in_flight_node_ids, + }, + now, + (dial_backoff_base, dial_backoff_max), + &mut rng, + ); + } + + ZakuraServiceCandidates { + connected, + discovered, + used_fallback, + } + } + /// Marks a discovery dial attempt for `node_id`. pub async fn mark_dial_attempt(&self, node_id: &NodeId) { let mut inner = self.inner.lock().await; @@ -2178,7 +2498,7 @@ impl ZakuraDiscoveryBook { now: u64, context: &DiscoveryRecordValidationContext, ) -> Result { - self.import_record_inner(record, source, false, now, context) + self.import_record_inner(record, source, false, false, now, context) } /// Imports one signed static/bootstrap record. @@ -2193,7 +2513,30 @@ impl ZakuraDiscoveryBook { now: u64, context: &DiscoveryRecordValidationContext, ) -> Result { - self.import_record_inner(record, None, true, now, context) + self.import_record_inner(record, None, true, false, now, context) + } + + /// Imports a batch of records whose signatures were already verified outside the lock. + /// + /// [`ZakuraDiscoveryHandle::import_peer_records`] performs signature verification and + /// applies the per-response cap before taking the global discovery mutex, so this only + /// runs the cheap storage-limit recheck, address-policy check, map mutation, and + /// eviction under the lock. `attempted`/`dropped_for_limit` are owned by the caller; + /// this updates the per-record success and rejection tallies on `outcome`. + fn import_pre_verified_records( + &mut self, + records: impl IntoIterator, + source: Option, + now: u64, + context: &DiscoveryRecordValidationContext, + outcome: &mut ImportBatchOutcome, + ) { + for record in records { + match self.import_record_inner(record, source, false, true, now, context) { + Ok(import_outcome) => outcome.record_success(import_outcome), + Err(_) => outcome.rejected += 1, + } + } } /// Imports a bounded batch of signed peer records from a single response. @@ -2231,6 +2574,12 @@ impl ZakuraDiscoveryBook { let exclude_node_ids: HashSet<_> = exclude_node_ids.iter().copied().collect(); let limit = limit.min(self.limits.max_imported_records_per_response); + // Reservoir-sample references and clone only the chosen records. The book can hold + // `max_records` (default 10_000) entries, each up to `max_encoded_record_bytes`, so cloning + // every record that passes the filter — when at most `limit` (default 32) are ever returned + // — was attacker-paced, allocation-heavy work performed while holding the global discovery + // mutex. Per-call clone work is now bounded by `limit`, not the book size. See finding + // `claude-discovery-expensive-work-under-global-mutex` (SR-2/SR-4). self.entries .iter() .filter(|(node_id, entry)| { @@ -2240,8 +2589,11 @@ impl ZakuraDiscoveryBook { && has_wanted_services(&entry.record, wanted_services) && has_discovery_dialable_direct_addrs(&entry.record) }) - .map(|(_, entry)| entry.record.clone()) + .map(|(_, entry)| &entry.record) .choose_multiple(rng, limit) + .into_iter() + .cloned() + .collect() } /// Returns bounded dial candidates for later dial-loop code. @@ -2258,7 +2610,7 @@ impl ZakuraDiscoveryBook { exclusions.connected_node_ids.iter().copied().collect(); let in_flight_node_ids: HashSet<_> = exclusions.in_flight_node_ids.iter().copied().collect(); - let mut candidates: Vec<_> = self + let candidates: Vec<_> = self .entries .values() .filter(|entry| { @@ -2273,6 +2625,7 @@ impl ZakuraDiscoveryBook { dial_backoff.0, dial_backoff.1, ) + && entry_has_confirmed_dial_authority(entry) && has_wanted_services(&entry.record, wanted_services) && has_discovery_usable_direct_addrs(entry) }) @@ -2303,31 +2656,28 @@ impl ZakuraDiscoveryBook { })) .collect(); - candidates.sort_by_cached_key(|entry| { - let non_static_random_tie = if !entry.is_static() { - rng.gen::() - } else { - 0 - }; - let static_deterministic_tie = if entry.is_static() { - node_id_sort_key(&entry.node_id()) - } else { - [0; NODE_ID_BYTES] - }; - ( - !entry.is_static(), - Reverse(entry.last_success().unwrap_or(0)), - entry.failure_count(), - Reverse(entry.last_seen()), - non_static_random_tie, - static_deterministic_tie, - ) - }); + // Bounded top-k selection instead of a full sort. The book can hold `max_records` + // (default 10_000) entries and this selection runs under the global discovery mutex on the + // per-second candidate-dialer path, yet only `limit` candidates are ever returned. Sorting + // the whole candidate set (O(n log n)) just to `take(limit)` is replaced with an O(n) + // partial select of the best `limit` candidates plus an O(limit log limit) sort of the + // survivors, keeping the order identical. See finding + // `claude-discovery-expensive-work-under-global-mutex` (SR-2/SR-4). + let mut keyed: Vec<(DialCandidateSortKey, DialCandidateRef<'_>)> = candidates + .into_iter() + .map(|candidate| (dial_candidate_sort_key(&candidate, rng), candidate)) + .collect(); + + let take = limit.min(keyed.len()); + if take < keyed.len() { + keyed.select_nth_unstable_by(take, |a, b| a.0.cmp(&b.0)); + keyed.truncate(take); + } + keyed.sort_by(|a, b| a.0.cmp(&b.0)); - candidates + keyed .into_iter() - .take(limit) - .map(DialCandidateRef::into_candidate) + .map(|(_, candidate)| candidate.into_candidate()) .collect() } @@ -2446,7 +2796,7 @@ impl ZakuraDiscoveryBook { last_success: entry.last_success, failure_count: entry.failure_count, }; - self.import_validated_record(entry.record, metadata, now, context) + self.import_validated_record(entry.record, metadata, false, now, context) } fn import_record_inner( @@ -2454,6 +2804,7 @@ impl ZakuraDiscoveryBook { record: ZakuraNodeRecord, source: Option, is_static: bool, + pre_verified: bool, now: u64, context: &DiscoveryRecordValidationContext, ) -> Result { @@ -2465,13 +2816,20 @@ impl ZakuraDiscoveryBook { last_success: None, failure_count: 0, }; - self.import_validated_record(record, metadata, now, context) + self.import_validated_record(record, metadata, pre_verified, now, context) } + /// Validates and stores one record. + /// + /// When `pre_verified` is set the signature/import-context check has already been run + /// outside the global discovery mutex by [`ZakuraDiscoveryHandle::import_peer_records`], + /// so it is not repeated here; the cheap storage-limit, address-policy, insertion, and + /// eviction steps still run under the caller's lock. fn import_validated_record( &mut self, record: ZakuraNodeRecord, metadata: DiscoveryEntryMetadata, + pre_verified: bool, now: u64, context: &DiscoveryRecordValidationContext, ) -> Result { @@ -2479,7 +2837,9 @@ impl ZakuraDiscoveryBook { return Err(DiscoveryBookError::SelfRecord); } - record.verify(context)?; + if !pre_verified { + record.verify(context)?; + } self.validate_record_storage_limits(&record)?; let direct_addr_policy = if metadata.is_static { DiscoveryDirectAddrPolicy::StaticConfigured @@ -2685,6 +3045,48 @@ impl DialCandidateRef<'_> { } } +/// Total dial-priority key for one candidate: prefer signed records over static ones, then most +/// recent success, fewest failures, most recently seen, with a per-call random tie-break for +/// non-static candidates and a deterministic node-id tie-break for static ones. +type DialCandidateSortKey = ( + bool, + Reverse, + u32, + Reverse, + u64, + [u8; NODE_ID_BYTES], +); + +/// Computes the dial-priority key used to select the best dial candidates. +/// +/// Factored out of [`ZakuraDiscoveryBook::dial_candidates`] so the key can be computed once per +/// candidate and fed to a bounded top-k selection instead of a full sort, which keeps expensive +/// per-book ordering work bounded under the global discovery mutex (finding +/// `claude-discovery-expensive-work-under-global-mutex`). +fn dial_candidate_sort_key( + candidate: &DialCandidateRef<'_>, + rng: &mut R, +) -> DialCandidateSortKey { + let non_static_random_tie = if !candidate.is_static() { + rng.gen::() + } else { + 0 + }; + let static_deterministic_tie = if candidate.is_static() { + node_id_sort_key(&candidate.node_id()) + } else { + [0; NODE_ID_BYTES] + }; + ( + !candidate.is_static(), + Reverse(candidate.last_success().unwrap_or(0)), + candidate.failure_count(), + Reverse(candidate.last_seen()), + non_static_random_tie, + static_deterministic_tie, + ) +} + fn update_existing_entry( entry: &mut ZakuraDiscoveryEntry, record: ZakuraNodeRecord, @@ -2697,7 +3099,13 @@ fn update_existing_entry( return ImportOutcome::IgnoredOlder; } - entry.source = metadata.source; + let preserve_first_party_source = incoming_sequence == stored_sequence + && entry.source == Some(entry.record.body.node_id) + && metadata.source != Some(entry.record.body.node_id); + + if !preserve_first_party_source { + entry.source = metadata.source; + } entry.is_static |= metadata.is_static; entry.last_seen = metadata.last_seen; @@ -2807,6 +3215,29 @@ fn header_sync_candidate_preference( .saturating_add(useful_height_delta) } +fn block_sync_candidate_preference( + summary: Option, + missing_block_bodies: &[block::Height], +) -> u32 { + let Some(summary) = summary else { + return 0; + }; + + let coverage_bonus = match summary.gap_coverage(missing_block_bodies) { + BlockSyncGapCoverage::Whole => 1_000_000u32, + BlockSyncGapCoverage::LowPrefix => 500_000u32, + BlockSyncGapCoverage::None => return 0, + }; + + coverage_bonus + .saturating_add(u32::from(summary.free_slots)) + .saturating_add( + summary + .max_blocks_per_response + .min(MAX_BS_BLOCKS_PER_REQUEST), + ) +} + fn bounded_services( services: &[ZakuraServiceId], limits: ZakuraDiscoveryBookLimits, @@ -2859,9 +3290,10 @@ fn has_wanted_services(record: &ZakuraNodeRecord, wanted_services: &[ZakuraServi /// Validates direct addresses for discovery-book storage. /// -/// Untrusted peer/gossip records must only contain globally dialable addresses, preserving the -/// discovery security rule that gossiped records cannot inject loopback, link-local, multicast, or -/// broadcast targets. Static records are trusted-by-configuration bootstrap records, so they may use +/// Untrusted peer/gossip records must only contain globally routable addresses, preserving the +/// discovery security rule that gossiped records cannot inject loopback, link-local, multicast, +/// broadcast, RFC 1918 private, RFC 6598 shared (CGNAT), or RFC 4193 unique-local targets. Static +/// records are trusted-by-configuration bootstrap records, so they may use /// local addresses for regtest and single-host deployments, but still reject empty, unspecified, and /// port-0 targets that cannot be dialed as configured peers. fn validate_discovery_direct_addrs( @@ -2907,6 +3339,18 @@ fn has_discovery_usable_direct_addrs(entry: &ZakuraDiscoveryEntry) -> bool { }) } +fn entry_has_confirmed_dial_authority(entry: &ZakuraDiscoveryEntry) -> bool { + entry.is_static || entry.source == Some(entry.record.body.node_id) +} + +/// Returns true only for addresses that are safe to dial from untrusted discovery gossip. +/// +/// A signed `ZakuraNodeRecord` proves control of the node key, not ownership of the advertised +/// `direct_addrs`. To stop an authenticated discovery peer from steering the candidate dialer at +/// arbitrary non-public targets, gossiped records must advertise only globally routable addresses. +/// Besides the obvious loopback/link-local/multicast/broadcast cases, this rejects RFC 1918 private, +/// RFC 6598 shared (CGNAT), and RFC 4193 unique-local ranges, which are reachable internal targets +/// the dialer would otherwise scan on the operator's network. fn is_discovery_dialable_addr(addr: &SocketAddr) -> bool { if addr.port() == 0 { return false; @@ -2919,12 +3363,15 @@ fn is_discovery_dialable_addr(addr: &SocketAddr) -> bool { && !ip.is_multicast() && !ip.is_broadcast() && !ip.is_link_local() + && !ip.is_private() + && !is_ipv4_shared(&ip) } IpAddr::V6(ip) => { !ip.is_unspecified() && !ip.is_loopback() && !ip.is_multicast() && !is_ipv6_unicast_link_local(&ip) + && !is_ipv6_unique_local(&ip) } } } @@ -2944,6 +3391,18 @@ fn is_ipv6_unicast_link_local(ip: &Ipv6Addr) -> bool { (ip.segments()[0] & 0xffc0) == 0xfe80 } +// `Ipv4Addr::is_shared` is still unstable, so match the RFC 6598 100.64.0.0/10 range directly, +// mirroring `is_ipv6_unicast_link_local`. +fn is_ipv4_shared(ip: &Ipv4Addr) -> bool { + let octets = ip.octets(); + octets[0] == 100 && (octets[1] & 0b1100_0000) == 0b0100_0000 +} + +// `Ipv6Addr::is_unique_local` is still unstable, so match the RFC 4193 fc00::/7 range directly. +fn is_ipv6_unique_local(ip: &Ipv6Addr) -> bool { + (ip.segments()[0] & 0xfe00) == 0xfc00 +} + // Import accepts records inside the clock-skew window, but runtime liveness is strict. fn entry_is_expired(entry: &ZakuraDiscoveryEntry, now: u64) -> bool { entry.record.body.expires_at_unix_secs < now @@ -3145,7 +3604,7 @@ fn validate_summary_service_binding( let expected = match summary_tag { SUMMARY_TAG_HEADER_SYNC_V1 => Some(ZakuraServiceId::header_sync()), SUMMARY_TAG_DISCOVERY_V1 => Some(ZakuraServiceId::discovery()), - SUMMARY_TAG_BLOCK_SYNC_V1_RESERVED => None, + SUMMARY_TAG_BLOCK_SYNC_V1 => Some(ZakuraServiceId::block_sync()), _ => None, }; @@ -3173,7 +3632,9 @@ fn validate_known_summary_payload( SUMMARY_TAG_DISCOVERY_V1 => { decode_discovery_summary(summary_bytes)?; } - SUMMARY_TAG_BLOCK_SYNC_V1_RESERVED => {} + SUMMARY_TAG_BLOCK_SYNC_V1 => { + decode_block_sync_summary(summary_bytes)?; + } _ => {} } Ok(()) @@ -3712,26 +4173,91 @@ fn decode_discovery_summary(bytes: &[u8]) -> Result Result<(), DiscoveryWireError> { - writer.write_u32::(height.0)?; +fn encode_block_sync_summary( + summary: &BlockSyncServiceSummary, + writer: &mut impl Write, +) -> Result<(), DiscoveryWireError> { + validate_block_sync_summary(summary)?; + write_height(writer, summary.servable_low)?; + write_height(writer, summary.servable_high)?; + writer.write_all(&summary.tip_hash.0)?; + writer.write_u16::(summary.free_slots)?; + writer.write_u32::(summary.max_blocks_per_response)?; + writer.write_u32::(summary.max_response_bytes)?; Ok(()) } -fn read_height(reader: &mut impl Read) -> Result { - let height = reader.read_u32::()?; - block::Height::try_from(height).map_err(|_| DiscoveryWireError::InvalidHeight(height)) +fn decode_block_sync_summary(bytes: &[u8]) -> Result { + if bytes.len() > MAX_SERVICE_SUMMARY_BYTES { + return Err(DiscoveryWireError::OversizedPayload { + actual: bytes.len(), + max: MAX_SERVICE_SUMMARY_BYTES, + }); + } + let mut reader = Cursor::new(bytes); + let servable_low = read_height(&mut reader)?; + let servable_high = read_height(&mut reader)?; + let mut tip_hash = [0u8; 32]; + reader.read_exact(&mut tip_hash)?; + let free_slots = reader.read_u16::()?; + let max_blocks_per_response = reader.read_u32::()?; + let max_response_bytes = reader.read_u32::()?; + reject_trailing(bytes, &reader)?; + let summary = BlockSyncServiceSummary { + servable_low, + servable_high, + tip_hash: block::Hash(tip_hash), + free_slots, + max_blocks_per_response, + max_response_bytes, + }; + validate_block_sync_summary(&summary)?; + Ok(summary) } -fn write_optional_height( - writer: &mut impl Write, - height: Option, +fn validate_block_sync_summary( + summary: &BlockSyncServiceSummary, ) -> Result<(), DiscoveryWireError> { - match height { - Some(height) => { - writer.write_u8(1)?; - write_height(writer, height)?; - } - None => writer.write_u8(0)?, + if summary.servable_low > summary.servable_high { + return Err(DiscoveryWireError::InvalidProtocolRange); + } + if summary.max_blocks_per_response == 0 + || summary.max_blocks_per_response > MAX_BS_BLOCKS_PER_REQUEST + { + return Err(DiscoveryWireError::OversizedPayload { + actual: usize::try_from(summary.max_blocks_per_response).unwrap_or(usize::MAX), + max: usize::try_from(MAX_BS_BLOCKS_PER_REQUEST).unwrap_or(usize::MAX), + }); + } + if summary.max_response_bytes == 0 || summary.max_response_bytes > MAX_BS_RESPONSE_BYTES { + return Err(DiscoveryWireError::OversizedPayload { + actual: usize::try_from(summary.max_response_bytes).unwrap_or(usize::MAX), + max: usize::try_from(MAX_BS_RESPONSE_BYTES).unwrap_or(usize::MAX), + }); + } + Ok(()) +} + +fn write_height(writer: &mut impl Write, height: block::Height) -> Result<(), DiscoveryWireError> { + writer.write_u32::(height.0)?; + Ok(()) +} + +fn read_height(reader: &mut impl Read) -> Result { + let height = reader.read_u32::()?; + block::Height::try_from(height).map_err(|_| DiscoveryWireError::InvalidHeight(height)) +} + +fn write_optional_height( + writer: &mut impl Write, + height: Option, +) -> Result<(), DiscoveryWireError> { + match height { + Some(height) => { + writer.write_u8(1)?; + write_height(writer, height)?; + } + None => writer.write_u8(0)?, } Ok(()) } @@ -3922,6 +4448,14 @@ mod tests { } } + fn import_confirmed_record( + book: &mut ZakuraDiscoveryBook, + record: ZakuraNodeRecord, + ) -> Result { + let source = record.body.node_id; + book.import_record(record, Some(source), NOW, &context()) + } + fn small_book(max_records: usize) -> ZakuraDiscoveryBook { ZakuraDiscoveryBook::new(ZakuraDiscoveryBookLimits { max_records, @@ -4039,6 +4573,28 @@ mod tests { } } + fn block_sync_summary() -> BlockSyncServiceSummary { + BlockSyncServiceSummary { + servable_low: block::Height(10), + servable_high: block::Height(42), + tip_hash: block::Hash([9; 32]), + free_slots: 3, + max_blocks_per_response: 16, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + } + } + + fn block_sync_status() -> BlockSyncStatus { + BlockSyncStatus { + servable_low: block::Height(10), + servable_high: block::Height(42), + tip_hash: block::Hash([9; 32]), + max_blocks_per_response: 16, + max_inflight_requests: 4, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + } + } + fn services_message(summaries: Vec) -> DiscoveryMessage { DiscoveryMessage::Services(Services { node_id: secret_key().public(), @@ -4079,6 +4635,27 @@ mod tests { )); } + #[test] + fn block_sync_summary_uses_live_peer_slots() { + let limits = ServicePeerLimits { + max_inbound_peers: 2, + ..ServicePeerLimits::default() + }; + let status = block_sync_status(); + + let open = BlockSyncServiceSummary::from_status_and_snapshot( + status, + ServicePeerSnapshot::new(1, 0, limits), + ); + assert_eq!(open.free_slots, 1); + + let full = BlockSyncServiceSummary::from_status_and_snapshot( + status, + ServicePeerSnapshot::new(2, 0, limits), + ); + assert_eq!(full.free_slots, 0); + } + #[test] fn node_record_encode_decode_roundtrip() { let record = signed_record(); @@ -4122,6 +4699,8 @@ mod tests { ServiceSummaryEnvelope::header_sync(&header_summary()).expect("summary encodes"); let discovery_envelope = ServiceSummaryEnvelope::discovery(&discovery_summary()).expect("summary encodes"); + let block_envelope = + ServiceSummaryEnvelope::block_sync(&block_sync_summary()).expect("summary encodes"); let messages = vec![ DiscoveryMessage::Hello { record: record.clone(), @@ -4137,7 +4716,7 @@ mod tests { DiscoveryMessage::GetServices(GetServices { wanted_services: services, }), - services_message(vec![header_envelope, discovery_envelope]), + services_message(vec![header_envelope, discovery_envelope, block_envelope]), ]; for message in messages { @@ -4150,10 +4729,13 @@ mod tests { fn service_summary_envelopes_roundtrip_and_unknown_tags_are_skipped() { let header = header_summary(); let discovery = discovery_summary(); + let block = block_sync_summary(); let header_envelope = ServiceSummaryEnvelope::header_sync(&header).expect("header summary encodes"); let discovery_envelope = ServiceSummaryEnvelope::discovery(&discovery).expect("discovery summary encodes"); + let block_envelope = + ServiceSummaryEnvelope::block_sync(&block).expect("block summary encodes"); let unknown_envelope = ServiceSummaryEnvelope { service_id: service(99), summary_tag: 999, @@ -4165,11 +4747,13 @@ mod tests { discovery_envelope.decode_discovery().unwrap(), Some(discovery) ); + assert_eq!(block_envelope.decode_block_sync().unwrap(), Some(block)); let message = services_message(vec![ unknown_envelope.clone(), header_envelope.clone(), discovery_envelope.clone(), + block_envelope.clone(), ]); let decoded = DiscoveryMessage::decode(&message.encode().expect("message encodes")) .expect("message decodes"); @@ -4186,10 +4770,14 @@ mod tests { services.summaries[2].decode_discovery().unwrap(), Some(discovery) ); + assert_eq!( + services.summaries[3].decode_block_sync().unwrap(), + Some(block) + ); } #[test] - fn service_summary_decode_rejects_lying_and_truncated_lengths() { + fn service_summary_decode_rejects_lying_truncated_and_oversized_lengths() { let node_id = secret_key().public(); let mut lying = vec![MSG_DISCOVERY_SERVICES]; lying.write_all(node_id.as_bytes()).unwrap(); @@ -4217,6 +4805,39 @@ mod tests { truncated_known.write_u8(0).unwrap(); assert!(DiscoveryMessage::decode(&truncated_known).is_err()); + + let mut truncated_block = vec![MSG_DISCOVERY_SERVICES]; + truncated_block.write_all(node_id.as_bytes()).unwrap(); + truncated_block.write_u64::(NOW + 30).unwrap(); + truncated_block.write_u16::(1).unwrap(); + encode_service_id(&ZakuraServiceId::block_sync(), &mut truncated_block).unwrap(); + truncated_block + .write_u16::(SUMMARY_TAG_BLOCK_SYNC_V1) + .unwrap(); + truncated_block.write_u16::(8).unwrap(); + write_height(&mut truncated_block, block::Height(10)).unwrap(); + write_height(&mut truncated_block, block::Height(42)).unwrap(); + + assert!(DiscoveryMessage::decode(&truncated_block).is_err()); + + let mut oversized = vec![MSG_DISCOVERY_SERVICES]; + oversized.write_all(node_id.as_bytes()).unwrap(); + oversized.write_u64::(NOW + 30).unwrap(); + oversized.write_u16::(1).unwrap(); + encode_service_id(&ZakuraServiceId::block_sync(), &mut oversized).unwrap(); + oversized + .write_u16::(SUMMARY_TAG_BLOCK_SYNC_V1) + .unwrap(); + oversized + .write_u16::( + u16::try_from(MAX_SERVICE_SUMMARY_BYTES + 1).expect("test length fits"), + ) + .unwrap(); + + assert!(matches!( + DiscoveryMessage::decode(&oversized), + Err(DiscoveryWireError::OversizedPayload { .. }) + )); } #[test] @@ -4250,6 +4871,15 @@ mod tests { DiscoveryMessage::decode(&encoded), Err(DiscoveryWireError::TrailingBytes) )); + + let mut block_envelope = + ServiceSummaryEnvelope::block_sync(&block_sync_summary()).expect("summary encodes"); + block_envelope.summary_bytes.push(0); + + assert!(matches!( + services_message(vec![block_envelope]).encode(), + Err(DiscoveryWireError::TrailingBytes) + )); } #[test] @@ -4527,12 +5157,23 @@ mod tests { }) )); - let reserved = ServiceSummaryEnvelope { - service_id: service(42), - summary_tag: SUMMARY_TAG_BLOCK_SYNC_V1_RESERVED, - summary_bytes: vec![1, 2, 3], + let mismatched_block = ServiceSummaryEnvelope { + service_id: ZakuraServiceId::header_sync(), + summary_tag: SUMMARY_TAG_BLOCK_SYNC_V1, + summary_bytes: { + let mut bytes = Vec::new(); + encode_block_sync_summary(&block_sync_summary(), &mut bytes) + .expect("test block summary encodes"); + bytes + }, }; - assert!(services_message(vec![reserved]).encode().is_ok()); + assert!(matches!( + services_message(vec![mismatched_block]).encode(), + Err(DiscoveryWireError::MismatchedSummaryService { + summary_tag: SUMMARY_TAG_BLOCK_SYNC_V1, + .. + }) + )); } #[test] @@ -4915,6 +5556,141 @@ mod tests { } } + #[test] + fn discovery_book_peer_import_rejects_private_and_internal_direct_addresses() { + // A signed record only proves control of the node key, not ownership of the advertised + // direct addresses. Untrusted gossip must therefore be restricted to globally routable + // targets; otherwise an authenticated discovery peer could seed signed records pointing at + // the honest node's private/internal network and turn the candidate dialer into an internal + // SSRF/scanner. RFC 1918 private, RFC 6598 shared (CGNAT), and RFC 4193 unique-local ranges + // are neither loopback nor link-local, so the original filter let them through. + let bad_addrs = [ + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)), 8233), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(172, 16, 5, 5)), 8233), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), 8233), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(100, 64, 0, 1)), 8233), + SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0xfc00, 0, 0, 0, 0, 0, 0, 1)), 8233), + SocketAddr::new( + IpAddr::V6(Ipv6Addr::new(0xfd12, 0x3456, 0, 0, 0, 0, 0, 1)), + 8233, + ), + ]; + + for (index, bad_addr) in bad_addrs.into_iter().enumerate() { + let mut book = ZakuraDiscoveryBook::default(); + // Pair the malicious target with a genuinely routable address to prove the whole + // record is rejected, not just dropped down to its usable addresses. + let record = signed_record_with_addrs( + u64::try_from(index).expect("small test index fits in u64"), + service(index), + vec![test_addr(30), bad_addr], + ); + + assert!( + matches!( + book.import_record(record, Some(secret_key().public()), NOW, &context()), + Err(DiscoveryBookError::NonDialableDirectAddress { addr }) if addr == bad_addr + ), + "untrusted gossip must reject internal direct address {bad_addr}" + ); + assert!(book.is_empty()); + } + + // A genuinely globally routable gossip target still imports. + let mut book = ZakuraDiscoveryBook::default(); + let public_record = signed_record_with_addrs(99, service(99), vec![test_addr(40)]); + assert_eq!( + book.import_record(public_record, Some(secret_key().public()), NOW, &context()) + .expect("public gossip record imports"), + ImportOutcome::Added + ); + + // Trusted/static configuration is unchanged: operators may still configure private targets + // for LAN/regtest deployments. + let mut static_book = ZakuraDiscoveryBook::default(); + let private_static = signed_record_with_addrs( + 1, + service(1), + vec![SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), + 8233, + )], + ); + assert_eq!( + static_book + .import_static_record(private_static, NOW, &context()) + .expect("configured static private record imports"), + ImportOutcome::Added + ); + } + + #[test] + fn discovery_book_gossiped_public_third_party_record_is_not_dialable() { + let mut book = ZakuraDiscoveryBook::default(); + let gossip_source = secret_key().public(); + let public_target = signed_record_with_addrs( + 1, + service(1), + vec![SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(93, 184, 216, 34)), + 8233, + )], + ); + let target_id = public_target.body.node_id; + + assert_eq!( + book.import_record(public_target.clone(), Some(gossip_source), NOW, &context()) + .expect("valid public third-party gossip record still imports"), + ImportOutcome::Added + ); + assert_eq!( + book.get(&target_id).expect("entry is stored").record(), + &public_target + ); + + assert!( + book.dial_candidates( + 10, + &[service(1)], + DialCandidateExclusions { + connected_node_ids: &[], + in_flight_node_ids: &[], + }, + NOW, + ( + DEFAULT_DISCOVERY_DIAL_BACKOFF_BASE, + DEFAULT_DISCOVERY_DIAL_BACKOFF_MAX, + ), + &mut StdRng::seed_from_u64(7), + ) + .is_empty(), + "unconfirmed third-party public gossip must not drive native dials" + ); + + assert_eq!( + book.import_record(public_target.clone(), Some(target_id), NOW + 1, &context()) + .expect("first-party self-record confirmation imports"), + ImportOutcome::MetadataUpdated + ); + assert_eq!( + book.dial_candidates( + 10, + &[service(1)], + DialCandidateExclusions { + connected_node_ids: &[], + in_flight_node_ids: &[], + }, + NOW + 1, + ( + DEFAULT_DISCOVERY_DIAL_BACKOFF_BASE, + DEFAULT_DISCOVERY_DIAL_BACKOFF_MAX, + ), + &mut StdRng::seed_from_u64(7), + ), + vec![candidate_for(&public_target, false)] + ); + } + #[test] fn discovery_book_static_import_allows_loopback_direct_address() { let mut book = ZakuraDiscoveryBook::default(); @@ -5256,10 +6032,8 @@ mod tests { let failed_id = failed.body.node_id; let preferred_id = preferred.body.node_id; - book.import_record(failed.clone(), None, NOW, &context()) - .unwrap(); - book.import_record(preferred.clone(), None, NOW, &context()) - .unwrap(); + import_confirmed_record(&mut book, failed.clone()).unwrap(); + import_confirmed_record(&mut book, preferred.clone()).unwrap(); book.mark_dial_attempt(&failed_id, NOW); book.mark_dial_failure(&failed_id, NOW); book.mark_dial_success(&preferred_id, NOW + 1); @@ -5313,8 +6087,7 @@ mod tests { .collect::>(); for record in &records { - book.import_record(record.clone(), None, NOW, &context()) - .expect("test record imports"); + import_confirmed_record(&mut book, record.clone()).expect("test record imports"); } let mut deterministic = records @@ -5373,8 +6146,7 @@ mod tests { let failed = signed_record_with(1, service(1), test_addr(1)); let failed_id = failed.body.node_id; - book.import_record(failed.clone(), None, NOW, &context()) - .unwrap(); + import_confirmed_record(&mut book, failed.clone()).unwrap(); book.mark_dial_attempt(&failed_id, NOW); book.mark_dial_failure(&failed_id, NOW); @@ -5414,8 +6186,7 @@ mod tests { let exchanged = signed_record_with(1, service(1), test_addr(1)); let exchanged_id = exchanged.body.node_id; - book.import_record(exchanged.clone(), None, NOW, &context()) - .unwrap(); + import_confirmed_record(&mut book, exchanged.clone()).unwrap(); book.mark_dial_success(&exchanged_id, NOW); book.mark_short_lived_exchange(&exchanged_id, NOW); @@ -5457,8 +6228,7 @@ mod tests { let mut book = ZakuraDiscoveryBook::default(); let record = signed_record_with(1, service(9), test_addr(9)); let node_id = record.body.node_id; - book.import_record(record.clone(), None, NOW, &context()) - .unwrap(); + import_confirmed_record(&mut book, record.clone()).unwrap(); book.mark_dial_attempt(&node_id, NOW + 1); book.mark_dial_failure(&node_id, NOW + 1); @@ -6018,8 +6788,13 @@ mod tests { .await .expect("connected self-record imports"); handle - .import_peer_records([discovered.clone(), general.clone()], None) - .await; + .import_peer_record(discovered.clone(), Some(discovered.body.node_id)) + .await + .expect("discovered first-party record imports"); + handle + .import_peer_record(general.clone(), Some(general.body.node_id)) + .await + .expect("general first-party record imports"); connected_tx.send_replace(vec![peer_id_for(active_id)]); let matching = handle.service_candidates(&service(1), true, &[]).await; @@ -6239,6 +7014,254 @@ mod tests { assert!(candidates.connected.contains(&open_id)); } + #[tokio::test] + async fn block_sync_service_candidates_prefer_covering_free_slot_summary() { + let (connected_tx, connected_rx) = watch::channel(Vec::new()); + let handle = discovery_handle_with_connected(connected_rx); + let now = current_unix_secs(); + let no_summary = runtime_record_with(1, ZakuraServiceId::block_sync(), test_addr(1)); + let covering = runtime_record_with(2, ZakuraServiceId::block_sync(), test_addr(2)); + let prefix = runtime_record_with(3, ZakuraServiceId::block_sync(), test_addr(3)); + let full = runtime_record_with(4, ZakuraServiceId::block_sync(), test_addr(4)); + let not_block_sync = runtime_record_with(5, ZakuraServiceId::header_sync(), test_addr(5)); + let no_summary_id = no_summary.body.node_id; + let covering_id = covering.body.node_id; + let prefix_id = prefix.body.node_id; + let full_id = full.body.node_id; + let not_block_sync_id = not_block_sync.body.node_id; + connected_tx.send_replace(vec![ + peer_id_for(no_summary_id), + peer_id_for(covering_id), + peer_id_for(prefix_id), + peer_id_for(full_id), + peer_id_for(not_block_sync_id), + ]); + + for record in [no_summary, covering, prefix, full, not_block_sync] { + let node_id = record.body.node_id; + handle + .import_connected_peer_record(record, node_id) + .await + .expect("connected record imports"); + } + + let mut covering_summary = block_sync_summary(); + covering_summary.servable_low = block::Height(10); + covering_summary.servable_high = block::Height(20); + covering_summary.free_slots = 2; + let mut prefix_summary = covering_summary; + prefix_summary.servable_high = block::Height(12); + prefix_summary.free_slots = 4; + let mut full_summary = covering_summary; + full_summary.free_slots = 0; + for (node_id, summary) in [ + (covering_id, covering_summary), + (prefix_id, prefix_summary), + (full_id, full_summary), + ] { + handle + .import_connected_peer_services_at( + first_party_services( + node_id, + now + 30, + vec![ServiceSummaryEnvelope::block_sync(&summary) + .expect("test block summary encodes")], + ), + node_id, + now, + ) + .await + .expect("first-party block summary imports"); + } + + let generic = handle + .service_candidates(&ZakuraServiceId::block_sync(), false, &[]) + .await; + assert!(generic.connected.contains(&covering_id)); + assert!(generic.connected.contains(&prefix_id)); + assert!(generic.connected.contains(&full_id)); + assert!(generic.connected.contains(&no_summary_id)); + assert!(!generic.connected.contains(¬_block_sync_id)); + + let candidates = handle + .block_sync_candidates( + &ZakuraBlockSyncCandidateState { + missing_block_bodies: vec![block::Height(10), block::Height(20)], + ..ZakuraBlockSyncCandidateState::default() + }, + false, + &[], + ) + .await; + + assert_eq!(candidates.connected[0], covering_id); + let prefix_position = candidates + .connected + .iter() + .position(|node_id| node_id == &prefix_id) + .expect("prefix-covering peer remains a candidate"); + let no_summary_position = candidates + .connected + .iter() + .position(|node_id| node_id == &no_summary_id) + .expect("no-summary peer remains a candidate"); + assert!(prefix_position < no_summary_position); + assert!(candidates.connected.contains(&no_summary_id)); + assert!(!candidates.connected.contains(&full_id)); + assert!(!candidates.connected.contains(¬_block_sync_id)); + + let candidates = handle + .block_sync_candidates( + &ZakuraBlockSyncCandidateState { + missing_block_bodies: vec![block::Height(10), block::Height(20)], + admitted_node_ids: vec![full_id], + }, + false, + &[], + ) + .await; + + assert!(candidates.connected.contains(&full_id)); + } + + #[tokio::test] + async fn first_party_block_sync_summary_only_imports_from_authenticated_peer() { + let (connected_tx, connected_rx) = watch::channel(Vec::new()); + let handle = discovery_handle_with_connected(connected_rx); + let peer_node_id = secret_key().public(); + let claimed_node_id = secret_key().public(); + let now = current_unix_secs(); + connected_tx.send_replace(vec![peer_id_for(peer_node_id)]); + + let error = handle + .import_connected_peer_services_at( + first_party_services( + claimed_node_id, + now + 30, + vec![ServiceSummaryEnvelope::block_sync(&block_sync_summary()) + .expect("test block summary encodes")], + ), + peer_node_id, + now, + ) + .await + .expect_err("mismatched first-party block summary is rejected"); + + assert!(matches!( + error, + DiscoveryWireError::MismatchedNodeId { + field: "services node id" + } + )); + assert_eq!( + handle.live_service_summaries_at(peer_node_id, now).await, + None + ); + assert_eq!( + handle.live_service_summaries_at(claimed_node_id, now).await, + None + ); + + let record = runtime_record_with(10, ZakuraServiceId::block_sync(), test_addr(10)); + let node_id = record.body.node_id; + let message = DiscoveryMessage::Peers { + records: vec![record], + }; + let encoded = message.encode().expect("test peers response encodes"); + let DiscoveryMessage::Peers { records } = + DiscoveryMessage::decode(&encoded).expect("test peers response decodes") + else { + panic!("PEERS response decoded as a different message"); + }; + + handle.import_peer_records(records, None).await; + connected_tx.send_replace(vec![peer_id_for(peer_node_id), peer_id_for(node_id)]); + + assert_eq!(handle.live_service_summaries_at(node_id, now).await, None); + assert_eq!(handle.active_services(node_id).await, None); + } + + #[tokio::test] + async fn block_sync_summary_mismatch_changes_selection_without_punitive_state() { + let (connected_tx, connected_rx) = watch::channel(Vec::new()); + let handle = discovery_handle_with_connected(connected_rx); + let now = current_unix_secs(); + let peer = runtime_record_with(1, ZakuraServiceId::block_sync(), test_addr(1)); + let covering_peer = runtime_record_with(2, ZakuraServiceId::block_sync(), test_addr(2)); + let peer_id = peer.body.node_id; + let covering_peer_id = covering_peer.body.node_id; + connected_tx.send_replace(vec![peer_id_for(peer_id), peer_id_for(covering_peer_id)]); + for record in [peer, covering_peer] { + let node_id = record.body.node_id; + handle + .import_connected_peer_record(record, node_id) + .await + .expect("connected record imports"); + } + + let mut summary = block_sync_summary(); + summary.servable_low = block::Height(10); + summary.servable_high = block::Height(20); + summary.free_slots = 3; + let mut covering_summary = summary; + covering_summary.free_slots = 1; + for (node_id, summary) in [(peer_id, summary), (covering_peer_id, covering_summary)] { + handle + .import_connected_peer_services_at( + first_party_services( + node_id, + now + 30, + vec![ServiceSummaryEnvelope::block_sync(&summary) + .expect("test block summary encodes")], + ), + node_id, + now, + ) + .await + .expect("first-party block summary imports"); + } + + let state = ZakuraBlockSyncCandidateState { + missing_block_bodies: vec![block::Height(10), block::Height(20)], + ..ZakuraBlockSyncCandidateState::default() + }; + assert_eq!( + handle + .block_sync_candidates(&state, false, &[]) + .await + .connected + .first(), + Some(&peer_id) + ); + + summary.servable_high = block::Height(11); + handle + .import_connected_peer_services_at( + first_party_services( + peer_id, + now + 31, + vec![ServiceSummaryEnvelope::block_sync(&summary) + .expect("test block summary encodes")], + ), + peer_id, + now + 1, + ) + .await + .expect("corrected first-party block summary imports"); + + let candidates = handle.block_sync_candidates(&state, false, &[]).await; + assert_eq!(candidates.connected.first(), Some(&covering_peer_id)); + assert!(candidates.connected.contains(&peer_id)); + let cached = handle + .live_service_summaries_at(peer_id, now + 1) + .await + .expect("live summary remains cached"); + assert_eq!( + cached[0].summary, + ZakuraLiveServiceSummary::BlockSync(summary) + ); + } + #[tokio::test] async fn live_summaries_expire_independently_of_signed_records() { let (connected_tx, connected_rx) = watch::channel(Vec::new()); @@ -6310,7 +7333,7 @@ mod tests { now + 30, vec![ServiceSummaryEnvelope { service_id: service.clone(), - summary_tag: SUMMARY_TAG_BLOCK_SYNC_V1_RESERVED, + summary_tag: 999, summary_bytes: vec![1, 2, 3], }], ), @@ -6344,7 +7367,7 @@ mod tests { now, vec![ServiceSummaryEnvelope { service_id: service.clone(), - summary_tag: SUMMARY_TAG_BLOCK_SYNC_V1_RESERVED, + summary_tag: 999, summary_bytes: vec![1, 2, 3], }], ), @@ -6386,6 +7409,203 @@ mod tests { assert_eq!(outcome.rejected, 1); } + /// Regression test for `claude-discovery-expensive-work-under-global-mutex`. + /// + /// An authenticated discovery peer can send a full `Peers` batch whose records carry + /// invalid signatures. Each record still triggers a full Ed25519 verification — the + /// expensive, attacker-driven work. Previously that verification ran while holding the + /// global discovery mutex, so an attacker could repeatedly stall every other discovery + /// operation (admission, sampling, dial selection). Signatures are now verified outside + /// the lock, and an all-invalid batch never needs the lock at all, so the import makes + /// progress even while another task holds the mutex. + #[tokio::test] + async fn import_peer_records_verifies_signatures_without_holding_global_mutex() { + let (_connected_tx, connected_rx) = watch::channel(Vec::new()); + let handle = discovery_handle_with_connected(connected_rx); + + // A full Peers batch of records with valid bodies but tampered signatures: bumping + // the sequence after signing leaves body validation passing while the Ed25519 check + // (re-encoding the body and verifying) runs in full and fails. + let mut batch = Vec::with_capacity(MAX_DISCOVERY_RECORDS_PER_RESPONSE); + for index in 0..MAX_DISCOVERY_RECORDS_PER_RESPONSE { + let octet = u8::try_from(index + 1).expect("test batch index fits in u8"); + let sequence = u64::try_from(index + 1).expect("test batch index fits in u64"); + let mut record = runtime_record_with(sequence, service(1), test_addr(octet)); + record.body.sequence += 1; + batch.push(record); + } + + // Hold the global discovery mutex for the entire import call. + let guard = handle.inner.lock().await; + + // Verification and rejection of the whole batch must make progress without the lock; + // if it were still performed under the mutex this would deadlock until the timeout. + let outcome = tokio::time::timeout( + Duration::from_secs(5), + handle.import_peer_records(batch, None), + ) + .await + .expect("import_peer_records must verify signatures without holding the global mutex"); + + drop(guard); + + assert_eq!(outcome.attempted, MAX_DISCOVERY_RECORDS_PER_RESPONSE); + assert_eq!(outcome.rejected, MAX_DISCOVERY_RECORDS_PER_RESPONSE); + assert_eq!(outcome.added, 0); + } + + /// Regression test for `claude-discovery-expensive-work-under-global-mutex` (GetPeers sampling + /// facet). + /// + /// `sample_peers` runs under the global discovery mutex and is driven by attacker-paced GetPeers + /// requests. It previously cloned *every* record that passed the filter — up to the whole book + /// (`max_records`, default 10_000) — even though at most `max_imported_records_per_response` + /// records are ever returned. The clone is now bounded to the chosen sample. + /// + /// The bound is proven machine-independently by comparing the production `sample_peers` against + /// an in-test reference that reproduces the pre-fix clone-the-whole-filtered-set behavior, over + /// the same large book in the same run. With the fix, production clones only the returned sample + /// and is markedly cheaper than the clone-everything reference; before the fix the two are the + /// same computation, so the production-is-cheaper bound fails. + #[test] + fn sample_peers_clone_cost_is_bounded_by_returned_sample() { + const BOOK: usize = 1024; + const ITERS: usize = 400; + + let wanted = service(1); + // Heavy records (maximum direct-address fan-out plus many services, with the wanted service + // first so the filter match itself stays cheap) make each *record clone* far more expensive + // than the allocation-free per-entry filter scan, so the clone count is the dominant cost + // and the bounded-vs-unbounded clone gap is large. + let addrs: Vec = (1u8..=MAX_DIRECT_ADDRS_PER_RECORD as u8) + .map(test_addr) + .collect(); + let mut services = vec![wanted.clone()]; + services.extend((100..100 + (MAX_SERVICES_PER_RECORD - 1)).map(service)); + + let mut book = ZakuraDiscoveryBook::new(ZakuraDiscoveryBookLimits { + max_records: BOOK, + ..ZakuraDiscoveryBookLimits::default() + }); + for seq in 0..BOOK { + let secret = secret_key(); + let mut record_body = body(&secret); + record_body.sequence = seq as u64 + 1; + record_body.direct_addrs = addrs.clone(); + record_body.services = services.clone(); + let record = ZakuraNodeRecord::sign(record_body, &secret).expect("test record signs"); + book.import_record(record, None, NOW, &context()) + .expect("test record imports"); + } + + let cap = book.limits.max_imported_records_per_response; + + // Reference implementation: the pre-fix behavior of cloning every record that passes the + // filter, then reservoir-sampling the clones. Mirrors `sample_peers`'s filter exactly. + let naive_sample = + |book: &ZakuraDiscoveryBook, rng: &mut StdRng| -> Vec { + book.entries + .iter() + .filter(|(node_id, entry)| { + book.local_node_id != Some(**node_id) + && !entry_is_expired(entry, NOW) + && has_wanted_services(&entry.record, std::slice::from_ref(&wanted)) + && has_discovery_dialable_direct_addrs(&entry.record) + }) + .map(|(_, entry)| entry.record.clone()) + .choose_multiple(rng, cap) + }; + + let mut rng = StdRng::seed_from_u64(5); + + // Warm up the allocator and instruction caches before timing. + let _ = naive_sample(&book, &mut rng); + let _ = book.sample_peers(cap, std::slice::from_ref(&wanted), &[], NOW, &mut rng); + + let naive_start = std::time::Instant::now(); + for _ in 0..ITERS { + assert_eq!(naive_sample(&book, &mut rng).len(), cap); + } + let naive_time = naive_start.elapsed(); + + let production_start = std::time::Instant::now(); + for _ in 0..ITERS { + assert_eq!( + book.sample_peers(cap, std::slice::from_ref(&wanted), &[], NOW, &mut rng) + .len(), + cap + ); + } + let production_time = production_start.elapsed(); + + // Both run the same O(book) filter scan and reservoir sampling; the only difference is how + // many records are cloned (production: the returned `cap`; reference: every match in the + // book). With the bound in place production must be clearly cheaper than cloning the whole + // filtered set. Before the fix they are the identical computation, so production is not + // meaningfully cheaper and this fails. The 0.8 factor is a ratio on the same machine, so it + // is machine-independent. + assert!( + production_time.as_nanos() * 5 < naive_time.as_nanos() * 4, + "sample_peers did not bound its clone work: production={production_time:?} \ + clone-everything reference={naive_time:?}; production should be clearly cheaper than \ + cloning every filtered record" + ); + } + + /// Guards the bounded top-k dial-candidate selection added for + /// `claude-discovery-expensive-work-under-global-mutex`. + /// + /// `dial_candidates` now selects the best `limit` candidates with a partial select instead of + /// sorting the whole book before `take(limit)`. With far more matching candidates than `limit`, + /// the result must still be exactly the highest dial-priority candidates (most recent + /// successful dial first), proving the partial selection keeps the same ordering as a full + /// sort. + #[test] + fn dial_candidates_selects_top_priority_subset_under_limit() { + let mut book = ZakuraDiscoveryBook::default(); + let records = (1u8..=10) + .map(|index| signed_record_with(index.into(), service(1), test_addr(index))) + .collect::>(); + for record in &records { + import_confirmed_record(&mut book, record.clone()).expect("test record imports"); + } + + // Three candidates get strictly increasing last-success times, so dial priority is + // deterministic (most recent success first); the remaining seven have no successful dial. + book.mark_dial_success(&records[5].body.node_id, NOW + 1); + book.mark_dial_success(&records[3].body.node_id, NOW + 2); + book.mark_dial_success(&records[7].body.node_id, NOW + 3); + let expected_top = vec![ + records[7].body.node_id, + records[3].body.node_id, + records[5].body.node_id, + ]; + + let mut rng = StdRng::seed_from_u64(99); + let selected = book.dial_candidates( + 3, + &[service(1)], + DialCandidateExclusions { + connected_node_ids: &[], + in_flight_node_ids: &[], + }, + NOW, + ( + DEFAULT_DISCOVERY_DIAL_BACKOFF_BASE, + DEFAULT_DISCOVERY_DIAL_BACKOFF_MAX, + ), + &mut rng, + ); + + assert_eq!( + selected + .iter() + .map(|candidate| candidate.node_id) + .collect::>(), + expected_top, + ); + } + #[tokio::test] async fn handle_samples_records_through_storage_path() { let (_connected_tx, connected_rx) = watch::channel(Vec::new()); @@ -6438,9 +7658,15 @@ mod tests { let candidate = runtime_record_with(1, service(1), test_addr(1)); let connected = runtime_record_with(2, service(1), test_addr(2)); let connected_id = connected.body.node_id; + let candidate_id = candidate.body.node_id; handle - .import_peer_records([candidate.clone(), connected], None) - .await; + .import_peer_record(candidate.clone(), Some(candidate_id)) + .await + .expect("candidate first-party record imports"); + handle + .import_connected_peer_record(connected, connected_id) + .await + .expect("connected self-record imports"); connected_tx.send_replace(vec![peer_id_for(connected_id)]); assert!(handle.dial_candidates(&[service(1)], &[]).await.is_empty()); @@ -6458,9 +7684,15 @@ mod tests { ); let connected = runtime_record_with(2, service(1), test_addr(2)); let connected_id = connected.body.node_id; + let candidate_id = candidate.body.node_id; handle - .import_peer_records([candidate.clone(), connected], None) - .await; + .import_peer_record(candidate.clone(), Some(candidate_id)) + .await + .expect("candidate first-party record imports"); + handle + .import_connected_peer_record(connected, connected_id) + .await + .expect("connected self-record imports"); connected_tx.send_replace(vec![peer_id_for(connected_id)]); assert_eq!( diff --git a/zebra-network/src/zakura/discovery/redial.rs b/zebra-network/src/zakura/discovery/redial.rs index a81ac8672fe..f2b89fbf9fa 100644 --- a/zebra-network/src/zakura/discovery/redial.rs +++ b/zebra-network/src/zakura/discovery/redial.rs @@ -96,8 +96,9 @@ pub(crate) async fn native_dial_supervised( return; }; + let shutdown = endpoint.background_shutdown_token(); let registered = endpoint.supervisor().subscribe(); - run_dial_supervisor(peer_id, registered, policy, move || { + let supervised = run_dial_supervisor(peer_id, registered, policy, move || { let endpoint = endpoint.clone(); let node_addr = node_addr.clone(); let limits = limits.clone(); @@ -114,8 +115,18 @@ pub(crate) async fn native_dial_supervised( } } }) as Pin + Send>> - }) - .await; + }); + + // Endpoint shutdown cancels this token; stop maintaining the dial promptly + // rather than looping on against a torn-down router. A `maintain` policy + // never exits on its own (no max attempts), and the supervisor registration + // watch stays open while this task holds an endpoint clone, so this is the + // only signal that reliably ends the loop at teardown. + tokio::select! { + biased; + _ = shutdown.cancelled() => {} + _ = supervised => {} + } } /// Retry/backoff loop shared by configured bootstrap peers and the upgrade dial. diff --git a/zebra-network/src/zakura/discovery/runtime.rs b/zebra-network/src/zakura/discovery/runtime.rs index 4294e6bf9d5..0cd3ccf6154 100644 --- a/zebra-network/src/zakura/discovery/runtime.rs +++ b/zebra-network/src/zakura/discovery/runtime.rs @@ -16,6 +16,7 @@ use super::protocol::{ pub(crate) fn default_advertised_services() -> Vec { vec![ ZakuraServiceId::discovery(), + ZakuraServiceId::block_sync(), ZakuraServiceId::header_sync(), ZakuraServiceId::legacy_gossip(), ZakuraServiceId::legacy_requests(), diff --git a/zebra-network/src/zakura/discovery/service.rs b/zebra-network/src/zakura/discovery/service.rs index 08fb80ec869..99acdbf65f3 100644 --- a/zebra-network/src/zakura/discovery/service.rs +++ b/zebra-network/src/zakura/discovery/service.rs @@ -21,22 +21,23 @@ use tokio::sync::Notify; use tokio_util::sync::CancellationToken; use crate::zakura::{ - BoxRunFuture, Frame, FramedRecv, FramedSend, HeaderSyncEvent, HeaderSyncHandle, - OrderedSendError, Peer, PeerStreamSession, Service, ServiceAdmissionDecision, - ServicePeerDirection, Sink, SinkReject, Stream, StreamMode, ZakuraPeerId, - LOCAL_MAX_CONTROL_FRAME_BYTES, ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_HEADER_SYNC, + handle_pipe_exit, spawn_supervised_peer_task, spawn_supervised_pipe, BlockSyncHandle, Flow, + Frame, FramedRecv, FramedSend, HeaderSyncEvent, HeaderSyncHandle, OrderedSendError, Peer, + PeerStreamSession, Pipe, Service, ServiceAdmissionDecision, ServicePeerDirection, SinkReject, + Stream, StreamMode, ZakuraPeerId, LOCAL_MAX_CONTROL_FRAME_BYTES, ZAKURA_CAP_DISCOVERY, + ZAKURA_CAP_HEADER_SYNC, }; +#[cfg(test)] +use super::pipe::decode_discovery_frame; +use super::pipe::{discovery_pipe, DsEnv, DsLocal, DISCOVERY_FRAME_MESSAGE_TYPE}; use super::protocol::{ - DiscoveryBookError, DiscoveryMessage, DiscoveryRecordError, GetServices, - HeaderSyncServiceSummary, ServiceSummaryEnvelope, Services, ZakuraDiscoveryHandle, + BlockSyncServiceSummary, DiscoveryBookError, DiscoveryMessage, DiscoveryRecordError, + GetServices, HeaderSyncServiceSummary, ServiceSummaryEnvelope, Services, ZakuraDiscoveryHandle, ZakuraNodeRecord, ZakuraServiceId, MAX_DISCOVERY_RECORDS_PER_RESPONSE, ZAKURA_DISCOVERY_STREAM_VERSION, ZAKURA_STREAM_DISCOVERY, }; -/// Frame message type carrying a discovery payload (matches the native wire). -const DISCOVERY_FRAME_MESSAGE_TYPE: u16 = 1; - /// Maximum time discovery waits for first-party exchange responses before releasing the session. const DISCOVERY_EXCHANGE_SETTLE_TIMEOUT: Duration = Duration::from_secs(2); @@ -153,6 +154,7 @@ impl DiscoveryPeerSession { pub struct DiscoveryService { handle: ZakuraDiscoveryHandle, header_sync: Option, + block_sync: Option, } impl DiscoveryService { @@ -161,17 +163,20 @@ impl DiscoveryService { Self { handle, header_sync: None, + block_sync: None, } } - /// Builds a discovery service with a header-sync first-party summary provider. - pub(crate) fn with_header_sync( + /// Builds a discovery service with header-sync and block-sync summary providers. + pub(crate) fn with_sync_services( handle: ZakuraDiscoveryHandle, header_sync: HeaderSyncHandle, + block_sync: Option, ) -> Self { Self { handle, header_sync: Some(header_sync), + block_sync, } } @@ -230,35 +235,53 @@ impl Service for DiscoveryService { let handle = self.handle.clone(); let header_sync = self.header_sync.clone(); - tokio::spawn(async move { - let decision = handle - .admit_peer( - discovery_session.peer_id().clone(), - discovery_session.direction(), - ) - .await; - if decision != ServiceAdmissionDecision::Admit { - tracing::debug!( - peer = ?discovery_session.peer_id(), - direction = ?discovery_session.direction(), - ?decision, - "locally parking Zakura discovery service session" - ); - service_cancel.cancel(); - return; - } + let block_sync = self.block_sync.clone(); + // SR-1: a panic in the admission task (before it hands off to the + // exchange) must still disconnect this one peer and cancel its discovery + // session instead of leaving admitted state behind a half-live + // connection. Normal/parked exits cancel `service_cancel` inline below; + // `on_panic` covers the unwind path only. + let admit_peer_id = discovery_session.peer_id().clone(); + let panic_service_cancel = service_cancel.clone(); + let panic_connection_cancel = connection_cancel.clone(); + spawn_supervised_peer_task( + admit_peer_id, + || {}, + move || { + panic_service_cancel.cancel(); + panic_connection_cancel.cancel(); + }, + async move { + let decision = handle + .admit_peer( + discovery_session.peer_id().clone(), + discovery_session.direction(), + ) + .await; + if decision != ServiceAdmissionDecision::Admit { + tracing::debug!( + peer = ?discovery_session.peer_id(), + direction = ?discovery_session.direction(), + ?decision, + "locally parking Zakura discovery service session" + ); + service_cancel.cancel(); + return; + } - spawn_discovery_exchange(DiscoveryExchangeStart { - handle, - header_sync, - peer_node_id, - discovery_session, - recv, - service_cancel, - connection_cancel, - other_service_negotiated, - }); - }); + spawn_discovery_exchange(DiscoveryExchangeStart { + handle, + header_sync, + block_sync, + peer_node_id, + discovery_session, + recv, + service_cancel, + connection_cancel, + other_service_negotiated, + }); + }, + ); } fn remove_peer(&self, peer: &ZakuraPeerId) { @@ -273,6 +296,7 @@ impl Service for DiscoveryService { struct DiscoveryExchangeStart { handle: ZakuraDiscoveryHandle, header_sync: Option, + block_sync: Option, peer_node_id: NodeId, discovery_session: DiscoveryPeerSession, recv: FramedRecv, @@ -285,6 +309,7 @@ fn spawn_discovery_exchange(start: DiscoveryExchangeStart) { let DiscoveryExchangeStart { handle, header_sync, + block_sync, peer_node_id, discovery_session, recv, @@ -298,85 +323,112 @@ fn spawn_discovery_exchange(start: DiscoveryExchangeStart) { let sink = DiscoverySink { handle: handle.clone(), header_sync, + block_sync, peer_node_id, session: discovery_session.clone(), progress: progress.clone(), }; let sink_service_cancel = service_cancel.clone(); - let sink_connection_cancel = connection_cancel.clone(); - tokio::spawn(async move { - match Box::new(sink).run(recv).await { - Ok(()) => {} - Err(SinkReject::Protocol(error)) => { - tracing::debug!( - ?error, - "Zakura discovery stream rejected protocol-invalid frame" - ); - sink_connection_cancel.cancel(); - } - Err(SinkReject::Local(error)) => { - tracing::debug!(?error, "Zakura discovery stream stopped on local error"); - } - } - sink_service_cancel.cancel(); - }); + let reject_connection_cancel = connection_cancel.clone(); + let panic_connection_cancel = connection_cancel.clone(); + let sink_peer_id = peer_id.clone(); + // A protocol reject is fatal to the connection; normal/parked exits leave it + // for the source task to tear down once it knows no other service owns the + // peer (below). Panic teardown is in `on_panic`. + let pipe = async move { + let mut pipe = discovery_pipe(sink_peer_id); + handle_pipe_exit( + "discovery", + &reject_connection_cancel, + run_discovery_pipe(&mut pipe, recv, sink).await, + ); + }; + let on_panic = move || panic_connection_cancel.cancel(); + // Let the returned handle drop to detach the supervised reader task; the + // `PipeTeardown` still runs on every exit path. + spawn_supervised_pipe(peer_id.clone(), sink_service_cancel, || {}, on_panic, pipe); let source = DiscoverySource { handle: handle.clone(), session: discovery_session, progress, }; - tokio::spawn(async move { - let exchanged = source.run().await; - if exchanged { - handle.mark_short_lived_exchange(&peer_node_id).await; - } - service_cancel.cancel(); - handle.remove_peer(&peer_id).await; - if exchanged - && !peer_has_other_service_owner( - source_header_sync.as_ref(), - peer_node_id, - other_service_negotiated, - ) - { - connection_cancel.cancel(); - } - }); + // SR-1: a panic in the source task skips its `service_cancel.cancel()`, + // `handle.remove_peer()`, and discovery-only connection cancellation, + // leaving admitted discovery state behind a half-live connection. On the + // unwind path, disconnect this one peer; the connection teardown then drives + // the async `remove_peer` through the registry. Normal exits run the inline + // cleanup below, so `on_panic` is the panic-only path. + let source_task_peer_id = peer_id.clone(); + let panic_source_service_cancel = service_cancel.clone(); + let panic_source_connection_cancel = connection_cancel.clone(); + spawn_supervised_peer_task( + source_task_peer_id, + || {}, + move || { + panic_source_service_cancel.cancel(); + panic_source_connection_cancel.cancel(); + }, + async move { + let exchanged = source.run().await; + if exchanged { + handle.mark_short_lived_exchange(&peer_node_id).await; + } + service_cancel.cancel(); + handle.remove_peer(&peer_id).await; + if exchanged + && !peer_has_other_service_owner( + source_header_sync.as_ref(), + peer_node_id, + other_service_negotiated, + ) + { + connection_cancel.cancel(); + } + }, + ); } /// Reader half of the discovery stream: imports peer records and answers queries. struct DiscoverySink { handle: ZakuraDiscoveryHandle, header_sync: Option, + block_sync: Option, peer_node_id: NodeId, session: DiscoveryPeerSession, progress: Arc, } -impl Sink for DiscoverySink { - fn run(self: Box, mut recv: FramedRecv) -> BoxRunFuture<'static, Result<(), SinkReject>> { - Box::pin(async move { - let cancel = self.session.cancel_token(); - loop { - tokio::select! { - biased; - _ = cancel.cancelled() => return Ok(()), - frame = recv.recv() => { - let Some(frame) = frame else { - return Ok(()); - }; - self.handle_frame(frame).await?; - } - } - } - }) +async fn run_discovery_pipe( + pipe: &mut Pipe, + mut recv: FramedRecv, + sink: DiscoverySink, +) -> Result<(), SinkReject> { + let cancel = sink.session.cancel_token(); + loop { + let frame = tokio::select! { + biased; + _ = cancel.cancelled() => return Ok(()), + frame = recv.recv() => frame, + }; + let Some(frame) = frame else { + return Ok(()); + }; + + match pipe.run_one(frame) { + Flow::Continue(()) | Flow::Done => {} + Flow::Reject(reject) => return Err(reject), + } + + let Some(message) = pipe.local_mut().take_decoded() else { + continue; + }; + sink.handle_message(message).await?; } } impl DiscoverySink { - async fn handle_frame(&self, frame: Frame) -> Result<(), SinkReject> { - let message = decode_discovery_frame(&frame).map_err(SinkReject::protocol)?; + async fn handle_message(&self, message: DiscoveryMessage) -> Result<(), SinkReject> { match message { DiscoveryMessage::Hello { record } => self.handle_hello(record).await, DiscoveryMessage::GetPeers { @@ -429,6 +481,17 @@ impl DiscoverySink { summaries.push(ServiceSummaryEnvelope::discovery(&summary).map_err(SinkReject::local)?); } + if service_wanted(&query.wanted_services, &ZakuraServiceId::block_sync()) { + if let Some(block_sync) = &self.block_sync { + let summary = BlockSyncServiceSummary::from_status_and_snapshot( + block_sync.local_status(), + block_sync.peer_snapshot(), + ); + summaries + .push(ServiceSummaryEnvelope::block_sync(&summary).map_err(SinkReject::local)?); + } + } + Ok(self.handle.local_services_response(summaries)) } @@ -645,19 +708,6 @@ fn peer_has_other_service_owner( }) } -/// Decodes a discovery message from a transport frame, rejecting a frame whose -/// envelope is not a discovery payload. -fn decode_discovery_frame(frame: &Frame) -> Result { - if frame.message_type != DISCOVERY_FRAME_MESSAGE_TYPE || frame.flags != 0 { - return Err(format!( - "unexpected discovery frame envelope (message_type={}, flags={})", - frame.message_type, frame.flags - ) - .into()); - } - DiscoveryMessage::decode(&frame.payload).map_err(Into::into) -} - /// Returns the iroh node id encoded by a discovery peer id, if it is a 32-byte /// node id. fn node_id_from_peer_id(peer_id: &ZakuraPeerId) -> Option { @@ -695,11 +745,12 @@ mod tests { DiscoveryServiceSummary, ZakuraLiveServiceSummary, ZakuraNodeRecordBody, }; use crate::zakura::{ - framed_channel, spawn_header_sync_reactor, HeaderSyncAction, HeaderSyncFrontiers, - HeaderSyncMessage, HeaderSyncPeerSession, HeaderSyncStartup, HeaderSyncStatus, - ServicePeerLimits, ZakuraDiscoveryConfig, ZakuraDiscoveryLocalConfig, + framed_channel, spawn_block_sync_reactor, spawn_header_sync_reactor, BlockSyncFrontiers, + BlockSyncStartup, HeaderSyncAction, HeaderSyncFrontiers, HeaderSyncMessage, + HeaderSyncPeerSession, HeaderSyncStartup, HeaderSyncStatus, ServicePeerLimits, + ZakuraBlockSyncConfig, ZakuraDiscoveryConfig, ZakuraDiscoveryLocalConfig, ZakuraHandshakeConfig, ZakuraHeaderSyncConfig, LOCAL_MAX_MESSAGE_BYTES, - ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_HEADER_SYNC, + MAX_BS_RESPONSE_BYTES, ZAKURA_CAP_BLOCK_SYNC, ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_HEADER_SYNC, }; use zebra_chain::{block, parameters::Network}; @@ -756,6 +807,7 @@ mod tests { HeaderSyncFrontiers { finalized_height: anchor.0, verified_block_tip: anchor.0, + verified_block_hash: anchor.1, }, Some(anchor), ZakuraHeaderSyncConfig::default(), @@ -807,8 +859,11 @@ mod tests { connected_rx, )?; let (header_sync, header_actions, header_task) = spawn_test_header_sync()?; - let service = - DiscoveryService::with_header_sync(discovery_handle.clone(), header_sync.clone()); + let service = DiscoveryService::with_sync_services( + discovery_handle.clone(), + header_sync.clone(), + None, + ); let peer_node_id = SecretKey::from_bytes(&[peer_seed; 32]).public(); let peer_id = ZakuraPeerId::new(peer_node_id.as_bytes().to_vec())?; connected_tx.send_replace(vec![peer_id.clone()]); @@ -989,7 +1044,10 @@ mod tests { .header_sync .send(HeaderSyncEvent::WireMessage { peer: fixture.peer_id.clone(), - msg: HeaderSyncMessage::Headers(Vec::new()), + msg: HeaderSyncMessage::Headers { + headers: Vec::new(), + body_sizes: Vec::new(), + }, }) .await?; tokio::time::sleep(Duration::from_millis(20)).await; @@ -1085,6 +1143,109 @@ mod tests { Ok(()) } + #[tokio::test] + async fn get_services_returns_local_first_party_block_sync_summary( + ) -> Result<(), crate::BoxError> { + let (_connected_tx, connected_rx) = watch::channel(Vec::new()); + let handshake = ZakuraHandshakeConfig::for_network(&Network::Mainnet); + let local_secret = SecretKey::from_bytes(&[24u8; 32]); + let discovery_handle = ZakuraDiscoveryHandle::new( + ZakuraDiscoveryLocalConfig { + secret_key: local_secret.clone(), + direct_addrs: Vec::new(), + services: vec![ZakuraServiceId::discovery(), ZakuraServiceId::block_sync()], + zakura_protocol_min: handshake.zakura_protocol_min, + zakura_protocol_max: handshake.zakura_protocol_max, + network_id: handshake.network_id, + chain_id: handshake.chain_id, + last_authored_sequence: None, + }, + ZakuraDiscoveryConfig::default(), + connected_rx, + )?; + let (header_sync, _header_actions, header_task) = spawn_test_header_sync()?; + let (tip_tx, tip_rx) = watch::channel((block::Height(5), block::Hash([5; 32]))); + drop(tip_tx); + let (block_sync, _block_actions, block_task) = + spawn_block_sync_reactor(BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(5), + verified_block_hash: block::Hash([5; 32]), + }, + (block::Height(5), block::Hash([5; 32])), + tip_rx, + ZakuraBlockSyncConfig::default(), + )); + let service = DiscoveryService::with_sync_services( + discovery_handle, + header_sync, + Some(block_sync.clone()), + ); + let peer_node_id = SecretKey::from_bytes(&[25u8; 32]).public(); + let peer_id = ZakuraPeerId::new(peer_node_id.as_bytes().to_vec())?; + let (peer_send, service_recv) = framed_channel(8); + let (service_send, mut peer_recv) = framed_channel(8); + let streams = HashMap::from([(ZAKURA_STREAM_DISCOVERY, (service_recv, service_send))]); + + service.add_peer(Peer::new( + peer_id, + None, + ZAKURA_CAP_DISCOVERY | ZAKURA_CAP_BLOCK_SYNC, + streams, + CancellationToken::new(), + )); + + peer_send + .send(Frame { + message_type: DISCOVERY_FRAME_MESSAGE_TYPE, + flags: 0, + payload: DiscoveryMessage::GetServices(GetServices { + wanted_services: vec![ZakuraServiceId::block_sync()], + }) + .encode()?, + }) + .await?; + + let services = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let frame = peer_recv.recv().await.expect("discovery stream stays open"); + let message = decode_discovery_frame(&frame).expect("outbound frame decodes"); + if let DiscoveryMessage::Services(services) = message { + return services; + } + } + }) + .await + .expect("service response is sent"); + + assert_eq!(services.node_id, local_secret.public()); + assert_eq!(services.summaries.len(), 1); + assert_eq!( + services.summaries[0].service_id, + ZakuraServiceId::block_sync() + ); + let summary = services.summaries[0] + .decode_block_sync()? + .expect("block summary tag decodes"); + assert_eq!(summary.servable_low, block::Height(0)); + assert_eq!(summary.servable_high, block::Height(5)); + assert_eq!(summary.tip_hash, block::Hash([5; 32])); + assert_eq!( + usize::from(summary.free_slots), + block_sync.peer_snapshot().inbound_slots_free + ); + assert_eq!( + summary.max_blocks_per_response, + ZakuraBlockSyncConfig::default().advertised_max_blocks_per_response() + ); + assert_eq!(summary.max_response_bytes, MAX_BS_RESPONSE_BYTES); + + header_task.abort(); + block_task.abort(); + Ok(()) + } + #[tokio::test] async fn inbound_services_updates_first_party_live_summary_cache() -> Result<(), crate::BoxError> { @@ -1353,8 +1514,11 @@ mod tests { connected_rx, )?; let (header_sync, _header_actions, header_task) = spawn_test_header_sync()?; - let service = - DiscoveryService::with_header_sync(discovery_handle.clone(), header_sync.clone()); + let service = DiscoveryService::with_sync_services( + discovery_handle.clone(), + header_sync.clone(), + None, + ); let peer_secret = SecretKey::from_bytes(&[43u8; 32]); let peer_node_id = peer_secret.public(); let peer_id = ZakuraPeerId::new(peer_node_id.as_bytes().to_vec())?; diff --git a/zebra-network/src/zakura/exchange.rs b/zebra-network/src/zakura/exchange.rs new file mode 100644 index 00000000000..517ed765c9e --- /dev/null +++ b/zebra-network/src/zakura/exchange.rs @@ -0,0 +1,923 @@ +//! Shared Zakura sync frontier contract. + +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +use futures::future::BoxFuture; +use serde_json::{Number, Value}; +use tokio::sync::watch; +use zebra_chain::block; + +use super::{ + commit_state_trace as cs_trace, BlockApplyResult, BlockApplyToken, BlockSyncBlockMeta, + HeaderSyncCommitFailureKind, ZakuraPeerId, ZakuraTrace, COMMIT_STATE_TABLE, +}; + +/// A height/hash pair at one chain frontier. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct Frontier { + /// Frontier height. + pub height: block::Height, + /// Frontier block hash. + pub hash: block::Hash, +} + +impl Frontier { + /// Construct a frontier from its height and hash. + pub fn new(height: block::Height, hash: block::Hash) -> Self { + Self { height, hash } + } +} + +/// Shared Zakura chain facts owned by the sync exchange. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct ChainFrontier { + /// Highest finalized block. + pub finalized: Frontier, + /// Highest verified block body. + pub verified_body: Frontier, + /// Highest committed header target. + pub best_header: Frontier, +} + +/// Cause for a shared frontier update. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum FrontierChange { + /// Initial or whole-state snapshot. + Snapshot, + /// Verified body frontier advanced. + VerifiedGrow, + /// Verified body frontier was reset, possibly lower. + VerifiedReset, + /// Best header target advanced. + HeaderAdvanced, + /// Best header target was reanchored, possibly lower. + HeaderReanchored, +} + +/// Latest shared frontier plus the transition cause. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct FrontierUpdate { + /// Current shared frontier after the change. + pub frontier: ChainFrontier, + /// Cause of the current value. + pub change: FrontierChange, +} + +/// Single owner of shared Zakura sync frontiers. +#[derive(Clone, Debug)] +pub struct ZakuraSyncExchange { + inner: Arc, +} + +#[derive(Debug)] +struct ZakuraSyncExchangeInner { + frontier: watch::Sender, + trace: ZakuraTrace, + sequence: AtomicU64, +} + +impl ZakuraSyncExchange { + /// Create a shared sync exchange with an initial coherent frontier snapshot. + pub fn new(initial: FrontierUpdate, trace: ZakuraTrace) -> Self { + let (frontier, _receiver) = watch::channel(initial); + + Self { + inner: Arc::new(ZakuraSyncExchangeInner { + frontier, + trace, + sequence: AtomicU64::new(0), + }), + } + } + + /// Return the currently cached shared frontier update. + pub fn current_frontier(&self) -> FrontierUpdate { + *self.inner.frontier.borrow() + } + + /// Subscribe to latest-value frontier updates. + pub fn subscribe_frontier(&self) -> watch::Receiver { + self.inner.frontier.subscribe() + } + + /// Publish a candidate frontier update from `source`. + pub fn publish_frontier(&self, requested: FrontierUpdate, source: &'static str) { + let mut transition = None; + + self.inner.frontier.send_if_modified(|current| { + let old = *current; + let sequence = self + .inner + .sequence + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1); + + match apply_frontier_update(old, requested) { + Some(update) => { + *current = update; + transition = Some((sequence, old, update, "accepted")); + true + } + None => { + let ignored = FrontierUpdate { + frontier: old.frontier, + change: requested.change, + }; + transition = Some((sequence, old, ignored, "ignored")); + false + } + } + }); + + if let Some((sequence, old, new, result)) = transition { + self.trace_transition(sequence, source, old, new, result); + } + } + + fn trace_transition( + &self, + sequence: u64, + source: &'static str, + old: FrontierUpdate, + new: FrontierUpdate, + result: &'static str, + ) { + self.inner.trace.emit_with(COMMIT_STATE_TABLE, |row| { + row.insert( + cs_trace::EVENT.to_string(), + Value::String(cs_trace::SYNC_FRONTIER_TRANSITION.to_string()), + ); + row.insert( + cs_trace::SOURCE.to_string(), + Value::String(source.to_string()), + ); + row.insert( + cs_trace::SEQUENCE.to_string(), + Value::Number(Number::from(sequence)), + ); + row.insert( + cs_trace::CAUSE.to_string(), + Value::String(frontier_change_label(new.change).to_string()), + ); + row.insert( + cs_trace::RESULT.to_string(), + Value::String(result.to_string()), + ); + insert_frontier_fields(row, "old", old.frontier); + insert_frontier_fields(row, "new", new.frontier); + }); + } +} + +/// Result of a header range commit through the sync exchange. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct HeaderRangeCommit { + /// First committed header height. + pub start_height: block::Height, + /// New durable best header frontier. + pub tip: Frontier, +} + +/// Result of a submitted block body through the sync exchange. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct BlockBodySubmit { + /// Submission token echoed from the reactor. + pub token: BlockApplyToken, + /// Submitted block height. + pub height: block::Height, + /// Submitted block hash. + pub hash: block::Hash, + /// Verifier result. + pub result: BlockApplyResult, + /// Locally observed shared frontier after the apply attempt. + pub local_frontier: Option, +} + +/// Header-sync view of shared Zakura state. +pub trait HeaderSyncStatePortImpl: Send + Sync + 'static { + /// Return the currently cached shared frontier. + fn current_frontier(&self) -> ChainFrontier; + + /// Subscribe to latest-value shared frontier updates. + fn subscribe_frontier(&self) -> watch::Receiver; + + /// Commit a contiguous header range. + fn commit_header_range( + &self, + peer: ZakuraPeerId, + anchor: block::Hash, + start_height: block::Height, + headers: Vec>, + body_sizes: Vec, + finalized: bool, + ) -> BoxFuture<'static, Result>; + + /// Publish a locally accepted best-header advance. + fn publish_best_header(&self, tip: Frontier) -> BoxFuture<'static, ()>; + + /// Publish a best-header reanchor. + fn publish_header_reanchor(&self, old: Frontier, new: Frontier) -> BoxFuture<'static, ()>; +} + +/// Block-sync view of shared Zakura state. +pub trait BlockSyncStatePortImpl: Send + Sync + 'static { + /// Return the currently cached shared frontier. + fn current_frontier(&self) -> ChainFrontier; + + /// Subscribe to latest-value shared frontier updates. + fn subscribe_frontier(&self) -> watch::Receiver; + + /// Query committed headers that still need block bodies. + fn query_missing_bodies( + &self, + verified_body: block::Height, + best_header: block::Height, + ) -> BoxFuture<'static, Result, zebra_chain::BoxError>>; + + /// Submit a downloaded body to the verifier/state pipeline. + fn submit_block_body( + &self, + token: BlockApplyToken, + block: Arc, + ) -> BoxFuture<'static, BlockBodySubmit>; + + /// Read committed blocks for serving stream-6 peers. + fn read_committed_blocks( + &self, + start: block::Height, + count: u32, + ) -> BoxFuture< + 'static, + Result, usize)>, zebra_chain::BoxError>, + >; + + /// Publish body-sync progress that did not come from a direct submit path. + fn publish_body_progress(&self, update: FrontierUpdate) -> BoxFuture<'static, ()>; +} + +/// Cloneable header-sync state port. +#[derive(Clone)] +pub struct HeaderSyncStatePort { + inner: Arc, +} + +impl HeaderSyncStatePort { + /// Wrap an implementation in a cloneable port. + pub fn new(inner: Arc) -> Self { + Self { inner } + } + + /// Return the currently cached shared frontier. + pub fn current_frontier(&self) -> ChainFrontier { + self.inner.current_frontier() + } + + /// Subscribe to latest-value shared frontier updates. + pub fn subscribe_frontier(&self) -> watch::Receiver { + self.inner.subscribe_frontier() + } + + /// Commit a contiguous header range. + pub fn commit_header_range( + &self, + peer: ZakuraPeerId, + anchor: block::Hash, + start_height: block::Height, + headers: Vec>, + body_sizes: Vec, + finalized: bool, + ) -> BoxFuture<'static, Result> { + self.inner + .commit_header_range(peer, anchor, start_height, headers, body_sizes, finalized) + } + + /// Publish a locally accepted best-header advance. + pub fn publish_best_header(&self, tip: Frontier) -> BoxFuture<'static, ()> { + self.inner.publish_best_header(tip) + } + + /// Publish a best-header reanchor. + pub fn publish_header_reanchor(&self, old: Frontier, new: Frontier) -> BoxFuture<'static, ()> { + self.inner.publish_header_reanchor(old, new) + } +} + +impl std::fmt::Debug for HeaderSyncStatePort { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("HeaderSyncStatePort") + } +} + +/// Cloneable block-sync state port. +#[derive(Clone)] +pub struct BlockSyncStatePort { + inner: Arc, +} + +impl BlockSyncStatePort { + /// Wrap an implementation in a cloneable port. + pub fn new(inner: Arc) -> Self { + Self { inner } + } + + /// Return the currently cached shared frontier. + pub fn current_frontier(&self) -> ChainFrontier { + self.inner.current_frontier() + } + + /// Subscribe to latest-value shared frontier updates. + pub fn subscribe_frontier(&self) -> watch::Receiver { + self.inner.subscribe_frontier() + } + + /// Query committed headers that still need block bodies. + pub fn query_missing_bodies( + &self, + verified_body: block::Height, + best_header: block::Height, + ) -> BoxFuture<'static, Result, zebra_chain::BoxError>> { + self.inner.query_missing_bodies(verified_body, best_header) + } + + /// Submit a downloaded body to the verifier/state pipeline. + pub fn submit_block_body( + &self, + token: BlockApplyToken, + block: Arc, + ) -> BoxFuture<'static, BlockBodySubmit> { + self.inner.submit_block_body(token, block) + } + + /// Read committed blocks for serving stream-6 peers. + pub fn read_committed_blocks( + &self, + start: block::Height, + count: u32, + ) -> BoxFuture< + 'static, + Result, usize)>, zebra_chain::BoxError>, + > { + self.inner.read_committed_blocks(start, count) + } + + /// Publish body-sync progress that did not come from a direct submit path. + pub fn publish_body_progress(&self, update: FrontierUpdate) -> BoxFuture<'static, ()> { + self.inner.publish_body_progress(update) + } +} + +impl std::fmt::Debug for BlockSyncStatePort { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("BlockSyncStatePort") + } +} + +/// Build a [`ChainFrontier`] from startup pieces that do not expose finalized hashes. +pub fn chain_frontier_from_parts( + finalized_height: block::Height, + verified_body: Frontier, + best_header: Frontier, +) -> ChainFrontier { + let finalized_hash = if finalized_height == verified_body.height { + verified_body.hash + } else { + block::Hash([0; 32]) + }; + + ChainFrontier { + finalized: Frontier::new(finalized_height, finalized_hash), + verified_body, + best_header, + } +} + +/// Apply one requested frontier update to the current exchange frontier. +/// +/// Returns `None` when the requested update is stale or redundant. +pub fn apply_frontier_update( + current: FrontierUpdate, + requested: FrontierUpdate, +) -> Option { + let mut frontier = current.frontier; + + match requested.change { + FrontierChange::Snapshot => { + frontier.finalized = + higher_frontier(current.frontier.finalized, requested.frontier.finalized); + frontier.verified_body = higher_frontier( + current.frontier.verified_body, + requested.frontier.verified_body, + ); + frontier.best_header = + higher_frontier(current.frontier.best_header, requested.frontier.best_header); + if frontier == current.frontier { + return None; + } + } + FrontierChange::VerifiedGrow => { + if requested.frontier.verified_body.height < current.frontier.verified_body.height { + return None; + } + frontier.finalized = + higher_frontier(current.frontier.finalized, requested.frontier.finalized); + frontier.verified_body = requested.frontier.verified_body; + } + FrontierChange::VerifiedReset => { + frontier.finalized = + higher_frontier(current.frontier.finalized, requested.frontier.finalized); + frontier.verified_body = requested.frontier.verified_body; + } + FrontierChange::HeaderAdvanced => { + if requested.frontier.best_header.height <= current.frontier.best_header.height { + return None; + } + frontier.best_header = requested.frontier.best_header; + } + FrontierChange::HeaderReanchored => { + if requested.frontier.best_header == current.frontier.best_header { + return None; + } + frontier.best_header = requested.frontier.best_header; + } + } + + Some(FrontierUpdate { + frontier, + change: requested.change, + }) +} + +fn higher_frontier(left: Frontier, right: Frontier) -> Frontier { + if right.height > left.height { + right + } else { + left + } +} + +fn frontier_change_label(change: FrontierChange) -> &'static str { + match change { + FrontierChange::Snapshot => "snapshot", + FrontierChange::VerifiedGrow => "verified_grow", + FrontierChange::VerifiedReset => "verified_reset", + FrontierChange::HeaderAdvanced => "header_advanced", + FrontierChange::HeaderReanchored => "header_reanchored", + } +} + +fn insert_frontier_fields( + row: &mut serde_json::Map, + prefix: &'static str, + frontier: ChainFrontier, +) { + let ( + finalized_height, + finalized_hash, + verified_body_height, + verified_body_hash, + best_header_height, + best_header_hash, + ) = match prefix { + "old" => ( + cs_trace::OLD_FINALIZED_HEIGHT, + cs_trace::OLD_FINALIZED_HASH, + cs_trace::OLD_VERIFIED_BODY_HEIGHT, + cs_trace::OLD_VERIFIED_BODY_HASH, + cs_trace::OLD_BEST_HEADER_HEIGHT, + cs_trace::OLD_BEST_HEADER_HASH, + ), + "new" => ( + cs_trace::NEW_FINALIZED_HEIGHT, + cs_trace::NEW_FINALIZED_HASH, + cs_trace::NEW_VERIFIED_BODY_HEIGHT, + cs_trace::NEW_VERIFIED_BODY_HASH, + cs_trace::NEW_BEST_HEADER_HEIGHT, + cs_trace::NEW_BEST_HEADER_HASH, + ), + _ => return, + }; + + insert_height(row, finalized_height, frontier.finalized.height); + insert_hash(row, finalized_hash, frontier.finalized.hash); + insert_height(row, verified_body_height, frontier.verified_body.height); + insert_hash(row, verified_body_hash, frontier.verified_body.hash); + insert_height(row, best_header_height, frontier.best_header.height); + insert_hash(row, best_header_hash, frontier.best_header.hash); +} + +fn insert_height( + row: &mut serde_json::Map, + key: &'static str, + height: block::Height, +) { + row.insert( + key.to_string(), + Value::Number(Number::from(u64::from(height.0))), + ); +} + +fn insert_hash(row: &mut serde_json::Map, key: &'static str, hash: block::Hash) { + row.insert(key.to_string(), Value::String(format!("{hash}"))); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + sync::{Arc, Barrier}, + thread, + }; + + fn frontier(height: u32, seed: u8) -> Frontier { + Frontier::new(block::Height(height), block::Hash([seed; 32])) + } + + fn update( + finalized: Frontier, + verified_body: Frontier, + best_header: Frontier, + change: FrontierChange, + ) -> FrontierUpdate { + FrontierUpdate { + frontier: ChainFrontier { + finalized, + verified_body, + best_header, + }, + change, + } + } + + #[test] + fn grow_advances_verified_body() { + let current = update( + frontier(1, 1), + frontier(2, 2), + frontier(5, 5), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(3, 3), + frontier(4, 4), + frontier(9, 9), + FrontierChange::VerifiedGrow, + ); + + let updated = apply_frontier_update(current, requested).expect("grow is accepted"); + + assert_eq!(updated.frontier.verified_body, frontier(4, 4)); + assert_eq!(updated.frontier.best_header, frontier(5, 5)); + } + + #[test] + fn grow_may_not_lower_best_header() { + let current = update( + frontier(1, 1), + frontier(2, 2), + frontier(9, 9), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(3, 3), + frontier(4, 4), + frontier(5, 5), + FrontierChange::VerifiedGrow, + ); + + let updated = apply_frontier_update(current, requested).expect("grow is accepted"); + + assert_eq!(updated.frontier.finalized, frontier(3, 3)); + assert_eq!(updated.frontier.verified_body, frontier(4, 4)); + assert_eq!(updated.frontier.best_header, frontier(9, 9)); + } + + #[test] + fn stale_lower_grow_is_ignored() { + let current = update( + frontier(1, 1), + frontier(8, 8), + frontier(9, 9), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(1, 1), + frontier(7, 7), + frontier(9, 9), + FrontierChange::VerifiedGrow, + ); + + assert!(apply_frontier_update(current, requested).is_none()); + } + + #[test] + fn equal_height_grow_refreshes_verified_hash() { + let current = update( + frontier(1, 1), + frontier(8, 8), + frontier(9, 9), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(1, 1), + frontier(8, 18), + frontier(9, 19), + FrontierChange::VerifiedGrow, + ); + + let updated = + apply_frontier_update(current, requested).expect("equal height grow is accepted"); + + assert_eq!(updated.frontier.verified_body, frontier(8, 18)); + assert_eq!(updated.frontier.best_header, frontier(9, 9)); + } + + #[test] + fn reset_may_lower_verified_body() { + let current = update( + frontier(5, 5), + frontier(8, 8), + frontier(10, 10), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(4, 4), + frontier(6, 6), + frontier(2, 2), + FrontierChange::VerifiedReset, + ); + + let updated = apply_frontier_update(current, requested).expect("reset is accepted"); + + assert_eq!(updated.frontier.finalized, frontier(5, 5)); + assert_eq!(updated.frontier.verified_body, frontier(6, 6)); + assert_eq!(updated.frontier.best_header, frontier(10, 10)); + } + + #[test] + fn finalized_height_never_decreases_across_mixed_updates() { + let mut current = update( + frontier(10, 10), + frontier(10, 10), + frontier(12, 12), + FrontierChange::Snapshot, + ); + for requested in [ + update( + frontier(9, 9), + frontier(11, 11), + frontier(12, 12), + FrontierChange::VerifiedGrow, + ), + update( + frontier(1, 1), + frontier(7, 7), + frontier(0, 0), + FrontierChange::VerifiedReset, + ), + update( + frontier(8, 8), + frontier(8, 8), + frontier(20, 20), + FrontierChange::HeaderAdvanced, + ), + update( + frontier(3, 3), + frontier(3, 3), + frontier(13, 13), + FrontierChange::HeaderReanchored, + ), + ] { + if let Some(updated) = apply_frontier_update(current, requested) { + current = updated; + assert!( + current.frontier.finalized.height >= block::Height(10), + "finalized frontier must never move below the original finalized height" + ); + } + } + } + + #[test] + fn snapshot_does_not_lower_existing_frontiers() { + let current = update( + frontier(5, 5), + frontier(8, 8), + frontier(12, 12), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(4, 4), + frontier(7, 7), + frontier(11, 11), + FrontierChange::Snapshot, + ); + + assert!(apply_frontier_update(current, requested).is_none()); + } + + #[test] + fn header_reanchor_lowers_only_best_header() { + let current = update( + frontier(5, 5), + frontier(8, 8), + frontier(12, 12), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(1, 1), + frontier(2, 2), + frontier(9, 9), + FrontierChange::HeaderReanchored, + ); + + let updated = apply_frontier_update(current, requested).expect("reanchor is accepted"); + + assert_eq!(updated.frontier.finalized, frontier(5, 5)); + assert_eq!(updated.frontier.verified_body, frontier(8, 8)); + assert_eq!(updated.frontier.best_header, frontier(9, 9)); + } + + #[test] + fn header_advance_cannot_change_verified_body() { + let current = update( + frontier(5, 5), + frontier(8, 8), + frontier(12, 12), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(1, 1), + frontier(2, 2), + frontier(13, 13), + FrontierChange::HeaderAdvanced, + ); + + let updated = apply_frontier_update(current, requested).expect("advance is accepted"); + + assert_eq!(updated.frontier.finalized, frontier(5, 5)); + assert_eq!(updated.frontier.verified_body, frontier(8, 8)); + assert_eq!(updated.frontier.best_header, frontier(13, 13)); + } + + #[test] + fn stale_header_advance_is_ignored() { + let current = update( + frontier(5, 5), + frontier(8, 8), + frontier(12, 12), + FrontierChange::Snapshot, + ); + let requested = update( + frontier(20, 20), + frontier(21, 21), + frontier(12, 22), + FrontierChange::HeaderAdvanced, + ); + + assert!(apply_frontier_update(current, requested).is_none()); + } + + #[test] + fn exchange_keeps_latest_value_without_external_receivers() { + let initial = update( + frontier(0, 0), + frontier(0, 0), + frontier(0, 0), + FrontierChange::Snapshot, + ); + let exchange = ZakuraSyncExchange::new(initial, ZakuraTrace::noop()); + let accepted = update( + frontier(0, 0), + frontier(0, 0), + frontier(10, 10), + FrontierChange::HeaderAdvanced, + ); + + exchange.publish_frontier(accepted, "test"); + + assert_eq!(exchange.current_frontier(), accepted); + + let stale = update( + frontier(0, 0), + frontier(0, 0), + frontier(9, 9), + FrontierChange::HeaderAdvanced, + ); + exchange.publish_frontier(stale, "test"); + + assert_eq!(exchange.current_frontier(), accepted); + } + + #[test] + fn concurrent_publishers_converge_to_highest_frontiers() { + const PUBLISHERS: u32 = 64; + + let initial = update( + frontier(0, 0), + frontier(0, 0), + frontier(0, 0), + FrontierChange::Snapshot, + ); + let exchange = ZakuraSyncExchange::new(initial, ZakuraTrace::noop()); + let barrier = Arc::new(Barrier::new((PUBLISHERS * 2 + 1) as usize)); + let mut publishers = Vec::new(); + + for height in 1..=PUBLISHERS { + let seed = u8::try_from(height).expect("test publisher height fits in u8"); + let header_exchange = exchange.clone(); + let header_barrier = barrier.clone(); + publishers.push(thread::spawn(move || { + header_barrier.wait(); + header_exchange.publish_frontier( + update( + frontier(0, 0), + frontier(0, 0), + frontier(height, seed), + FrontierChange::HeaderAdvanced, + ), + "test", + ); + })); + + let seed = u8::try_from(height).expect("test publisher height fits in u8"); + let grow_exchange = exchange.clone(); + let grow_barrier = barrier.clone(); + publishers.push(thread::spawn(move || { + grow_barrier.wait(); + grow_exchange.publish_frontier( + update( + frontier(height, seed), + frontier(height, seed), + frontier(0, 0), + FrontierChange::VerifiedGrow, + ), + "test", + ); + })); + } + + barrier.wait(); + for publisher in publishers { + publisher.join().expect("publisher thread should not panic"); + } + + let current = exchange.current_frontier().frontier; + let seed = u8::try_from(PUBLISHERS).expect("test publisher height fits in u8"); + assert_eq!(current.finalized, frontier(PUBLISHERS, seed)); + assert_eq!(current.verified_body, frontier(PUBLISHERS, seed)); + assert_eq!(current.best_header, frontier(PUBLISHERS, seed)); + } + + #[test] + fn concurrent_header_advance_is_not_lost_to_verified_grow() { + let initial = update( + frontier(0, 0), + frontier(5, 5), + frontier(10, 10), + FrontierChange::Snapshot, + ); + let exchange = ZakuraSyncExchange::new(initial, ZakuraTrace::noop()); + let barrier = Arc::new(Barrier::new(3)); + + let header_exchange = exchange.clone(); + let header_barrier = barrier.clone(); + let header = thread::spawn(move || { + header_barrier.wait(); + header_exchange.publish_frontier( + update( + frontier(0, 0), + frontier(0, 0), + frontier(20, 20), + FrontierChange::HeaderAdvanced, + ), + "test", + ); + }); + + let grow_exchange = exchange.clone(); + let grow_barrier = barrier.clone(); + let grow = thread::spawn(move || { + grow_barrier.wait(); + grow_exchange.publish_frontier( + update( + frontier(0, 0), + frontier(8, 8), + frontier(0, 0), + FrontierChange::VerifiedGrow, + ), + "test", + ); + }); + + barrier.wait(); + header.join().expect("header publisher should not panic"); + grow.join().expect("grow publisher should not panic"); + + let current = exchange.current_frontier().frontier; + assert_eq!(current.verified_body, frontier(8, 8)); + assert_eq!(current.best_header, frontier(20, 20)); + } +} diff --git a/zebra-network/src/zakura/handler.rs b/zebra-network/src/zakura/handler.rs index fab9023f718..c014f700e60 100644 --- a/zebra-network/src/zakura/handler.rs +++ b/zebra-network/src/zakura/handler.rs @@ -4,9 +4,8 @@ use std::{ collections::{HashMap, HashSet}, future, io::{Cursor, Read}, - net::{IpAddr, SocketAddr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, path::PathBuf, - str::FromStr, sync::{ atomic::{AtomicU64, Ordering}, Arc, Mutex as StdMutex, @@ -17,15 +16,17 @@ use std::{ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use iroh::Watcher as _; use iroh::{ - endpoint::{Connection, RecvStream, SendStream, TransportConfig, VarInt}, + endpoint::{ + Connection, ConnectionType, Endpoint, RecvStream, SendStream, TransportConfig, VarInt, + }, protocol::{AcceptError, ProtocolHandler, Router}, - NodeAddr, SecretKey, + NodeAddr, NodeId, SecretKey, }; use rand::{rngs::OsRng, RngCore}; use thiserror::Error; use tokio::{ sync::{mpsc, oneshot, watch, Mutex, OwnedSemaphorePermit, Semaphore}, - task::{JoinHandle, JoinSet}, + task::{AbortHandle, JoinHandle, JoinSet}, time::{timeout, Instant}, }; use tokio_util::sync::CancellationToken; @@ -47,18 +48,21 @@ use super::{ use crate::{ protocol::external::InventoryHash, zakura::{ - direct_endpoint_builder, drive_header_sync_actions, spawn_header_sync_reactor, Clock, - Frame, FramedRecv, FramedSend, HeaderSyncAction, HeaderSyncFrontiers, + direct_endpoint_builder, drive_header_sync_actions, spawn_block_sync_reactor, + spawn_header_sync_reactor, BlockSyncAction, BlockSyncFrontiers, BlockSyncHandle, + BlockSyncService, BlockSyncStartup, Clock, Frame, FramedRecv, FramedSend, Frontier, + FrontierChange, FrontierUpdate, HeaderSyncAction, HeaderSyncFrontiers, HeaderSyncPassthroughService, HeaderSyncService, HeaderSyncStartup, Peer, RealClock, Service, ServicePeerDirection, ServiceRegistry, ServiceStream, SinkReject, Stream, - StreamMode, StreamPrelude, ZakuraAcceptedLimits, ZakuraControlAck, ZakuraControlHello, - ZakuraControlRole, ZakuraControlValidation, ZakuraHandshakeConfig, ZakuraHandshakePath, - ZakuraHeaderSyncConfig, ZakuraInitialLimits, ZakuraLimits, ZakuraPeerId, - ZakuraPeerSupervisor, ZakuraProtocolError, ZakuraRejectReason, ZakuraUpgradeOutcome, - CONTROL_ACK_MAGIC, CONTROL_HELLO_MAGIC, CONTROL_VERSION, FRAME_HEADER_BYTES, - LOCAL_MAX_CONTROL_FRAME_BYTES, MAX_HS_MESSAGE_BYTES, P2P_V2_ALPN, STREAM_PRELUDE_MAGIC, + StreamMode, StreamPrelude, ZakuraAcceptedLimits, ZakuraBlockSyncConfig, ZakuraControlAck, + ZakuraControlHello, ZakuraControlRole, ZakuraControlValidation, ZakuraHandshakeConfig, + ZakuraHandshakePath, ZakuraHeaderSyncConfig, ZakuraInitialLimits, ZakuraLimits, + ZakuraPeerId, ZakuraPeerSupervisor, ZakuraProtocolError, ZakuraRejectReason, + ZakuraSyncExchange, ZakuraUpgradeOutcome, CONTROL_ACK_MAGIC, CONTROL_HELLO_MAGIC, + CONTROL_VERSION, FRAME_HEADER_BYTES, LOCAL_MAX_CONTROL_FRAME_BYTES, MAX_BS_FRAME_BYTES, + MAX_CONTROL_PAYLOAD_BYTES, MAX_HS_MESSAGE_BYTES, P2P_V2_ALPN, STREAM_PRELUDE_MAGIC, TRANSCRIPT_HASH_BYTES, ZAKURA_HEADER_SYNC_STREAM_VERSION, ZAKURA_PROTOCOL_VERSION_1, - ZAKURA_STREAM_HEADER_SYNC, + ZAKURA_STREAM_BLOCK_SYNC, ZAKURA_STREAM_HEADER_SYNC, }, }; use crate::{BoxError, Config, MAX_TX_INV_IN_SENT_MESSAGE}; @@ -69,8 +73,19 @@ pub const DEFAULT_ZAKURA_MAX_CONNECTIONS: usize = 32; pub const DEFAULT_ZAKURA_MAX_PENDING_HANDSHAKES: usize = 8; /// Conservative default for stream-open churn per connection. pub const DEFAULT_ZAKURA_STREAM_OPEN_RATE_PER_SECOND: u32 = 16; -/// Conservative default for per-kind message rate per connection. -pub const DEFAULT_ZAKURA_MESSAGE_RATE_PER_SECOND: u32 = 128; +/// Per-kind inbound message rate per connection. +/// +/// This is a generous universal cap: block-sync legitimately delivers +/// hundreds of solicited bodies per second in bursts, so a low limit +/// starves sync. Exceeding it is treated as misbehavior and disconnects the +/// peer (we never silently drop a solicited frame -- a dropped block body is +/// a permanent gap on a reliable stream). Longer term this should be split +/// per message type (some unbounded, some near-one-shot) rather than a single +/// universal value. +pub const DEFAULT_ZAKURA_MESSAGE_RATE_PER_SECOND: u32 = 2048; +/// Default native Zakura QUIC listen address. +pub const DEFAULT_ZAKURA_LISTEN_ADDR: SocketAddr = + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 8234)); /// Default maximum bytes read before the peer's stream prelude is decoded. pub const DEFAULT_ZAKURA_PRELUDE_TIMEOUT: Duration = Duration::from_secs(3); /// Default timeout for one control-handshake read or write. @@ -85,6 +100,23 @@ pub const DEFAULT_ZAKURA_CONTROL_TIMEOUT: Duration = Duration::from_secs(10); pub const DEFAULT_ZAKURA_QUIC_IDLE_TIMEOUT: Duration = Duration::from_secs(150); /// QUIC keepalive interval used by Zakura endpoints. pub const DEFAULT_ZAKURA_KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(10); +/// Minimum age of an incumbent Zakura connection before a duplicate connection +/// for the same identity is allowed to evict it. +/// +/// A duplicate almost always means the peer restarted or redialed (the redial +/// supervisor skips peers that are already registered). If the incumbent has +/// been registered longer than this, it is treated as a stale connection left +/// behind by a restarted peer: that peer's process is gone, so the connection +/// only lingers until the QUIC idle timeout ([`DEFAULT_ZAKURA_QUIC_IDLE_TIMEOUT`], +/// ~150s). Evicting it immediately lets the peer's redial take the freed slot in +/// seconds instead of stalling for the whole idle window. +/// +/// A younger incumbent is kept: two connections registered close together are a +/// simultaneous-open race (both sides dialed before either registered), and +/// evicting on every duplicate would make those flap. The redial backoff +/// resolves that race instead. This threshold must stay above the worst-case +/// simultaneous-dial window and below the QUIC idle timeout. +pub const ZAKURA_DUPLICATE_EVICT_MIN_AGE: Duration = Duration::from_secs(30); /// QUIC stream receive window used by Zakura endpoints. pub const DEFAULT_ZAKURA_STREAM_RECEIVE_WINDOW: u32 = 512 * 1024; /// QUIC connection receive window used by Zakura endpoints. @@ -131,12 +163,26 @@ const LEGACY_COMPACT_SIZE_PREFIX_BYTES: usize = 9; const LEGACY_BLOCK_HASH_BYTES: usize = 32; const LEGACY_INVENTORY_HASH_BYTES: usize = 36; const LEGACY_RESPONSE_MAX_FRAMES_PER_ITEM: usize = 8; +/// Maximum cumulative response payload bytes the requester will retain in the +/// accepted-frame `Vec` before `decode_response` consumes it. +/// +/// `LegacyResponseBudget::from_request` otherwise derives the per-response byte +/// budget for Blocks/Transactions as `item_count * max_message_bytes`, so a +/// request naming the protocol-max inventory count (`MAX_TX_INV_IN_SENT_MESSAGE`) +/// would let a hostile responder fill ~`25_000 * MAX_PROTOCOL_MESSAGE_LEN` (tens +/// of GiB) of validated frames before any decode begins. The inbound responder +/// already caps a single response's cumulative payload at the same value +/// (`legacy_gossip::LEGACY_RESPONSE_MAX_AGGREGATE_BYTES`), so an honest peer +/// never sends more than this; clamping the requester budget to the same +/// operational aggregate is the symmetric requester-side mirror. +const LEGACY_RESPONSE_MAX_AGGREGATE_BYTES: usize = + 8 * zebra_chain::serialization::MAX_PROTOCOL_MESSAGE_LEN; const _: () = assert!(LEGACY_GOSSIP_STREAM_KIND == super::legacy_gossip::ZAKURA_STREAM_GOSSIP); const _: () = assert!(LEGACY_REQUEST_STREAM_KIND == super::legacy_gossip::ZAKURA_STREAM_LEGACY_REQUESTS); const _: () = assert!(DISCOVERY_STREAM_KIND == super::discovery::ZAKURA_STREAM_DISCOVERY); const _: () = assert!(HEADER_SYNC_STREAM_KIND == super::header_sync::ZAKURA_STREAM_HEADER_SYNC); -const _: () = assert!(ZAKURA_STREAM_VERSION_1 == ZAKURA_HEADER_SYNC_STREAM_VERSION); +const _: () = assert!(ZAKURA_STREAM_VERSION_2 == ZAKURA_HEADER_SYNC_STREAM_VERSION); const _: () = assert!(LEGACY_REQUEST_BLOCKS_BY_HASH == super::legacy_gossip::MSG_REQUEST_BLOCKS_BY_HASH); const _: () = assert!( @@ -178,7 +224,6 @@ const ZAKURA_CLOSE_OVERSIZE: u32 = 4; /// from the malformed-prelude code so peers can tell a parse failure from an /// unsupported-but-well-formed stream. const ZAKURA_CLOSE_UNKNOWN_STREAM: u32 = 5; -const NATIVE_TRANSCRIPT_HASH: [u8; TRANSCRIPT_HASH_BYTES] = [0; TRANSCRIPT_HASH_BYTES]; /// Native Zakura endpoint and handler configuration. #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -192,12 +237,13 @@ pub struct ZakuraConfig { pub bootstrap_peers: Vec, /// Address the native Zakura QUIC endpoint binds to. /// - /// When unset the endpoint binds an OS-assigned ephemeral port on the - /// unspecified address, which is fine for a node that only dials out. Set a - /// fixed address to give this node a stable, advertisable Zakura endpoint so - /// other nodes can list it in their [`bootstrap_peers`](Self::bootstrap_peers) - /// — required for a node that acts as a Zakura seed, since relays and - /// discovery are disabled. + /// Defaults to `0.0.0.0:8234`, giving the node a stable, advertisable + /// Zakura endpoint so other nodes can list it in their + /// [`bootstrap_peers`](Self::bootstrap_peers). If code constructs this as + /// `None`, the endpoint binds an OS-assigned ephemeral port on loopback + /// only (`127.0.0.1` and `::1`), which is fine for a node that only dials + /// out and keeps the experimental native P2P_V2_ALPN surface off all + /// non-loopback interfaces. pub listen_addr: Option, /// Total concurrent Zakura connections, inbound plus outbound. pub max_connections: usize, @@ -214,19 +260,22 @@ pub struct ZakuraConfig { pub trace_dir: Option, /// Native stream-5 header-sync wire settings. pub header_sync: ZakuraHeaderSyncConfig, + /// Native stream-6 block-sync wire, scheduling, serving, and rollout settings. + pub block_sync: ZakuraBlockSyncConfig, } impl Default for ZakuraConfig { fn default() -> Self { Self { bootstrap_peers: Vec::new(), - listen_addr: None, + listen_addr: Some(DEFAULT_ZAKURA_LISTEN_ADDR), max_connections: DEFAULT_ZAKURA_MAX_CONNECTIONS, max_pending_handshakes: DEFAULT_ZAKURA_MAX_PENDING_HANDSHAKES, stream_open_rate_per_second: DEFAULT_ZAKURA_STREAM_OPEN_RATE_PER_SECOND, message_rate_per_second: DEFAULT_ZAKURA_MESSAGE_RATE_PER_SECOND, trace_dir: None, header_sync: ZakuraHeaderSyncConfig::default(), + block_sync: ZakuraBlockSyncConfig::default(), } } } @@ -373,9 +422,16 @@ pub struct ZakuraEndpoint { supervisor: ZakuraSupervisorHandle, handler: ZakuraProtocolHandler, header_sync: Option, + block_sync: Option, + sync_frontier: Option, header_sync_tasks: Option>, header_sync_actions: Option>>>>, - upgrade_dials: Arc>>, + block_sync_actions: Option>>>>, + /// Maintained native dials started by the legacy->Zakura upgrade hand-off, + /// keyed by the advertised peer id. The [`AbortHandle`] lets a failed + /// hand-off cancel its maintain-forever dial instead of leaking it; see + /// [`Self::ensure_upgrade_native_dial`] and [`Self::cancel_upgrade_native_dial`]. + upgrade_dials: Arc>>, } #[derive(Debug)] @@ -391,6 +447,8 @@ pub struct ZakuraHeaderSyncDriverStartup { pub frontiers: HeaderSyncFrontiers, /// Durable best header tip loaded from state. pub best_header_tip: Option<(block::Height, block::Hash)>, + /// Hash of `frontiers.verified_block_tip`. + pub verified_block_tip_hash: block::Hash, } impl ZakuraEndpoint { @@ -431,12 +489,49 @@ impl ZakuraEndpoint { self.header_sync.clone() } + /// Returns the block-sync handle when native block sync is active. + pub fn block_sync(&self) -> Option { + self.block_sync.clone() + } + + /// Subscribe to the shared Zakura sync frontier stream. + pub fn subscribe_sync_frontier(&self) -> Option> { + self.sync_frontier + .as_ref() + .map(ZakuraSyncExchange::subscribe_frontier) + } + + /// Return the currently cached shared Zakura sync frontier update. + pub fn current_sync_frontier(&self) -> Option { + self.sync_frontier + .as_ref() + .map(ZakuraSyncExchange::current_frontier) + } + + /// Publish a shared Zakura sync frontier update. + pub fn publish_sync_frontier(&self, update: FrontierUpdate) { + self.publish_sync_frontier_from(update, "unknown"); + } + + /// Publish a shared Zakura sync frontier update with a trace source. + pub fn publish_sync_frontier_from(&self, update: FrontierUpdate, source: &'static str) { + if let Some(sync_frontier) = &self.sync_frontier { + sync_frontier.publish_frontier(update, source); + } + } + /// Take the header-sync action receiver when this endpoint was started in external-driver mode. pub async fn take_header_sync_actions(&self) -> Option> { let actions = self.header_sync_actions.as_ref()?; actions.lock().await.take() } + /// Take the block-sync action receiver when this endpoint was started in external-driver mode. + pub async fn take_block_sync_actions(&self) -> Option> { + let actions = self.block_sync_actions.as_ref()?; + actions.lock().await.take() + } + /// Returns the endpoint-owned header-sync shutdown token. pub fn header_sync_shutdown(&self) -> Option { self.header_sync_tasks @@ -444,6 +539,22 @@ impl ZakuraEndpoint { .map(|tasks| tasks.shutdown.clone()) } + /// The endpoint-wide background-task shutdown token, cancelled by + /// [`ZakuraEndpoint::shutdown`]. + /// + /// Detached dial/discovery loops observe this so they stop promptly at + /// teardown instead of running on against a torn-down router. They each hold + /// an endpoint clone, which keeps the supervisor registration watch open, so + /// watch closure is *not* a reliable exit signal for them; this token is. + /// Falls back to an un-cancelled token for endpoints built without a + /// background-task owner (recorder-only test nodes). + pub(crate) fn background_shutdown_token(&self) -> CancellationToken { + self.header_sync_tasks + .as_ref() + .map(|tasks| tasks.shutdown.clone()) + .unwrap_or_default() + } + /// Track a header-sync integration task under the endpoint shutdown owner. pub async fn push_header_sync_task(&self, task: JoinHandle<()>) { if let Some(tasks) = self.header_sync_tasks.as_ref() { @@ -451,6 +562,11 @@ impl ZakuraEndpoint { } } + /// Track a block-sync integration task under the endpoint shutdown owner. + pub async fn push_block_sync_task(&self, task: JoinHandle<()>) { + self.push_header_sync_task(task).await; + } + /// Returns the endpoint's current direct node address. pub async fn node_addr(&self) -> NodeAddr { self.router.endpoint().node_addr().initialized().await @@ -496,14 +612,17 @@ impl ZakuraEndpoint { return false; }; - { - let mut upgrade_dials = self - .upgrade_dials - .lock() - .expect("Zakura upgrade dial registry mutex is never poisoned"); - if !upgrade_dials.insert(peer_id.clone()) { - return true; - } + // Hold the registry lock across the spawn so the dedup check and the + // abort-handle insert are atomic against a concurrent upgrade to the + // same peer. `tokio::spawn` does not await, and the spawned task only + // re-locks after its (non-instant) dial returns, so this cannot + // deadlock or `.await` under the lock. + let mut upgrade_dials = self + .upgrade_dials + .lock() + .expect("Zakura upgrade dial registry mutex is never poisoned"); + if upgrade_dials.contains_key(&peer_id) { + return true; } let endpoint = self.clone(); @@ -512,17 +631,41 @@ impl ZakuraEndpoint { DEFAULT_ZAKURA_REDIAL_INITIAL_BACKOFF, DEFAULT_ZAKURA_REDIAL_MAX_BACKOFF, ); - tokio::spawn(async move { + let task_peer_id = peer_id.clone(); + let dial = tokio::spawn(async move { native_dial_supervised(endpoint.clone(), node_addr, limits, policy).await; endpoint .upgrade_dials .lock() .expect("Zakura upgrade dial registry mutex is never poisoned") - .remove(&peer_id); + .remove(&task_peer_id); }); + upgrade_dials.insert(peer_id, dial.abort_handle()); true } + /// Cancel and forget the maintained native dial started by the legacy + /// upgrade hand-off for `peer_id`, if this node still owns one. + /// + /// Called when the upgrade hand-off wait times out without the peer + /// registering: the maintained dial uses [`RedialPolicy::maintain`], so it + /// would otherwise redial a peer-supplied, possibly unreachable address + /// forever and keep its `upgrade_dials` entry. Repeating the failed upgrade + /// with distinct node ids would then grow maintained dial tasks and + /// outbound QUIC traffic without bound. A no-op if the peer already + /// registered (its entry is reclaimed only when the maintained dial ends on + /// shutdown) or if another upgrade owns the dedup slot. + pub(crate) fn cancel_upgrade_native_dial(&self, peer_id: &ZakuraPeerId) { + let handle = self + .upgrade_dials + .lock() + .expect("Zakura upgrade dial registry mutex is never poisoned") + .remove(peer_id); + if let Some(handle) = handle { + handle.abort(); + } + } + /// Returns whether the local admission semaphore has a free permit, i.e. /// whether this node can accept another inbound/dialed Zakura connection. /// Used by the discovery dialer to avoid starting candidate dials that would @@ -558,13 +701,17 @@ impl ZakuraEndpoint { supervisor, handler, header_sync: None, + block_sync: None, + sync_frontier: None, header_sync_tasks: None, header_sync_actions: None, - upgrade_dials: Arc::new(StdMutex::new(HashSet::new())), + block_sync_actions: None, + upgrade_dials: Arc::new(StdMutex::new(HashMap::new())), } } #[cfg(any(test, feature = "zakura-testkit"))] + #[allow(dead_code)] pub(crate) fn from_parts_with_header_sync( router: Router, supervisor: ZakuraSupervisorHandle, @@ -579,12 +726,47 @@ impl ZakuraEndpoint { supervisor, handler, header_sync: Some(header_sync), + block_sync: None, + sync_frontier: None, header_sync_tasks: Some(Arc::new(HeaderSyncBackgroundTasks { shutdown, tasks: Mutex::new(tasks), })), header_sync_actions: actions.map(|actions| Arc::new(Mutex::new(Some(actions)))), - upgrade_dials: Arc::new(StdMutex::new(HashSet::new())), + block_sync_actions: None, + upgrade_dials: Arc::new(StdMutex::new(HashMap::new())), + } + } + + #[cfg(any(test, feature = "zakura-testkit"))] + #[allow(clippy::too_many_arguments)] + pub(crate) fn from_parts_with_sync_services( + router: Router, + supervisor: ZakuraSupervisorHandle, + handler: ZakuraProtocolHandler, + header_sync: super::HeaderSyncHandle, + block_sync: BlockSyncHandle, + shutdown: CancellationToken, + tasks: Vec>, + header_sync_actions: Option>, + block_sync_actions: Option>, + ) -> Self { + Self { + router, + supervisor, + handler, + header_sync: Some(header_sync), + block_sync: Some(block_sync), + sync_frontier: None, + header_sync_tasks: Some(Arc::new(HeaderSyncBackgroundTasks { + shutdown, + tasks: Mutex::new(tasks), + })), + header_sync_actions: header_sync_actions + .map(|actions| Arc::new(Mutex::new(Some(actions)))), + block_sync_actions: block_sync_actions + .map(|actions| Arc::new(Mutex::new(Some(actions)))), + upgrade_dials: Arc::new(StdMutex::new(HashMap::new())), } } } @@ -607,6 +789,10 @@ struct ZakuraSupervisorState { outbound_by_peer: HashMap, disconnect_by_peer: HashMap, caps_by_peer: HashMap, + /// When each authenticated peer's current connection registered, used to + /// decide whether a duplicate may evict a stale incumbent (see + /// [`ZAKURA_DUPLICATE_EVICT_MIN_AGE`]). + registered_at: HashMap, active_by_ip: HashMap, max_connections_per_ip: usize, } @@ -695,6 +881,7 @@ impl ZakuraSupervisorHandle { outbound_by_peer: HashMap::new(), disconnect_by_peer: HashMap::new(), caps_by_peer: HashMap::new(), + registered_at: HashMap::new(), active_by_ip: HashMap::new(), max_connections_per_ip: max_connections_per_ip.max(1), })), @@ -754,15 +941,27 @@ impl ZakuraSupervisorHandle { accepted_capabilities: u64, ) -> ZakuraRegistration { let mut state = self.inner.lock().await; + // A re-registration for a peer id that is already active is a duplicate + // redial, not a new connection: the incumbent already holds the per-IP + // slot, and the duplicate branch below either keeps the incumbent and + // closes the newcomer or evicts a stale incumbent in its place, so it + // never consumes an additional per-IP slot. Exempting duplicates from the + // per-IP cap precheck lets a same-peer redial from an IP already at the cap + // reach the stale-incumbent eviction path instead of being rejected as a + // resource limit, so a dead incumbent is evicted in milliseconds rather + // than blocking the peer until the QUIC idle timeout (~150s). + let is_duplicate_redial = state.active_by_peer.contains_key(&peer_id); if let Some(remote_ip) = remote_ip { - let ip_count = state - .active_by_ip - .get(&remote_ip) - .copied() - .unwrap_or_default(); - if ip_count >= state.max_connections_per_ip { - metrics::counter!("zakura.p2p.conn.rejected.admission").increment(1); - return ZakuraRegistration::Rejected(ZakuraRejectReason::ResourceLimit); + if !is_duplicate_redial { + let ip_count = state + .active_by_ip + .get(&remote_ip) + .copied() + .unwrap_or_default(); + if ip_count >= state.max_connections_per_ip { + metrics::counter!("zakura.p2p.conn.rejected.admission").increment(1); + return ZakuraRegistration::Rejected(ZakuraRejectReason::ResourceLimit); + } } } @@ -786,6 +985,7 @@ impl ZakuraSupervisorHandle { state .caps_by_peer .insert(peer_id.clone(), accepted_capabilities); + state.registered_at.insert(peer_id.clone(), Instant::now()); let registered_ids: Vec<_> = state.active_by_peer.keys().cloned().collect(); set_active_connection_gauge(registered_ids.len()); self.peer_set_tx.send_replace(registered_ids); @@ -800,7 +1000,29 @@ impl ZakuraSupervisorHandle { disconnect_token, } } - ZakuraUpgradeOutcome::Duplicate { .. } => ZakuraRegistration::Duplicate { peer_id }, + ZakuraUpgradeOutcome::Duplicate { .. } => { + // A duplicate for an identity that already has a connection is + // almost always a restart or redial. If the incumbent has been + // registered long enough to be a stale connection left behind by + // a restarted peer, cancel it now so it tears down through its + // normal cleanup path and frees the slot in milliseconds; the + // peer's redial then takes the freed slot instead of waiting for + // the dead connection's QUIC idle timeout (~150s). A young + // incumbent is kept to avoid flapping on simultaneous-open races. + // The newcomer is still closed (its redial reconnects cleanly + // once the slot is free), which avoids racing the incumbent's + // service-registration teardown. + if let Some(registered_at) = state.registered_at.get(&peer_id) { + if registered_at.elapsed() >= ZAKURA_DUPLICATE_EVICT_MIN_AGE { + if let Some(token) = state.disconnect_by_peer.get(&peer_id) { + token.cancel(); + metrics::counter!("zakura.p2p.conn.duplicate.evicted_stale") + .increment(1); + } + } + } + ZakuraRegistration::Duplicate { peer_id } + } ZakuraUpgradeOutcome::Rejected { reason } => ZakuraRegistration::Rejected(reason), } } @@ -811,6 +1033,7 @@ impl ZakuraSupervisorHandle { state.outbound_by_peer.remove(peer_id); state.disconnect_by_peer.remove(peer_id); state.caps_by_peer.remove(peer_id); + state.registered_at.remove(peer_id); if let Some(remote_ip) = remote_ip { if let Some(count) = state.active_by_ip.get_mut(&remote_ip) { *count = count.saturating_sub(1); @@ -931,6 +1154,7 @@ struct ConnectionServeContext { accepted_capabilities: u64, role: &'static str, direction: ServicePeerDirection, + transcript_hash: [u8; TRANSCRIPT_HASH_BYTES], conn: ZakuraConnTrace, } @@ -1029,16 +1253,29 @@ pub(crate) struct NativeHandshakeNegotiated { pub(crate) fn service_registry( _supervisor: &ZakuraSupervisorHandle, header_sync: Option, + block_sync: Option, + block_sync_config: ZakuraBlockSyncConfig, legacy_service: Arc, discovery_service: Arc, ) -> Result, BoxError> { let mut services = vec![legacy_service.clone(), discovery_service]; - if let Some(header_sync) = header_sync { - services.push(Arc::new(HeaderSyncService::new(header_sync)) as Arc); + if let Some(header_sync) = &header_sync { + services.push(Arc::new(HeaderSyncService::new(header_sync.clone())) as Arc); } else { services .push(Arc::new(HeaderSyncPassthroughService::new(legacy_service)) as Arc); } + let block_sync = match block_sync { + Some(block_sync) => BlockSyncService::new_with_handle(block_sync_config, block_sync), + None => match header_sync.as_ref() { + Some(header_sync) => BlockSyncService::new_with_header_tip( + block_sync_config, + header_sync.subscribe_tip(), + ), + None => BlockSyncService::new(block_sync_config), + }, + }; + services.push(Arc::new(block_sync) as Arc); Ok(Arc::new( ServiceRegistry::new(services).map_err(|error| -> BoxError { Box::new(error) })?, @@ -1058,6 +1295,57 @@ pub struct ZakuraProtocolHandler { admission: Arc, pending_handshakes: Arc, shutdown: CancellationToken, + // Bound iroh endpoint, used to recover the inbound peer's UDP source IP so + // the per-IP admission cap applies to Router-accepted connections. Iroh's + // Router consumes the `Incoming` (which carries the address) before + // `ProtocolHandler::accept` hands us the established `Connection`, so the + // address is looked up from the endpoint's node map instead. `None` for + // unit tests that drive the handler without a bound endpoint, in which case + // inbound accepts fall back to the previous `remote_ip = None` behaviour. + endpoint: Option, +} + +/// Resolve the inbound peer's UDP source IP from the endpoint's node map so the +/// per-IP connection cap can be enforced for Router-accepted connections. +/// +/// Iroh exposes the peer address on `Incoming`, which the Router consumes before +/// `ProtocolHandler::accept`. After the native control handshake the connection +/// has exchanged real QUIC payload over its direct path, so the endpoint's node +/// map knows the peer's UDP address: prefer the connection's confirmed current +/// path (`conn_type`), then fall back to the direct address that most recently +/// delivered payload from this peer. Relay-only paths have no single +/// attributable source IP, so they correctly yield `None` (the global +/// connection cap still bounds them). +fn inbound_remote_ip(endpoint: &Endpoint, node_id: NodeId) -> Option { + if let Some(mut conn_type) = endpoint.conn_type(node_id) { + match conn_type.get() { + ConnectionType::Direct(addr) | ConnectionType::Mixed(addr, _) => { + return Some(addr.ip()) + } + ConnectionType::Relay(_) | ConnectionType::None => {} + } + } + endpoint.remote_info(node_id).and_then(|info| { + info.addrs + .into_iter() + .filter(|addr| addr.last_payload.is_some()) + // Smallest elapsed-since-last-payload is the path actively carrying + // this connection's traffic, i.e. the real inbound source address. + .min_by_key(|addr| addr.last_payload) + .map(|addr| addr.addr.ip()) + }) +} + +fn native_connection_transcript_hash( + direction: ServicePeerDirection, + local_node_id: &NodeId, + remote_node_id: &NodeId, +) -> [u8; TRANSCRIPT_HASH_BYTES] { + let initiator = match direction { + ServicePeerDirection::Inbound => remote_node_id, + ServicePeerDirection::Outbound => local_node_id, + }; + *initiator.as_bytes() } impl ZakuraProtocolHandler { @@ -1117,9 +1405,17 @@ impl ZakuraProtocolHandler { pending_handshakes: Arc::new(Semaphore::new(limits.max_pending_handshakes)), shutdown: CancellationToken::new(), limits, + endpoint: None, } } + /// Attach the bound iroh endpoint so inbound Router-accepted connections can + /// resolve the peer's UDP source IP and enforce the per-IP connection cap. + pub fn with_endpoint(mut self, endpoint: Endpoint) -> Self { + self.endpoint = Some(endpoint); + self + } + async fn accept_connection(&self, connection: Connection) -> Result<(), AcceptError> { let conn_id = self.next_conn_id.fetch_add(1, Ordering::Relaxed); let Ok(_admission) = self.admission.clone().try_acquire_owned() else { @@ -1154,12 +1450,24 @@ impl ZakuraProtocolHandler { }; let conn_limits = self.limits.clamp(&negotiated.limits); - // Iroh's Router hands ProtocolHandler only the established Connection. - // In iroh 0.92.0 the peer UDP address is exposed on Incoming, which the - // Router consumes before this point, not on Connection/Connecting. The - // inbound per-IP cap therefore remains deferred while 01.a keeps Router - // ownership. Native outbound dials still pass the configured direct IP. - let remote_ip = None; + // Iroh's Router hands ProtocolHandler only the established Connection; + // the peer UDP address lives on Incoming, which the Router consumes + // before this point. Recover it from the endpoint's node map so the + // per-IP admission cap applies to inbound accepts and a single source + // IP cannot fill the global connection budget with distinct node ids. + // The handshake above has already exchanged QUIC payload over the + // direct path, so the node map knows the peer's address. Native + // outbound dials still pass the configured direct IP. + let remote_ip = self + .endpoint + .as_ref() + .and_then(|endpoint| inbound_remote_ip(endpoint, remote_node_id)); + let local_node_id = self + .endpoint + .as_ref() + .map(|endpoint| endpoint.node_id()) + .unwrap_or(remote_node_id); + let direction = ServicePeerDirection::Inbound; self.register_and_serve( connection, remote_peer_id, @@ -1168,7 +1476,12 @@ impl ZakuraProtocolHandler { limits: conn_limits, accepted_capabilities: negotiated.accepted_capabilities, role: "responder", - direction: ServicePeerDirection::Inbound, + direction, + transcript_hash: native_connection_transcript_hash( + direction, + &local_node_id, + &remote_node_id, + ), conn, }, ) @@ -1304,6 +1617,13 @@ impl ZakuraProtocolHandler { let mut open_limiter = TokenBucket::new(limits.stream_open_rate_per_second); let mut message_buckets = MessageRateBuckets::new(); let (freshness_tx, freshness_rx) = watch::channel(Instant::now()); + // Bounded label describing why this connection closed. Updated at each + // local teardown site and attached to the final `closed.neutral` trace + // so readers can distinguish idle timeouts, peer-side closes, resource + // limits, and protocol faults. The default covers the external path: + // the connection token was cancelled by a disconnect request, peerset + // eviction, or process shutdown. + let mut close_reason: &'static str = "cancelled"; let negotiated_ordered_streams = self .registry .ordered_streams_for_negotiated(accepted_capabilities); @@ -1327,6 +1647,7 @@ impl ZakuraProtocolHandler { "closing Zakura peer because negotiated ordered streams exceed max-open-streams" ); connection.close(VarInt::from_u32(ZAKURA_CLOSE_RESOURCE), b"ordered streams"); + close_reason = "resource_ordered_streams"; connection_token.cancel(); } else if !ordered_streams.is_empty() && usize::from(limits.max_inbound_queue_depth) < ordered_streams.len() @@ -1337,6 +1658,7 @@ impl ZakuraProtocolHandler { "closing Zakura peer because inbound queue depth cannot be split across ordered streams" ); connection.close(VarInt::from_u32(ZAKURA_CLOSE_RESOURCE), b"queue split"); + close_reason = "resource_queue_split"; connection_token.cancel(); } let ordered_kinds: HashSet = negotiated_ordered_streams @@ -1395,6 +1717,7 @@ impl ZakuraProtocolHandler { VarInt::from_u32(ZAKURA_CLOSE_RESOURCE), b"ordered stream setup", ); + close_reason = "stream_setup_failed"; connection_token.cancel(); break; } @@ -1427,6 +1750,7 @@ impl ZakuraProtocolHandler { _ = connection_token.cancelled() => break, _ = freshness_reaper(freshness_rx.clone(), limits.idle_timeout), if run_freshness_reaper => { connection.close(VarInt::from_u32(ZAKURA_CLOSE_NEUTRAL), b"idle"); + close_reason = "idle_timeout"; break; } Some(joined) = workers.join_next() => { @@ -1462,6 +1786,7 @@ impl ZakuraProtocolHandler { stream_kind = admitted.kind, "closing peer after duplicate or unexpected ordered stream" ); + close_reason = "duplicate_stream"; connection_token.cancel(); continue; } @@ -1511,12 +1836,14 @@ impl ZakuraProtocolHandler { } Err(error) => { debug!(?error, "Zakura connection stopped accepting streams"); + close_reason = "accept_failed"; break; } } } outbound = outbound_rx.recv() => { let Some(outbound) = outbound else { + close_reason = "outbound_closed"; break; }; match outbound { @@ -1557,6 +1884,7 @@ impl ZakuraProtocolHandler { VarInt::from_u32(ZAKURA_CLOSE_BAD_PRELUDE), b"malformed response", ); + close_reason = "bad_response"; connection_token.cancel(); let _ = completion.send(Err(error)); } @@ -1583,7 +1911,10 @@ impl ZakuraProtocolHandler { } self.supervisor.deregister(&peer_id, remote_ip).await; metrics::counter!("zakura.p2p.conn.closed.neutral").increment(1); - self.trace.emit(CONN_TABLE, conn.event("closed.neutral")); + self.trace.emit( + CONN_TABLE, + conn.event("closed.neutral").reason(close_reason), + ); Ok(()) } @@ -1685,6 +2016,23 @@ impl ZakuraProtocolHandler { return None; }; + // Charge the per-connection stream-open rate token immediately after + // acquiring a concurrency permit, before parsing the prelude or running + // the kind/capability/mode checks below. Each of those rejections resets + // the stream only and keeps the connection, so charging the token here is + // what bounds protocol-invalid stream churn (bad prelude, unknown kind, + // unnegotiated capability): otherwise an authenticated peer could open + // such streams indefinitely without ever spending open-rate budget. + if !admission.open_limiter.try_take() { + let _ = send.reset(VarInt::from_u32(ZAKURA_CLOSE_RATE_LIMIT)); + metrics::counter!("zakura.p2p.stream.rejected.open_rate").increment(1); + admission.trace.emit( + STREAM_TABLE, + admission.event("rejected.open_rate", stream_id), + ); + return None; + } + let prelude = match read_stream_prelude(&mut recv, admission.limits.prelude_timeout).await { Ok(prelude) => prelude, Err(error) => { @@ -1773,18 +2121,6 @@ impl ZakuraProtocolHandler { return None; } - if !admission.open_limiter.try_take() { - let _ = send.reset(VarInt::from_u32(ZAKURA_CLOSE_RATE_LIMIT)); - metrics::counter!("zakura.p2p.stream.rejected.open_rate").increment(1); - admission.trace.emit( - STREAM_TABLE, - admission - .event("rejected.open_rate", stream_id) - .stream_kind(stream_kind), - ); - return None; - } - metrics::counter!( "zakura.p2p.stream.accepted", "stream_kind" => stream_kind, @@ -1892,7 +2228,7 @@ impl ZakuraProtocolHandler { .register( peer_id, remote_ip, - NATIVE_TRANSCRIPT_HASH, + context.transcript_hash, outbound_handle, connection_token.clone(), context.accepted_capabilities, @@ -1975,6 +2311,35 @@ impl ProtocolHandler for ZakuraProtocolHandler { } } +/// Loopback IPv4 address the native Zakura endpoint binds to when no +/// `zakura.listen_addr` is configured, so the unset (dial-out-only) state does +/// not expose the P2P_V2_ALPN surface on all interfaces. +const ZAKURA_LOOPBACK_BIND_V4: SocketAddrV4 = SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0); +/// Loopback IPv6 counterpart of [`ZAKURA_LOOPBACK_BIND_V4`]. +const ZAKURA_LOOPBACK_BIND_V6: SocketAddrV6 = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0); + +/// Applies the native endpoint bind-address selection to `builder`. +/// +/// When `listen_addr` is set, the endpoint binds exactly that address so the +/// node has a stable, advertisable Zakura endpoint. When it is unset, the +/// endpoint binds loopback-only for **both** IPv4 and IPv6 rather than relying +/// on iroh's default unspecified bind (`0.0.0.0:0` / `[::]:0`). Without the +/// explicit loopback bind the unconfigured / dial-out-only case silently exposes +/// the experimental native P2P_V2_ALPN handshake/session surface on every +/// interface on an OS-assigned ephemeral port. +fn bind_native_endpoint( + builder: iroh::endpoint::Builder, + listen_addr: Option, +) -> iroh::endpoint::Builder { + match listen_addr { + Some(SocketAddr::V4(addr)) => builder.bind_addr_v4(addr), + Some(SocketAddr::V6(addr)) => builder.bind_addr_v6(addr), + None => builder + .bind_addr_v4(ZAKURA_LOOPBACK_BIND_V4) + .bind_addr_v6(ZAKURA_LOOPBACK_BIND_V6), + } +} + /// Start a Zakura endpoint and router when P2P v2 is enabled. pub async fn spawn_zakura_endpoint( config: &Config, @@ -1997,15 +2362,11 @@ pub async fn spawn_zakura_endpoint_with_header_sync_driver( validate_idle_invariant(&limits)?; let secret_key = zakura_secret_key(config)?; let discovery_secret_key = secret_key.clone(); - let mut builder = - direct_endpoint_builder(secret_key).transport_config(limits.transport_config()); + let builder = direct_endpoint_builder(secret_key).transport_config(limits.transport_config()); // Bind a fixed address when configured so this node has a stable, advertisable - // Zakura endpoint; otherwise iroh assigns an ephemeral port (dial-out only). - match config.zakura.listen_addr { - Some(SocketAddr::V4(addr)) => builder = builder.bind_addr_v4(addr), - Some(SocketAddr::V6(addr)) => builder = builder.bind_addr_v6(addr), - None => {} - } + // Zakura endpoint; otherwise bind loopback-only so the unset (dial-out-only) + // case does not expose the native P2P_V2_ALPN surface on all interfaces. + let builder = bind_native_endpoint(builder, config.zakura.listen_addr); let endpoint = builder.bind().await?; let supervisor = ZakuraSupervisorHandle::new(config.max_connections_per_ip); let tracer = config @@ -2030,12 +2391,28 @@ pub async fn spawn_zakura_endpoint_with_header_sync_driver( HeaderSyncFrontiers { finalized_height: anchor.0, verified_block_tip: anchor.0, + verified_block_hash: anchor.1, }, |startup| startup.frontiers, ); let best_header_tip = header_sync_driver_startup .as_ref() .map_or(Some(anchor), |startup| startup.best_header_tip); + let sync_frontier = header_sync_driver_startup.as_ref().map(|driver_startup| { + let best_header_tip = driver_startup.best_header_tip.unwrap_or(anchor); + let initial = FrontierUpdate { + frontier: crate::zakura::chain_frontier_from_parts( + driver_startup.frontiers.finalized_height, + Frontier::new( + driver_startup.frontiers.verified_block_tip, + driver_startup.verified_block_tip_hash, + ), + Frontier::new(best_header_tip.0, best_header_tip.1), + ), + change: FrontierChange::Snapshot, + }; + ZakuraSyncExchange::new(initial, trace.clone()) + }); let mut startup = HeaderSyncStartup::new( config.network.clone(), anchor, @@ -2046,26 +2423,60 @@ pub async fn spawn_zakura_endpoint_with_header_sync_driver( ); startup.status_refresh_interval = config.zakura.header_sync.status_refresh_interval; startup.trace = trace.clone(); + startup.frontier_updates = sync_frontier + .as_ref() + .map(ZakuraSyncExchange::subscribe_frontier); let header_sync_shutdown = CancellationToken::new(); startup.shutdown = header_sync_shutdown.clone(); if header_sync_driver_startup.is_some() { startup.range_state_actions_enabled = true; - startup.inbound_new_block_acceptance_enabled = true; + startup.inbound_new_block_acceptance_enabled = config.zakura.header_sync.accept_new_blocks; } let (header_sync, header_sync_actions, header_sync_task) = spawn_header_sync_reactor(startup)?; - let discovery_service = Arc::new(super::DiscoveryService::with_header_sync( + let block_sync_driver_enabled = header_sync_driver_startup.is_some(); + let (block_sync, block_sync_actions, block_sync_task) = + if let Some(driver_startup) = header_sync_driver_startup.as_ref() { + let best_header_tip = driver_startup.best_header_tip.unwrap_or(anchor); + let frontier_updates = sync_frontier + .as_ref() + .expect("sync frontier is initialized when block sync driver is enabled") + .subscribe_frontier(); + let mut startup = BlockSyncStartup::new_with_exchange( + BlockSyncFrontiers { + finalized_height: driver_startup.frontiers.finalized_height, + verified_block_tip: driver_startup.frontiers.verified_block_tip, + verified_block_hash: driver_startup.verified_block_tip_hash, + }, + best_header_tip, + frontier_updates, + config.zakura.block_sync.clone(), + ); + startup.shutdown = header_sync_shutdown.clone(); + startup.trace = trace.clone(); + let (handle, actions, task) = spawn_block_sync_reactor(startup); + (Some(handle), Some(actions), Some(task)) + } else { + (None, None, None) + }; + let discovery_service = Arc::new(super::DiscoveryService::with_sync_services( discovery.clone(), header_sync.clone(), + block_sync.clone(), )) as Arc; let legacy_service = sink_factory(supervisor.clone(), trace.clone()); let registry = service_registry( &supervisor, Some(header_sync.clone()), + block_sync.clone(), + config.zakura.block_sync.clone(), legacy_service, discovery_service, )?; let mut tasks = vec![header_sync_task]; - let header_sync_actions = if header_sync_driver_startup.is_some() { + if let Some(task) = block_sync_task { + tasks.push(task); + } + let header_sync_actions = if block_sync_driver_enabled { Some(Arc::new(Mutex::new(Some(header_sync_actions)))) } else { let action_driver_task = tokio::spawn(drive_header_sync_actions( @@ -2077,6 +2488,9 @@ pub async fn spawn_zakura_endpoint_with_header_sync_driver( tasks.push(action_driver_task); None }; + let block_sync_actions = block_sync_actions + .filter(|_| block_sync_driver_enabled) + .map(|actions| Arc::new(Mutex::new(Some(actions)))); let header_sync_tasks = Arc::new(HeaderSyncBackgroundTasks { shutdown: header_sync_shutdown, tasks: Mutex::new(tasks), @@ -2088,7 +2502,10 @@ pub async fn spawn_zakura_endpoint_with_header_sync_driver( limits.clone(), registry, trace, - ); + ) + // Give the handler the bound endpoint so inbound accepts can resolve the + // peer's source IP and enforce the per-IP connection cap. + .with_endpoint(endpoint.clone()); let router = Router::builder(endpoint) .accept(P2P_V2_ALPN, handler.clone()) .spawn(); @@ -2097,27 +2514,39 @@ pub async fn spawn_zakura_endpoint_with_header_sync_driver( supervisor, handler, header_sync: Some(header_sync), + block_sync, + sync_frontier, header_sync_tasks: Some(header_sync_tasks), header_sync_actions, - upgrade_dials: Arc::new(StdMutex::new(HashSet::new())), + block_sync_actions, + upgrade_dials: Arc::new(StdMutex::new(HashMap::new())), }; // Log our own dial address once iroh has resolved it, so operators can hand // out `@` for other nodes' `zakura.bootstrap_peers`. + // Tracked under the endpoint shutdown owner and shutdown-aware so it cannot + // outlive endpoint teardown while awaiting address resolution. { - let endpoint = endpoint.clone(); - tokio::spawn(async move { - let node_addr = endpoint.node_addr().await; - let direct_addresses: Vec = node_addr - .direct_addresses() - .map(|addr| addr.to_string()) - .collect(); - info!( - node_id = %node_addr.node_id, - ?direct_addresses, - "Zakura P2P endpoint ready; advertise @ as a bootstrap peer", - ); + let shutdown = endpoint.background_shutdown_token(); + let log_endpoint = endpoint.clone(); + let task = tokio::spawn(async move { + tokio::select! { + biased; + _ = shutdown.cancelled() => {} + node_addr = log_endpoint.node_addr() => { + let direct_addresses: Vec = node_addr + .direct_addresses() + .map(|addr| addr.to_string()) + .collect(); + info!( + node_id = %node_addr.node_id, + ?direct_addresses, + "Zakura P2P endpoint ready; advertise @ as a bootstrap peer", + ); + } + } }); + endpoint.push_header_sync_task(task).await; } super::discovery::insert_static_bootstrap_candidates( @@ -2125,12 +2554,21 @@ pub async fn spawn_zakura_endpoint_with_header_sync_driver( &config.zakura.bootstrap_peers, ) .await; - spawn_native_bootstrap_dialer( + // Track the maintained bootstrap redial tasks and the discovery candidate + // dialer under the endpoint shutdown owner so `ZakuraEndpoint::shutdown` + // drains them. Previously their handles were dropped, leaving detached loops + // that kept retrying dials against a torn-down router while holding endpoint + // clones past teardown. + for task in spawn_native_bootstrap_dialer( endpoint.clone(), config.zakura.bootstrap_peers.clone(), limits.clone(), - ); - super::discovery::spawn_native_discovery_dialer(endpoint.clone(), discovery, limits); + ) { + endpoint.push_header_sync_task(task).await; + } + let discovery_dialer = + super::discovery::spawn_native_discovery_dialer(endpoint.clone(), discovery, limits); + endpoint.push_header_sync_task(discovery_dialer).await; Ok(Some(endpoint)) } @@ -2179,6 +2617,7 @@ pub(crate) async fn serve_native_dial_connection( .await? }; let conn_limits = limits.clamp(&negotiated.limits); + let direction = ServicePeerDirection::Outbound; endpoint .handler .register_and_serve( @@ -2189,7 +2628,12 @@ pub(crate) async fn serve_native_dial_connection( limits: conn_limits, accepted_capabilities: negotiated.accepted_capabilities, role: "initiator", - direction: ServicePeerDirection::Outbound, + direction, + transcript_hash: native_connection_transcript_hash( + direction, + &local_node_id, + &remote_node_id, + ), conn, }, ) @@ -2314,13 +2758,97 @@ fn spawn_persistent_stream_worker( async fn persistent_stream_worker( mut send: SendStream, - mut recv: RecvStream, + recv: RecvStream, prelude: StreamPrelude, context: StreamWorkerContext, inbound_tx: mpsc::Sender, outbound_rx: mpsc::Receiver, queue_depth_limit: usize, ) { + let context = Arc::new(context); + let stream_kind = prelude.stream_kind; + + // The inbound reader runs in its own task, not as a `select!` branch racing + // the outbound writer below. `read_frame` is NOT cancellation-safe: it + // consumes the fixed frame header, then awaits the (multi-packet) payload. If + // it shared this `select!` with the outbound arm, an outbound frame becoming + // ready mid-read would drop the `read_frame` future and discard the header + // bytes it already consumed, desyncing the stream forever -- the next read + // decodes body bytes as a header, yielding a garbage multi-GiB `payload_len`, + // an `OversizeFrame` error, and a stream reset. Heavy concurrent body-sync + // (inbound bodies + outbound `GetBlocks`) made this fire constantly. Reading + // in a dedicated task removes the write/read race; the main loop only ever + // *receives* fully-read frames over a channel, which is cancellation-safe. + let (frame_tx, mut frame_rx) = mpsc::channel::>(1); + let reader_context = Arc::clone(&context); + let reader = tokio::spawn(async move { + let mut recv = recv; + loop { + let frame = tokio::select! { + biased; + _ = reader_context.connection_token.cancelled() => break, + _ = reader_context.stream_token.cancelled() => break, + frame = read_frame( + &mut recv, + inbound_frame_cap_for_stream_kind(&reader_context.limits, stream_kind), + reader_context.limits.idle_timeout, + ) => frame, + }; + // Admit (rate/oversize) at ingress, the instant a frame is read, so + // throttling never trails behind the main loop draining queued + // outbound writes or forwarding an earlier frame to a service that + // might disconnect first. The main loop only ever receives frames + // that already cleared admission, plus terminal errors it maps to a + // reset code (it owns the send half). Admission is charged exactly + // once, here. + let message = match frame { + Ok(frame) => { + let _ = reader_context.freshness_tx.send(Instant::now()); + match admit_inbound_message(frame.payload.len(), &reader_context, stream_kind) { + InboundMessageAdmission::Admit => Ok(frame), + InboundMessageAdmission::Oversize => Err(ZakuraHandlerError::Oversize), + InboundMessageAdmission::Throttled => Err(ZakuraHandlerError::RateLimited), + } + } + Err(error) => { + // Emit the oversize-desync diagnostic here, at the read, so + // it is recorded even if the main loop tears the worker down + // for an outbound write that stopped first. + if let Some((payload_len, frame_len, max_frame_bytes)) = + error.oversize_frame_details() + { + reader_context.trace.emit( + RATELIMIT_TABLE, + reader_context + .event("frame.oversize") + .stream_kind(stream_kind_label(stream_kind)) + .payload_len(payload_len) + .frame_len(frame_len) + .max_frame_bytes(max_frame_bytes), + ); + } + Err(error) + } + }; + // Any error is terminal. `Closed` is a clean peer-initiated close; + // every other error is a protocol/limit violation that must + // disconnect the peer. Forward the error first so the main loop can + // map it to a reset code, then cancel the connection ourselves so + // the disconnect is guaranteed even if the main loop tore the worker + // down for a stopped outbound write before processing it. + let is_terminal = message.is_err(); + let must_disconnect = + matches!(&message, Err(error) if !matches!(error, ZakuraHandlerError::Closed)); + let forward_failed = frame_tx.send(message).await.is_err(); + if must_disconnect { + reader_context.connection_token.cancel(); + } + if forward_failed || is_terminal { + break; + } + } + }); + let mut outbound_rx = Some(outbound_rx); loop { tokio::select! { @@ -2335,7 +2863,7 @@ async fn persistent_stream_worker( } => { match outbound { Some(frame) => { - if let Err(error) = write_ordered_frame(&mut send, frame, context.limits, prelude.stream_kind).await { + if let Err(error) = write_ordered_frame(&mut send, frame, context.limits, stream_kind).await { if ordered_stream_write_was_stopped(&error) { debug!(?error, "closing Zakura ordered stream after peer stopped receiving"); break; @@ -2351,48 +2879,39 @@ async fn persistent_stream_worker( } } } - frame = read_frame( - &mut recv, - app_frame_cap_for_stream_kind(&context.limits, prelude.stream_kind), - context.limits.idle_timeout, - ) => { - match frame { - Ok(frame) => { - let _ = context.freshness_tx.send(Instant::now()); - match admit_inbound_message(frame.payload.len(), &context, prelude.stream_kind) { - InboundMessageAdmission::Admit => {} - InboundMessageAdmission::Oversize => { - let _ = send.reset(VarInt::from_u32(ZAKURA_CLOSE_OVERSIZE)); - context.connection_token.cancel(); - break; - } - InboundMessageAdmission::Throttled => continue, - } + inbound = frame_rx.recv() => { + match inbound { + // Frames here already cleared ingress admission in the reader. + Some(Ok(frame)) => { if inbound_tx.send(frame).await.is_err() { debug!( - stream_kind = prelude.stream_kind, + stream_kind, "closing Zakura ordered stream after local service receiver dropped" ); break; } metrics::gauge!( "zakura.p2p.queue.depth", - "stream_kind" => stream_kind_label(prelude.stream_kind), + "stream_kind" => stream_kind_label(stream_kind), ) .set(queue_depth_limit.saturating_sub(inbound_tx.capacity()) as f64); } - Err(ZakuraHandlerError::Closed) => { + // The reader signalled an oversize message: disconnect it. + Some(Err(ZakuraHandlerError::Oversize)) => { + let _ = send.reset(VarInt::from_u32(ZAKURA_CLOSE_OVERSIZE)); + context.connection_token.cancel(); break; } - Err(error) => { - if matches!(error, ZakuraHandlerError::Oversize) { - context.trace.emit( - RATELIMIT_TABLE, - context - .event("frame.oversize") - .stream_kind(stream_kind_label(prelude.stream_kind)), - ); - } + Some(Err(ZakuraHandlerError::RateLimited)) => { + let _ = send.reset(VarInt::from_u32(ZAKURA_CLOSE_RATE_LIMIT)); + context.connection_token.cancel(); + break; + } + Some(Err(ZakuraHandlerError::Closed)) | None => { + break; + } + // The reader already emitted any oversize-desync diagnostic. + Some(Err(error)) => { debug!(?error, "closing Zakura stream worker"); let _ = send.reset(VarInt::from_u32(ZAKURA_CLOSE_BAD_PRELUDE)); context.connection_token.cancel(); @@ -2402,6 +2921,11 @@ async fn persistent_stream_worker( } } } + + // Stop the reader: it also observes the cancellation tokens, but abort + // guarantees a prompt exit on the paths that break without cancelling one + // (e.g. a peer that stopped receiving, or the local service receiver closing). + reader.abort(); } fn ordered_stream_write_was_stopped(error: &BoxError) -> bool { @@ -2429,7 +2953,7 @@ async fn request_stream_worker( _ = context.connection_token.cancelled() => return, frame = read_frame( &mut recv, - app_frame_cap_for_stream_kind(&context.limits, prelude.stream_kind), + inbound_frame_cap_for_stream_kind(&context.limits, prelude.stream_kind), context.limits.idle_timeout, ) => frame, }; @@ -2456,8 +2980,10 @@ async fn request_stream_worker( return; } InboundMessageAdmission::Throttled => { - let _ = send.reset(VarInt::from_u32(ZAKURA_CLOSE_RATE_LIMIT)); - return; + // The admission path already counted/traced the over-rate frame. + // Do not reject request streams here: request/response peers also + // will not resend a dropped frame, and backpressure is safer than + // data loss while block sync is catching up. } } @@ -2467,6 +2993,7 @@ async fn request_stream_worker( prelude.stream_kind, request_id, app_frame_cap_for_stream_kind(&context.limits, prelude.stream_kind), + context.limits.max_message_bytes, frame, ) .await @@ -2587,7 +3114,11 @@ async fn read_frame( let frame_len = FRAME_HEADER_BYTES.saturating_add(payload_len); if frame_len > max_frame_bytes { metrics::counter!("zakura.p2p.ratelimit.frame.oversize").increment(1); - return Err(ZakuraHandlerError::Oversize); + return Err(ZakuraHandlerError::OversizeFrame { + payload_len, + frame_len, + max_frame_bytes, + }); } let mut payload = vec![0; payload_len]; timeout(read_timeout, recv.read_exact(&mut payload)) @@ -2605,6 +3136,15 @@ async fn read_control_payload( max_bytes: u32, read_timeout: Duration, ) -> Result, ZakuraHandlerError> { + // Control payloads (Hello/Ack) carry a hard 16 KiB cap that is otherwise only + // enforced later in ZakuraControlHello/Ack::decode. Clamp the caller-supplied + // frame cap (the configured `max_control_frame_bytes`, 1 MiB by default) to it + // here so a peer cannot make us allocate/read a payload up to the much larger + // frame cap before decode rejects it as oversized. + // + // `MAX_CONTROL_PAYLOAD_BYTES` (16 KiB) is a small compile-time constant, so the + // cast to u32 cannot truncate; `.min` also preserves any tighter negotiated cap. + let max_bytes = max_bytes.min(MAX_CONTROL_PAYLOAD_BYTES as u32); let mut len_bytes = [0; CONTROL_LENGTH_BYTES]; timeout(read_timeout, recv.read_exact(&mut len_bytes)) .await @@ -2643,6 +3183,20 @@ async fn write_ordered_frame( limits: ZakuraConnectionLimits, stream_kind: u16, ) -> Result<(), BoxError> { + // Mirror `write_response_frame`: a persistent ordered-stream frame whose + // payload exceeds the peer's negotiated `max_message_bytes` would be + // rejected by the peer as oversize, so reject it locally before wasting + // encode/write work rather than writing a frame the peer cannot accept. The + // handshake clamps `max_frame_bytes` and `max_message_bytes` independently, + // so the frame cap alone does not bound this. + if frame.payload.len() > limits.max_message_bytes as usize { + return Err(format!( + "Zakura outbound ordered frame payload {} exceeds negotiated max_message_bytes {}", + frame.payload.len(), + limits.max_message_bytes, + ) + .into()); + } let frame = frame.encode(app_frame_cap_for_stream_kind(&limits, stream_kind))?; timeout(OUTBOUND_STREAM_WRITE_TIMEOUT, send.write_all(&frame)) .await @@ -2743,10 +3297,9 @@ async fn write_outbound_request_frame_inner( ZakuraHandlerError::Timeout("outbound response"), ))); } - Err(ZakuraHandlerError::Oversize) => { - return Err(OutboundRequestError::Fatal(Box::new( - ZakuraHandlerError::Oversize, - ))); + Err(error @ ZakuraHandlerError::OversizeFrame { .. }) + | Err(error @ ZakuraHandlerError::Oversize) => { + return Err(OutboundRequestError::Fatal(Box::new(error))); } Err(error) => return Err(OutboundRequestError::Fatal(Box::new(error))), } @@ -2847,6 +3400,16 @@ impl LegacyResponseBudget { } }; + // Clamp the retained-frame byte budget to a fixed operational aggregate + // cap. For Blocks/Transactions the derived `max_bytes` scales with the + // requested inventory count (`item_count * max_message_bytes`), so a + // request naming the protocol-max inventory count would otherwise let a + // hostile responder accumulate tens of GiB of validated frames in the + // accepted-frame `Vec` before `decode_response` runs. The inbound + // responder already bounds a single response to the same aggregate, so + // this only rejects responses an honest peer would never send. + let max_bytes = max_bytes.min(LEGACY_RESPONSE_MAX_AGGREGATE_BYTES); + Ok(Self { kind, max_items, @@ -3073,18 +3636,35 @@ impl LegacyResponseReadState { self.add_items(1) } - /// Accept a `MSG_RESPONSE_NIL` empty-result sentinel for any request kind. + /// Validate a `MSG_RESPONSE_NIL` empty-result sentinel against the request kind. /// - /// The inbound service answers an empty `FindBlocks`/`FindHeaders`/ + /// NIL is the empty-result sentinel only for chain-discovery and mempool + /// queries: the inbound service answers an empty `FindBlocks`/`FindHeaders`/ /// `MempoolTransactionIds` (and a queued `PushTransaction`) with - /// `Response::Nil`, so a lone nil frame is a valid empty response for every - /// request kind, not just `PushTransaction`. The kind-specific empty - /// `Response` is produced later by `LegacyResponseCodec::decode_response`. + /// `Response::Nil`. Inventory fetches (`BlocksByHash`/`TransactionsById`) and + /// `Ping` must never receive a bare NIL, so reject it as `Fatal` for those + /// kinds — fail closed and let the request stream worker disconnect the peer, + /// matching `LegacyResponseCodec::decode_response`, which rejects NIL for the + /// same kinds. The kind-specific empty `Response` is produced later by + /// `decode_response`. fn validate_nil( &mut self, request_id: u64, payload: &[u8], ) -> Result<(), OutboundRequestError> { + match self.budget.kind { + LegacyResponseKind::BlockHashes + | LegacyResponseKind::BlockHeaders + | LegacyResponseKind::TransactionIds + | LegacyResponseKind::Nil => {} + LegacyResponseKind::Blocks + | LegacyResponseKind::Transactions + | LegacyResponseKind::Pong => { + return Err(OutboundRequestError::Fatal( + "unexpected legacy nil response for inventory or ping request".into(), + )); + } + } if self.active_chunk_type.is_some() { return Err(OutboundRequestError::Fatal( "legacy nil response interleaved with response chunk".into(), @@ -3277,12 +3857,11 @@ fn validate_idle_invariant(limits: &ZakuraLocalLimits) -> Result<(), ZakuraHandl } fn zakura_secret_key(config: &Config) -> Result { - if let Some(secret) = &config.zakura_node_secret_key { - return SecretKey::from_str(secret.expose_secret()) - .map_err(|_| ZakuraHandlerError::InvalidSecretKey); - } - - Ok(SecretKey::generate(OsRng)) + // Loads the configured key, or loads/generates+persists a stable key under the + // cache dir so the node keeps a consistent NodeId across restarts. + config + .zakura_secret_key() + .map_err(|_| ZakuraHandlerError::InvalidSecretKey) } fn stream_kind_label(stream_kind: u16) -> &'static str { @@ -3293,6 +3872,7 @@ fn stream_kind_label(stream_kind: u16) -> &'static str { LEGACY_REQUEST_STREAM_KIND => "legacy_request", DISCOVERY_STREAM_KIND => "discovery", HEADER_SYNC_STREAM_KIND => "header_sync", + ZAKURA_STREAM_BLOCK_SYNC => "block_sync", _ => "unknown", } } @@ -3305,11 +3885,31 @@ fn app_frame_cap_for_stream_kind(limits: &ZakuraConnectionLimits, stream_kind: u .expect("header-sync frame cap fits in u32"); limits.max_frame_bytes.min(header_sync_cap) } + ZAKURA_STREAM_BLOCK_SYNC => limits.max_frame_bytes.min(MAX_BS_FRAME_BYTES), _ => limits.max_frame_bytes.min(LOCAL_MAX_CONTROL_FRAME_BYTES), } .max(1) } +/// Frame cap for reading on an admitted inbound stream, never larger than the +/// message cap allows. +/// +/// On an admitted ordered/request stream a frame payload *is* the message, so +/// `admit_inbound_message` rejects any payload over `max_message_bytes`. A peer +/// can negotiate `max_frame_bytes > max_message_bytes` (the two caps are clamped +/// independently in `ZakuraLocalLimits::clamp`), so the cap handed to +/// `read_frame` must also be limited to the message size. Otherwise a frame whose +/// `payload_len` falls between the two limits is allocated and read in full by +/// `read_frame` before `admit_inbound_message` rejects it as oversize, letting a +/// peer force per-frame allocation/I/O up to the larger frame cap across many +/// streams. +fn inbound_frame_cap_for_stream_kind(limits: &ZakuraConnectionLimits, stream_kind: u16) -> u32 { + let frame_header_bytes = + u32::try_from(FRAME_HEADER_BYTES).expect("frame header byte count fits in u32"); + app_frame_cap_for_stream_kind(limits, stream_kind) + .min(limits.max_message_bytes.saturating_add(frame_header_bytes)) +} + fn per_stream_inbound_queue_depth( max_inbound_queue_depth: u16, ordered_stream_count: usize, @@ -3332,6 +3932,7 @@ fn should_run_freshness_reaper( /// The only stream-kind version this v1 handler serves. Every known kind is /// at version 1; a peer naming any other version of a known kind is rejected. const ZAKURA_STREAM_VERSION_1: u16 = 1; +const ZAKURA_STREAM_VERSION_2: u16 = 2; /// Returns whether the handler can serve a stream with this kind and version. /// @@ -3442,6 +4043,19 @@ pub enum ZakuraHandlerError { /// A peer-controlled payload exceeded its cap. #[error("Zakura payload exceeded its cap")] Oversize, + /// A peer declared a frame larger than the effective receiver cap. + #[error( + "Zakura frame length {frame_len} exceeded cap {max_frame_bytes} \ + (payload length {payload_len})" + )] + OversizeFrame { + /// Declared frame payload length. + payload_len: usize, + /// Full frame length including the fixed header. + frame_len: usize, + /// Effective frame cap used before reading the payload. + max_frame_bytes: usize, + }, /// The peer closed the stream or connection. #[error("Zakura stream closed")] Closed, @@ -3457,6 +4071,9 @@ pub enum ZakuraHandlerError { /// A local resource cap rejected the operation. #[error("Zakura resource limit exceeded: {0}")] ResourceLimit(&'static str), + /// The peer exceeded its per-kind inbound message rate. + #[error("Zakura message rate exceeded")] + RateLimited, /// Iroh connection error. #[error(transparent)] IrohConnection(#[from] iroh::endpoint::ConnectionError), @@ -3486,6 +4103,25 @@ pub enum ZakuraHandlerError { Io(#[from] std::io::Error), } +impl ZakuraHandlerError { + fn oversize_frame_details(&self) -> Option<(u64, u64, u64)> { + let Self::OversizeFrame { + payload_len, + frame_len, + max_frame_bytes, + } = self + else { + return None; + }; + + Some(( + u64::try_from(*payload_len).unwrap_or(u64::MAX), + u64::try_from(*frame_len).unwrap_or(u64::MAX), + u64::try_from(*max_frame_bytes).unwrap_or(u64::MAX), + )) + } +} + #[cfg(test)] mod tests { use super::*; @@ -3511,6 +4147,33 @@ mod tests { }; use zebra_test::vectors::BLOCK_TESTNET_141042_BYTES; + /// With no configured `zakura.listen_addr`, the native endpoint must bind + /// loopback-only. Otherwise iroh's default bind (`0.0.0.0:0` / `[::]:0`) + /// exposes the experimental P2P_V2_ALPN handshake/session surface on every + /// interface on an OS-assigned ephemeral port, even though the unset state is + /// documented as dial-out only. + #[tokio::test] + async fn unset_listen_addr_binds_loopback_not_unspecified() { + let builder = direct_endpoint_builder(SecretKey::generate(OsRng)); + let builder = bind_native_endpoint(builder, None); + let endpoint = builder.bind().await.expect("loopback bind should succeed"); + + let sockets = endpoint.bound_sockets(); + assert!( + !sockets.is_empty(), + "endpoint should bind at least one socket" + ); + for socket in &sockets { + assert!( + socket.ip().is_loopback(), + "unset listen_addr must bind loopback only, but bound {socket} \ + (exposes P2P_V2_ALPN on all interfaces)" + ); + } + + endpoint.close().await; + } + #[derive(Debug, Clone)] struct CaptureConnection { connection_tx: mpsc::Sender, @@ -3611,6 +4274,57 @@ mod tests { ZakuraPeerId::new(vec![byte; 32]).expect("32-byte node id is valid") } + #[test] + fn native_duplicate_tie_breaker_converges_for_simultaneous_open() { + let node_a = LocalEndpointFactory::secret_key(1).public(); + let node_b = LocalEndpointFactory::secret_key(2).public(); + let a_outbound = + native_connection_transcript_hash(ServicePeerDirection::Outbound, &node_a, &node_b); + let a_inbound = + native_connection_transcript_hash(ServicePeerDirection::Inbound, &node_a, &node_b); + let b_outbound = + native_connection_transcript_hash(ServicePeerDirection::Outbound, &node_b, &node_a); + let b_inbound = + native_connection_transcript_hash(ServicePeerDirection::Inbound, &node_b, &node_a); + + assert_eq!(a_outbound, b_inbound); + assert_eq!(a_inbound, b_outbound); + assert_ne!(a_outbound, a_inbound); + + let winning_key = a_outbound.min(a_inbound); + let losing_key = a_outbound.max(a_inbound); + let peer = test_peer(7); + let mut supervisor_a = ZakuraPeerSupervisor::default(); + let mut supervisor_b = ZakuraPeerSupervisor::default(); + assert!(matches!( + supervisor_a.register_authenticated(peer.clone(), a_outbound), + ZakuraUpgradeOutcome::Upgraded { .. } + )); + let _ = supervisor_a.register_authenticated(peer.clone(), a_inbound); + assert!(matches!( + supervisor_b.register_authenticated(peer.clone(), b_inbound), + ZakuraUpgradeOutcome::Upgraded { .. } + )); + let _ = supervisor_b.register_authenticated(peer.clone(), b_outbound); + + assert!(matches!( + supervisor_a.register_authenticated(peer.clone(), winning_key), + ZakuraUpgradeOutcome::Duplicate { .. } + )); + assert!(matches!( + supervisor_b.register_authenticated(peer.clone(), winning_key), + ZakuraUpgradeOutcome::Duplicate { .. } + )); + assert!(matches!( + supervisor_a.register_authenticated(peer.clone(), losing_key), + ZakuraUpgradeOutcome::Duplicate { .. } + )); + assert!(matches!( + supervisor_b.register_authenticated(peer, losing_key), + ZakuraUpgradeOutcome::Duplicate { .. } + )); + } + fn header_sync_test_session( peer: ZakuraPeerId, ) -> (HeaderSyncPeerSession, crate::zakura::FramedRecv) { @@ -3664,6 +4378,7 @@ mod tests { HeaderSyncFrontiers { finalized_height: anchor.0, verified_block_tip: anchor.0, + verified_block_hash: anchor.1, }, Some(anchor), ZakuraHeaderSyncConfig::default(), @@ -3816,43 +4531,184 @@ mod tests { Ok(()) } + /// A maintained native dial loop (shared by configured bootstrap peers, the + /// legacy->Zakura upgrade hand-off, and `spawn_native_dial`) targets an + /// unreachable peer, so its `maintain` policy retries forever and never + /// exits on its own. `ZakuraEndpoint::shutdown` must still stop it: the task + /// holds an endpoint clone, so the supervisor registration watch stays open + /// and only the endpoint shutdown token can end the loop. Without that + /// signal the detached loop outlives shutdown and keeps dialing. #[tokio::test] - async fn registry_routes_legacy_and_header_sync_and_drops_unknown() -> Result<(), BoxError> { + async fn endpoint_shutdown_stops_maintained_native_dial_loop() -> Result<(), BoxError> { let _guard = zebra_test::init(); - let shutdown = CancellationToken::new(); - let startup = header_sync_startup(shutdown.clone()); - let (header_sync, mut actions, task) = spawn_header_sync_reactor(startup)?; - let recorder = Arc::new(RecordingService::default()); - let supervisor = ZakuraSupervisorHandle::new(1); - let registry = service_registry( - &supervisor, - Some(header_sync.clone()), - recorder.clone(), - test_discovery_service(&supervisor), - )?; - let peer = test_peer(6); + let endpoint = spawn_zakura_endpoint(&Config::default(), |_supervisor, _trace| { + Arc::new(NoopService) as Arc + }) + .await? + .expect("v2_p2p is enabled by default"); - let (session, _recv) = header_sync_test_session(peer.clone()); - header_sync - .send(HeaderSyncEvent::PeerConnected(session)) - .await?; - assert!(matches!( - next_header_sync_action(&mut actions).await, - HeaderSyncAction::SendMessage { - msg: HeaderSyncMessage::Status(_), - .. - } - )); + // 192.0.2.0/24 is TEST-NET-1 (RFC 5737): guaranteed unreachable, so the + // maintained loop stays in connect/backoff and never finishes on its own. + let unreachable_addr: SocketAddr = "192.0.2.1:65535".parse().expect("valid test address"); + let unreachable = NodeAddr::new(LocalEndpointFactory::secret_key(987_654).public()) + .with_direct_addresses([unreachable_addr]); + let dial = endpoint.spawn_native_dial(unreachable); - let gossip_frame = Frame { - message_type: 11, - flags: 0, - payload: Vec::new(), - }; - let request_frame = Frame { - message_type: 12, - flags: 0, - payload: Vec::new(), + // Let the maintained loop start before tearing the endpoint down. + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !dial.is_finished(), + "maintained dial loop to an unreachable peer should still be running" + ); + + endpoint.shutdown().await; + + tokio::time::timeout(Duration::from_secs(5), dial) + .await + .expect("maintained native dial loop must exit after endpoint shutdown") + .expect("native dial task must not panic"); + Ok(()) + } + + /// The discovery candidate dialer is a long-lived loop that holds an + /// endpoint clone, so its supervisor registration watch never closes on its + /// own. `ZakuraEndpoint::shutdown` must still stop it via the endpoint + /// shutdown token; otherwise it outlives teardown, polling the discovery + /// book and attempting dials against a torn-down router. + #[tokio::test] + async fn endpoint_shutdown_stops_discovery_candidate_dialer() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let config = Config::default(); + let endpoint = spawn_zakura_endpoint(&config, |_supervisor, _trace| { + Arc::new(NoopService) as Arc + }) + .await? + .expect("v2_p2p is enabled by default"); + + let limits = ZakuraLocalLimits::from_config(&config); + let handshake = ZakuraHandshakeConfig::for_network(&config.network); + let discovery = crate::zakura::discovery::build_discovery_handle( + SecretKey::generate(OsRng), + Vec::new(), + crate::zakura::discovery::default_advertised_services(), + &handshake, + limits.max_connections, + 0, + endpoint.supervisor().subscribe(), + )?; + let dialer = crate::zakura::discovery::spawn_native_discovery_dialer( + endpoint.clone(), + discovery, + limits, + ); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !dialer.is_finished(), + "discovery candidate dialer should still be running" + ); + + endpoint.shutdown().await; + + tokio::time::timeout(Duration::from_secs(5), dialer) + .await + .expect("discovery candidate dialer must exit after endpoint shutdown") + .expect("discovery dialer task must not panic"); + Ok(()) + } + + /// A malicious legacy responder can return a syntactically valid + /// `P2pV2UpgradeAccept` (a unique, real iroh node id with a parseable but + /// unreachable direct address) that never completes native Zakura + /// registration. After the upgrade hand-off wait times out, no + /// maintain-forever native dial task or `upgrade_dials` entry may survive; + /// otherwise repeating the failed upgrade with distinct node ids grows + /// maintained dials and outbound QUIC traffic without bound. + /// + /// Regression test for `claude-legacy-upgrade-maintained-dial-leak`. Uses a + /// paused clock so the 15s appear-timeout elapses without real waiting; the + /// dial target is RFC 5737 TEST-NET-1, which never connects. + #[tokio::test(start_paused = true)] + async fn failed_legacy_upgrade_does_not_leak_maintained_dial() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let endpoint = spawn_zakura_endpoint(&Config::default(), |_supervisor, _trace| { + Arc::new(NoopService) as Arc + }) + .await? + .expect("v2_p2p is enabled by default"); + + // The Accept the attacker would advertise: a real 32-byte iroh node id + // (so `node_addr_from_hints` builds a `NodeAddr` and the dial spawns) + // pointing at an unreachable address that never registers. + let node_id = LocalEndpointFactory::secret_key(0x0BAD_C0DE) + .public() + .as_bytes() + .to_vec(); + let peer_id = ZakuraPeerId::new(node_id.clone()).expect("32-byte node id is valid"); + let direct_addresses = vec![b"192.0.2.1:1".to_vec()]; + + let connector = + crate::zakura::ZakuraHandshakeConnector::new_with_endpoint(endpoint.clone()); + let upgraded = connector + .spawn_zakura_dial_to_hints_and_wait(&peer_id, &node_id, &direct_addresses) + .await; + + assert!( + !upgraded, + "an unreachable upgrade peer must not report a completed hand-off", + ); + assert!( + endpoint + .upgrade_dials + .lock() + .expect("Zakura upgrade dial registry mutex is never poisoned") + .is_empty(), + "a failed legacy upgrade leaked a maintained native dial / upgrade_dials entry", + ); + + endpoint.shutdown().await; + Ok(()) + } + + #[tokio::test] + async fn registry_routes_legacy_and_header_sync_and_drops_unknown() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let shutdown = CancellationToken::new(); + let startup = header_sync_startup(shutdown.clone()); + let (header_sync, mut actions, task) = spawn_header_sync_reactor(startup)?; + let recorder = Arc::new(RecordingService::default()); + let supervisor = ZakuraSupervisorHandle::new(1); + let registry = service_registry( + &supervisor, + Some(header_sync.clone()), + None, + ZakuraBlockSyncConfig::default(), + recorder.clone(), + test_discovery_service(&supervisor), + )?; + let peer = test_peer(6); + + let (session, _recv) = header_sync_test_session(peer.clone()); + header_sync + .send(HeaderSyncEvent::PeerConnected(session)) + .await?; + assert!(matches!( + next_header_sync_action(&mut actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); + + let gossip_frame = Frame { + message_type: 11, + flags: 0, + payload: Vec::new(), + }; + let request_frame = Frame { + message_type: 12, + flags: 0, + payload: Vec::new(), }; let get_headers_frame = HeaderSyncMessage::GetHeaders { start_height: block::Height(1), @@ -3909,6 +4765,7 @@ mod tests { HEADER_SYNC_STREAM_KIND, 99, LOCAL_MAX_CONTROL_FRAME_BYTES, + LOCAL_MAX_CONTROL_FRAME_BYTES, Frame { message_type: 1, flags: 0, @@ -4045,6 +4902,8 @@ mod tests { let registry = service_registry( &supervisor, Some(header_sync.clone()), + None, + ZakuraBlockSyncConfig::default(), Arc::new(RecordingService::default()), test_discovery_service(&supervisor), )?; @@ -4123,6 +4982,8 @@ mod tests { let registry = service_registry( &supervisor, Some(header_sync.clone()), + None, + ZakuraBlockSyncConfig::default(), Arc::new(RecordingService::default()), discovery_service, )?; @@ -4224,6 +5085,8 @@ mod tests { let registry = service_registry( &supervisor, Some(header_sync.clone()), + None, + ZakuraBlockSyncConfig::default(), Arc::new(RecordingService::default()), discovery_service, )?; @@ -4297,6 +5160,143 @@ mod tests { Ok(()) } + #[tokio::test(start_paused = true)] + async fn duplicate_evicts_stale_incumbent_but_keeps_fresh_one() -> Result<(), BoxError> { + // A duplicate connection for an identity that already has one either + // means the peer restarted (incumbent is a dead, stale connection that + // should be evicted so the redial can reclaim the slot) or that two + // connections raced at startup (simultaneous open, both fresh, must NOT + // be evicted or they flap). Incumbent age distinguishes the two. + let supervisor = ZakuraSupervisorHandle::new(4); + + async fn register_duplicate( + supervisor: &ZakuraSupervisorHandle, + peer: &ZakuraPeerId, + token: CancellationToken, + ) -> ZakuraRegistration { + let (outbound_tx, _outbound_rx) = mpsc::channel(1); + let outbound_handle = ZakuraPeerHandle::new_for_tests(peer.clone(), outbound_tx); + supervisor + .register( + peer.clone(), + None, + [peer.as_bytes()[0]; TRANSCRIPT_HASH_BYTES], + outbound_handle, + token, + ZAKURA_CAP_LEGACY_GOSSIP | ZAKURA_CAP_HEADER_SYNC, + ) + .await + } + + // Fresh incumbent: an immediate duplicate is a simultaneous-open race and + // must not evict it. + let fresh_peer = test_peer(8); + let fresh_incumbent = CancellationToken::new(); + register_test_peer(&supervisor, fresh_peer.clone(), fresh_incumbent.clone()).await; + let registration = + register_duplicate(&supervisor, &fresh_peer, CancellationToken::new()).await; + assert!(matches!(registration, ZakuraRegistration::Duplicate { .. })); + assert!( + !fresh_incumbent.is_cancelled(), + "a young incumbent is kept so simultaneous-open races do not flap", + ); + + // Stale incumbent: once it is older than the threshold, a duplicate + // evicts it so a restarted peer's redial can reclaim the slot. + let stale_peer = test_peer(9); + let stale_incumbent = CancellationToken::new(); + register_test_peer(&supervisor, stale_peer.clone(), stale_incumbent.clone()).await; + tokio::time::advance(ZAKURA_DUPLICATE_EVICT_MIN_AGE + Duration::from_secs(1)).await; + let newcomer = CancellationToken::new(); + let registration = register_duplicate(&supervisor, &stale_peer, newcomer.clone()).await; + assert!(matches!(registration, ZakuraRegistration::Duplicate { .. })); + assert!( + stale_incumbent.is_cancelled(), + "a stale incumbent is evicted so the restarted peer's redial reclaims the slot", + ); + assert!( + !newcomer.is_cancelled(), + "the rejected newcomer's token is never registered, so it is left to redial", + ); + + Ok(()) + } + + // SECURITY AUDIT (candidate claude-per-ip-cap-blocks-stale-duplicate-eviction / + // codex-ip-cap-blocks-stale-duplicate-eviction / + // subset-admission-identity-state-ip-cap-blocks-stale-duplicate-eviction): + // SR-4 liveness. + // + // Outbound/native-direct dials register with `remote_ip = Some(ip)`, so the + // per-IP cap precheck runs. A same-peer redial from an IP already at the cap + // must NOT be rejected as a resource limit before the duplicate stale-eviction + // path runs: the incumbent already occupies the only per-IP slot, so a duplicate + // for the same identity cannot consume an additional slot. If the precheck + // rejects it first, a dead incumbent keeps its own peer blocked until the QUIC + // idle timeout (~150s) instead of being evicted in milliseconds. + #[tokio::test(start_paused = true)] + async fn same_peer_duplicate_at_per_ip_cap_still_evicts_stale_incumbent() -> Result<(), BoxError> + { + async fn register_from_ip( + supervisor: &ZakuraSupervisorHandle, + peer: &ZakuraPeerId, + ip: IpAddr, + token: CancellationToken, + ) -> ZakuraRegistration { + let (outbound_tx, _outbound_rx) = mpsc::channel(1); + let outbound_handle = ZakuraPeerHandle::new_for_tests(peer.clone(), outbound_tx); + supervisor + .register( + peer.clone(), + Some(ip), + [peer.as_bytes()[0]; TRANSCRIPT_HASH_BYTES], + outbound_handle, + token, + ZAKURA_CAP_LEGACY_GOSSIP | ZAKURA_CAP_HEADER_SYNC, + ) + .await + } + + let supervisor = ZakuraSupervisorHandle::new(1); + let ip: IpAddr = "203.0.113.9".parse().expect("test ip parses"); + let peer = test_peer(42); + + // Incumbent: a native-direct dial registers from the IP and fills the + // cap-1 per-IP slot. + let incumbent = CancellationToken::new(); + let registration = register_from_ip(&supervisor, &peer, ip, incumbent.clone()).await; + assert!( + matches!(registration, ZakuraRegistration::Registered { .. }), + "the first connection from the IP registers", + ); + + // The incumbent ages past the stale-eviction threshold: it is now a likely + // dead connection left behind by a peer restart. + tokio::time::advance(ZAKURA_DUPLICATE_EVICT_MIN_AGE + Duration::from_secs(1)).await; + + // The same peer redials from the same IP. The IP is already at cap 1, but + // the duplicate must still reach the stale-eviction path and cancel the dead + // incumbent so the redial reclaims the slot in milliseconds. + let newcomer = CancellationToken::new(); + let registration = register_from_ip(&supervisor, &peer, ip, newcomer.clone()).await; + assert!( + matches!(registration, ZakuraRegistration::Duplicate { .. }), + "a same-peer redial from a capped IP must reach duplicate handling, not be \ + rejected as a resource limit before stale eviction can run", + ); + assert!( + incumbent.is_cancelled(), + "the stale incumbent must be evicted so the restarted peer's redial reclaims \ + the slot in milliseconds instead of waiting for the QUIC idle timeout", + ); + assert!( + !newcomer.is_cancelled(), + "the rejected newcomer's token is never registered, so it is left to redial", + ); + + Ok(()) + } + #[tokio::test] async fn header_sync_misbehavior_action_disconnects_peer() -> Result<(), BoxError> { let _guard = zebra_test::init(); @@ -4452,6 +5452,475 @@ mod tests { Ok(()) } + /// Regression for `claude-outbound-write-ignores-message-cap` (persistent + /// ordered-stream facet). + /// + /// `write_ordered_frame` sized outbound persistent-stream frames against the + /// negotiated frame cap only. The handshake clamps `max_frame_bytes` and + /// `max_message_bytes` independently, so a peer can negotiate a small message + /// cap with a large frame cap. A persistent service that queues a frame + /// larger than that message cap then has it encoded and written, and the peer + /// rejects it as oversize and disconnects us, wasting the encode/write. + /// `write_ordered_frame` must mirror `write_response_frame` and reject an + /// over-message-cap frame locally before encoding/writing it, while still + /// writing frames within the cap. + #[tokio::test] + async fn write_ordered_frame_rejects_payload_over_message_cap() -> Result<(), BoxError> { + const ALPN: &[u8] = b"/zakura/testkit/ordered-message-cap/0"; + + let _guard = zebra_test::init(); + let server = LocalEndpointFactory::new().endpoint(74).await?; + let (conn_tx, _conn_rx) = mpsc::channel(1); + let (stream_tx, _stream_rx) = mpsc::channel(2); + let router = Router::builder(server) + .accept( + ALPN, + CaptureConnection { + connection_tx: conn_tx, + stream_tx, + }, + ) + .spawn(); + let client = LocalEndpointFactory::new().endpoint(75).await?; + let server_addr = router.endpoint().node_addr().initialized().await; + client.add_node_addr(server_addr.clone())?; + + let client_conn = timeout(Duration::from_secs(10), client.connect(server_addr, ALPN)) + .await + .expect("client connects to the ordered-message-cap endpoint")?; + let (mut send, _recv) = timeout(Duration::from_secs(1), client_conn.open_bi()) + .await + .expect("client opens an ordered stream")?; + + // A small negotiated message cap with a large frame cap, as the handshake + // permits. + let mut limits = test_connection_limits(); + limits.max_message_bytes = 256; + let stream_kind = DISCOVERY_STREAM_KIND; + + // A payload above the message cap but well within the frame cap, so only + // the message cap can reject it. + let oversized = Frame { + message_type: 1, + flags: 0, + payload: vec![0xab; 4096], + }; + assert!( + oversized.payload.len() <= app_frame_cap_for_stream_kind(&limits, stream_kind) as usize, + "test payload must fit the frame cap so only the message cap can reject it" + ); + + let result = write_ordered_frame(&mut send, oversized, limits, stream_kind).await; + assert!( + result.is_err(), + "write_ordered_frame must reject a payload over the negotiated max_message_bytes \ + before encoding/writing it, mirroring write_response_frame; got {result:?}" + ); + + // A payload within the negotiated message cap must still be written. + let within_cap = Frame { + message_type: 1, + flags: 0, + payload: vec![0xcd; 128], + }; + write_ordered_frame(&mut send, within_cap, limits, stream_kind) + .await + .expect("a frame within the negotiated message cap must still be written"); + + client_conn.close(0u32.into(), b"done"); + client.close().await; + router.shutdown().await?; + Ok(()) + } + + // Regression for `claude-control-payload-late-hard-cap`: the native control + // hello/ack reads passed the configured `max_control_frame_bytes` (1 MiB + // default) to `read_control_payload`, so a peer could force allocation/read + // of a control payload between the 16 KiB hard cap and 1 MiB before + // ZakuraControlHello/Ack::decode rejected it. The reader must instead reject + // an oversized control length on the length prefix alone, before reading the + // body, while still accepting payloads within the 16 KiB hard cap. + #[tokio::test] + async fn read_control_payload_enforces_hard_cap_before_reading_body() -> Result<(), BoxError> { + const ALPN: &[u8] = b"/zakura/testkit/control-hard-cap/0"; + + let _guard = zebra_test::init(); + let server = LocalEndpointFactory::new().endpoint(70).await?; + let (conn_tx, _conn_rx) = mpsc::channel(1); + let (stream_tx, mut stream_rx) = mpsc::channel(2); + let router = Router::builder(server) + .accept( + ALPN, + CaptureConnection { + connection_tx: conn_tx, + stream_tx, + }, + ) + .spawn(); + let client = LocalEndpointFactory::new().endpoint(71).await?; + let server_addr = router.endpoint().node_addr().initialized().await; + client.add_node_addr(server_addr.clone())?; + + let client_conn = timeout(Duration::from_secs(10), client.connect(server_addr, ALPN)) + .await + .expect("client connects to the control-hard-cap endpoint")?; + + // A control length above the 16 KiB hard cap but below the 1 MiB frame + // cap the production responder/initiator pass. The body is never sent: a + // correct reader must reject on the length prefix alone. + let oversized_len = + u32::try_from(MAX_CONTROL_PAYLOAD_BYTES + 1).expect("control hard cap + 1 fits in u32"); + let (mut over_send, _over_recv) = timeout(Duration::from_secs(1), client_conn.open_bi()) + .await + .expect("client opens the oversized control stream")?; + timeout( + Duration::from_secs(1), + over_send.write_all(&oversized_len.to_le_bytes()), + ) + .await + .expect("client writes the oversized control length")?; + let _ = over_send.finish(); + + let (_over_server_send, mut over_server_recv) = + timeout(Duration::from_secs(1), stream_rx.recv()) + .await + .expect("server accepts the oversized control stream") + .expect("capture handler forwards the oversized control stream"); + + // Pass exactly what production passes: the configured frame cap (1 MiB), + // NOT the 16 KiB hard cap. The fix clamps internally to the hard cap. + let oversized = read_control_payload( + &mut over_server_recv, + LOCAL_MAX_CONTROL_FRAME_BYTES, + Duration::from_secs(2), + ) + .await; + assert!( + matches!(oversized, Err(ZakuraHandlerError::Oversize)), + "a control length over the 16 KiB hard cap must be rejected as Oversize \ + on the length prefix, before the body is read; got {oversized:?}" + ); + + // A control payload within the hard cap must still be read normally, so + // the clamp does not break legitimate sub-16 KiB control frames. + let valid_body = vec![0xa5u8; 128]; + let valid_len = u32::try_from(valid_body.len()).expect("128 fits in u32"); + let (mut ok_send, _ok_recv) = timeout(Duration::from_secs(1), client_conn.open_bi()) + .await + .expect("client opens the in-cap control stream")?; + timeout( + Duration::from_secs(1), + ok_send.write_all(&valid_len.to_le_bytes()), + ) + .await + .expect("client writes the in-cap control length")?; + timeout(Duration::from_secs(1), ok_send.write_all(&valid_body)) + .await + .expect("client writes the in-cap control body")?; + let _ = ok_send.finish(); + + let (_ok_server_send, mut ok_server_recv) = + timeout(Duration::from_secs(1), stream_rx.recv()) + .await + .expect("server accepts the in-cap control stream") + .expect("capture handler forwards the in-cap control stream"); + let read_back = read_control_payload( + &mut ok_server_recv, + LOCAL_MAX_CONTROL_FRAME_BYTES, + Duration::from_secs(2), + ) + .await + .expect("a control payload within the hard cap is read"); + assert_eq!( + read_back, valid_body, + "an in-cap control payload must round-trip unchanged" + ); + + client_conn.close(0u32.into(), b"done"); + client.close().await; + router.shutdown().await?; + Ok(()) + } + + // claude-late-message-cap-allocation: read_frame checks only + // frame_len > max_frame_bytes before `vec![0; payload_len]`, while the smaller + // max_message_bytes is enforced later in admit_inbound_message. A peer can + // negotiate max_frame_bytes > max_message_bytes (the caps are clamped + // independently), so on every admitted ordered/request stream it could force + // read_frame to allocate and read a payload between the two limits before the + // message cap rejects it. The inbound read path must instead be handed a cap + // already limited to the message size, so an over-message frame is rejected on + // its header alone, before the payload is allocated and read. + #[tokio::test] + async fn inbound_frame_cap_rejects_over_message_frame_before_reading_payload( + ) -> Result<(), BoxError> { + const ALPN: &[u8] = b"/zakura/testkit/late-message-cap/0"; + // The negotiated frame cap is far larger than the message cap: exactly the + // precondition the finding requires (caps allowed to diverge). + const MAX_FRAME_BYTES: u32 = 64 * 1024; + const MAX_MESSAGE_BYTES: u32 = 1024; + // A payload between the message cap and the frame cap. admit_inbound_message + // would reject it, but only after read_frame allocated and read it. + const OVER_MESSAGE_PAYLOAD_LEN: u32 = 2048; + let stream_kind = LEGACY_GOSSIP_STREAM_KIND; + + let limits = ZakuraConnectionLimits { + max_frame_bytes: MAX_FRAME_BYTES, + max_message_bytes: MAX_MESSAGE_BYTES, + ..test_connection_limits() + }; + // Production now passes the message-limited inbound cap; the raw + // application cap (what the unfixed read path used) stays at the frame cap. + let inbound_cap = inbound_frame_cap_for_stream_kind(&limits, stream_kind); + let raw_cap = app_frame_cap_for_stream_kind(&limits, stream_kind); + assert!( + inbound_cap < raw_cap, + "the inbound cap must be tighter than the raw frame cap when the caps diverge \ + (inbound_cap={inbound_cap}, raw_cap={raw_cap})" + ); + + let _guard = zebra_test::init(); + let server = LocalEndpointFactory::new().endpoint(72).await?; + let (conn_tx, _conn_rx) = mpsc::channel(2); + let (stream_tx, mut stream_rx) = mpsc::channel(4); + let router = Router::builder(server) + .accept( + ALPN, + CaptureConnection { + connection_tx: conn_tx, + stream_tx, + }, + ) + .spawn(); + let client = LocalEndpointFactory::new().endpoint(73).await?; + let server_addr = router.endpoint().node_addr().initialized().await; + client.add_node_addr(server_addr.clone())?; + + // A frame header (message_type, flags, payload_len) with no payload bytes. + let frame_header = |payload_len: u32| -> Vec { + let mut header = Vec::with_capacity(FRAME_HEADER_BYTES); + header.extend_from_slice(&0u16.to_le_bytes()); + header.extend_from_slice(&0u16.to_le_bytes()); + header.extend_from_slice(&payload_len.to_le_bytes()); + header + }; + + // CaptureConnection forwards at most two streams per connection, so the + // two oversized streams share one connection and the in-cap stream uses + // another. + let conn_a = timeout( + Duration::from_secs(10), + client.connect(server_addr.clone(), ALPN), + ) + .await + .expect("client connects for the oversized streams")?; + + // Stream 1: oversized header read with the message-limited inbound cap. + // The fix rejects it as Oversize from the header alone, before allocating. + let (mut over_send, _over_recv) = + timeout(Duration::from_secs(1), conn_a.open_bi()) + .await + .expect("client opens the oversized inbound-cap stream")?; + timeout( + Duration::from_secs(1), + over_send.write_all(&frame_header(OVER_MESSAGE_PAYLOAD_LEN)), + ) + .await + .expect("client writes the oversized frame header")?; + let _ = over_send.finish(); + let (_s1_send, mut s1_recv) = timeout(Duration::from_secs(1), stream_rx.recv()) + .await + .expect("server accepts the oversized inbound-cap stream") + .expect("capture handler forwards the oversized inbound-cap stream"); + let rejected = read_frame(&mut s1_recv, inbound_cap, Duration::from_secs(2)).await; + assert!( + matches!(rejected, Err(ZakuraHandlerError::OversizeFrame { .. })), + "a frame whose payload exceeds max_message_bytes must be rejected as Oversize \ + on the header alone with the inbound (message-limited) cap, before the payload \ + is allocated and read; got {rejected:?}" + ); + + // Stream 2: the SAME oversized header read with the raw frame cap an + // unfixed path used. It passes the frame-cap size check, so read_frame + // allocates `vec![0; payload_len]` and reads the body (failing only because + // the body was never sent) -- i.e. NOT rejected as Oversize. This is the + // allocate-before-reject amplification the fix removes. + let (mut raw_send, _raw_recv) = timeout(Duration::from_secs(1), conn_a.open_bi()) + .await + .expect("client opens the oversized raw-cap stream")?; + timeout( + Duration::from_secs(1), + raw_send.write_all(&frame_header(OVER_MESSAGE_PAYLOAD_LEN)), + ) + .await + .expect("client writes the oversized frame header again")?; + let _ = raw_send.finish(); + let (_s2_send, mut s2_recv) = timeout(Duration::from_secs(1), stream_rx.recv()) + .await + .expect("server accepts the oversized raw-cap stream") + .expect("capture handler forwards the oversized raw-cap stream"); + let allocated = read_frame(&mut s2_recv, raw_cap, Duration::from_secs(2)).await; + assert!( + allocated.is_err() + && !matches!( + allocated, + Err(ZakuraHandlerError::Oversize | ZakuraHandlerError::OversizeFrame { .. }) + ), + "with the raw frame cap the same oversized frame passes the size check and \ + read_frame proceeds to allocate/read the payload (it is not rejected as \ + Oversize), proving the message cap is enforced too late; got {allocated:?}" + ); + + // Stream 3 (fresh connection): a frame within the message cap must still + // round-trip with the inbound cap, so the clamp rejects nothing legitimate. + let conn_b = timeout(Duration::from_secs(10), client.connect(server_addr, ALPN)) + .await + .expect("client connects for the in-cap stream")?; + let valid_payload = vec![0x5au8; (MAX_MESSAGE_BYTES / 2) as usize]; + let mut valid_bytes = + frame_header(u32::try_from(valid_payload.len()).expect("in-cap payload len fits u32")); + valid_bytes.extend_from_slice(&valid_payload); + let (mut ok_send, _ok_recv) = timeout(Duration::from_secs(1), conn_b.open_bi()) + .await + .expect("client opens the in-cap stream")?; + timeout(Duration::from_secs(1), ok_send.write_all(&valid_bytes)) + .await + .expect("client writes the in-cap frame")?; + let _ = ok_send.finish(); + let (_s3_send, mut s3_recv) = timeout(Duration::from_secs(1), stream_rx.recv()) + .await + .expect("server accepts the in-cap stream") + .expect("capture handler forwards the in-cap stream"); + let frame = read_frame(&mut s3_recv, inbound_cap, Duration::from_secs(2)) + .await + .expect("a frame within the message cap is read with the inbound cap"); + assert_eq!( + frame.payload, valid_payload, + "an in-cap frame payload must round-trip unchanged" + ); + + conn_a.close(0u32.into(), b"done"); + conn_b.close(0u32.into(), b"done"); + client.close().await; + router.shutdown().await?; + Ok(()) + } + + #[tokio::test] + async fn invalid_prelude_stream_churn_charges_open_rate_token() -> Result<(), BoxError> { + // claude-invalid-stream-prelude-churn-bypasses-open-rate: a peer that + // opens a stream naming an unregistered kind is reset stream-only and the + // connection is kept. The per-connection stream-open rate token MUST be + // charged for that protocol-invalid open; otherwise an authenticated peer + // could churn bad-prelude/unknown-kind/unnegotiated stream opens forever + // without ever spending open-rate budget. This drives admit_bi_stream + // over a real Iroh transport and asserts the token was consumed even + // though the stream itself was rejected as unknown-kind. + const ALPN: &[u8] = b"/zakura/testkit/open-rate-churn/0"; + + let _guard = zebra_test::init(); + let server = LocalEndpointFactory::new().endpoint(90).await?; + let (conn_tx, _conn_rx) = mpsc::channel(1); + let (stream_tx, mut stream_rx) = mpsc::channel(2); + let router = Router::builder(server) + .accept( + ALPN, + CaptureConnection { + connection_tx: conn_tx, + stream_tx, + }, + ) + .spawn(); + let client = LocalEndpointFactory::new().endpoint(91).await?; + let server_addr = router.endpoint().node_addr().initialized().await; + client.add_node_addr(server_addr.clone())?; + let client_conn = timeout(Duration::from_secs(10), client.connect(server_addr, ALPN)) + .await + .expect("client connects to the open-rate-churn endpoint")?; + + // A well-formed prelude that names an unregistered stream kind. It parses + // cleanly, so admission reaches the unknown-kind reject (a stream-only + // reset that keeps the connection) rather than the bad-prelude path. + let prelude = StreamPrelude { + magic: STREAM_PRELUDE_MAGIC, + stream_kind: 9, + stream_version: 1, + request_id: None, + max_frame_bytes: 1024, + }; + let (mut client_send, _client_recv) = + timeout(Duration::from_secs(1), client_conn.open_bi()) + .await + .expect("client opens the unknown-kind stream")?; + timeout( + Duration::from_secs(1), + client_send.write_all(&prelude.encode()?), + ) + .await + .expect("client writes the unknown-kind prelude")?; + let _ = client_send.finish(); + + let (server_send, server_recv) = timeout(Duration::from_secs(1), stream_rx.recv()) + .await + .expect("server accepts the unknown-kind stream") + .expect("capture handler forwards the unknown-kind stream"); + + let supervisor = ZakuraSupervisorHandle::new(16); + let handler = ZakuraProtocolHandler::new( + supervisor, + Network::Mainnet, + ZakuraHandshakeConfig::for_network(&Network::Mainnet), + ZakuraLocalLimits::from_config(&Config::default()), + ); + + let peer_id = test_peer(9); + let stream_sem = Arc::new(Semaphore::new(16)); + // A full bucket with a known capacity so the token charge is observable + // as an exact decrement. + let mut open_limiter = TokenBucket::new(4); + let mut message_buckets = MessageRateBuckets::new(); + let mut workers = JoinSet::new(); + let connection_token = CancellationToken::new(); + let (freshness_tx, _freshness_rx) = watch::channel(Instant::now()); + + let mut admission = StreamAdmission { + trace: handler.trace.clone(), + conn: ZakuraConnTrace::placeholder(), + peer_id: &peer_id, + stream_sem: &stream_sem, + open_limiter: &mut open_limiter, + message_buckets: &mut message_buckets, + workers: &mut workers, + limits: test_connection_limits(), + accepted_capabilities: 0, + connection_token: connection_token.clone(), + freshness_tx, + }; + + let admitted = handler + .admit_bi_stream(server_send, server_recv, &mut admission, 16) + .await; + + assert!( + admitted.is_none(), + "an unknown-kind stream must be rejected, not admitted" + ); + assert!( + !connection_token.is_cancelled(), + "an unknown-kind stream is reset stream-only and must keep the connection alive" + ); + assert_eq!( + admission.open_limiter.tokens, 3, + "the protocol-invalid stream open must spend exactly one open-rate token \ + (capacity 4 -> 3); before the fix the unknown-kind reject returned before \ + reaching the limiter, leaving the bucket full at 4" + ); + + client.close().await; + router.shutdown().await?; + Ok(()) + } + #[test] fn local_application_frame_cap_admits_default_header_sync_response() { let limits = ZakuraLocalLimits::from_config(&Config::default()); @@ -4594,48 +6063,62 @@ mod tests { }, Stream { kind: HEADER_SYNC_STREAM_KIND, - version: ZAKURA_STREAM_VERSION_1, + version: ZAKURA_STREAM_VERSION_2, frame_cap: 1024, capability: ZAKURA_CAP_HEADER_SYNC, mode: StreamMode::Ordered, }, + Stream { + kind: ZAKURA_STREAM_BLOCK_SYNC, + version: ZAKURA_STREAM_VERSION_1, + frame_cap: MAX_BS_FRAME_BYTES, + capability: crate::zakura::ZAKURA_CAP_BLOCK_SYNC, + mode: StreamMode::Ordered, + }, ], }) as Arc]) .expect("test registry declares unique stream kinds"); - for kind in [ - LEGACY_GOSSIP_STREAM_KIND, - LEGACY_REQUEST_STREAM_KIND, - DISCOVERY_STREAM_KIND, - HEADER_SYNC_STREAM_KIND, + for (kind, version) in [ + (LEGACY_GOSSIP_STREAM_KIND, ZAKURA_STREAM_VERSION_1), + (LEGACY_REQUEST_STREAM_KIND, ZAKURA_STREAM_VERSION_1), + (DISCOVERY_STREAM_KIND, ZAKURA_STREAM_VERSION_1), + (HEADER_SYNC_STREAM_KIND, ZAKURA_STREAM_VERSION_2), + (ZAKURA_STREAM_BLOCK_SYNC, ZAKURA_STREAM_VERSION_1), ] { assert!( - is_supported_stream(®istry, kind, ZAKURA_STREAM_VERSION_1), - "registered kind {kind} at version 1 must be supported" + is_supported_stream(®istry, kind, version), + "registered kind {kind} at declared version {version} must be supported" ); assert!( !is_supported_stream(®istry, kind, 0), "registered kind {kind} at version 0 must be rejected" ); assert!( - !is_supported_stream(®istry, kind, 2), + !is_supported_stream(®istry, kind, version.saturating_add(1)), "registered kind {kind} at an unsupported version must be rejected" ); } + assert!( + !is_supported_stream(®istry, HEADER_SYNC_STREAM_KIND, ZAKURA_STREAM_VERSION_1), + "header-sync v1 is intentionally rejected after the clean v2 break" + ); + assert_eq!(stream_kind_label(2), "gossip"); assert_eq!(stream_kind_label(3), "legacy_request"); assert_eq!(stream_kind_label(4), "discovery"); assert_eq!(stream_kind_label(5), "header_sync"); + assert_eq!(stream_kind_label(6), "block_sync"); - for kind in [0u16, 1, 6, 7, 255, u16::MAX] { + for kind in [0u16, 1, 7, 255, u16::MAX] { assert!( !is_supported_stream(®istry, kind, ZAKURA_STREAM_VERSION_1), "unknown kind {kind} must be rejected even at version 1" ); } - for kind in [6u16, 7, 255, u16::MAX] { + for kind in [7u16, 255, u16::MAX] { assert_eq!(stream_kind_label(kind), "unknown"); } } @@ -4735,8 +6218,12 @@ mod tests { limits, ) .map_err(|error| -> BoxError { format!("{error:?}").into() })?; - let frames = - LegacyResponseCodec::encode_response(request_id, response, limits.max_frame_bytes)?; + let frames = LegacyResponseCodec::encode_response( + request_id, + response, + limits.max_frame_bytes, + limits.max_message_bytes, + )?; LegacyResponseCodec::decode_response(request_id, request_kind, frames.clone())?; let mut state = LegacyResponseReadState::new(budget); @@ -4893,4 +6380,354 @@ mod tests { < limits.quic_idle_timeout.as_millis() ); } + + // SECURITY AUDIT (candidate claude-legacy-requester-response-frame-growth / + // trace-gossip-response-reassembler-frame-vector-growth): SR-4 amplification. + // + // `write_outbound_request_frame_inner` accumulates every validated response + // Frame into a `Vec` until the responder closes the stream, then hands + // the whole vector to `decode_response`. The only thing bounding how much it + // retains is `LegacyResponseBudget::max_bytes`, which `validate_frame` checks + // as a cumulative byte budget. For Blocks/Transactions that budget was + // derived as `item_count * max_message_bytes`, so a request naming the + // protocol-max inventory count (`MAX_TX_INV_IN_SENT_MESSAGE` = 25_000) handed + // a hostile responder a ~50 GiB retained-frame budget before any decode. + // + // The inbound responder already caps a single response's cumulative payload + // at `LEGACY_RESPONSE_MAX_AGGREGATE_BYTES` (8 * MAX_PROTOCOL_MESSAGE_LEN), so + // an honest peer never sends more than that. The requester must clamp its + // retained-frame budget to the same operational aggregate. This test asserts + // the budget is bounded regardless of requested item count; it FAILS before + // the fix (budget ~= 50 GiB) and passes after. Do not weaken it to pass. + #[test] + fn requester_response_budget_is_capped_for_large_inventory_request() { + let limits = test_connection_limits(); + assert_eq!( + limits.max_message_bytes as usize, MAX_PROTOCOL_MESSAGE_LEN, + "fixture should negotiate the protocol-max message cap so the unclamped \ + budget is maximal", + ); + + let max_items = + usize::try_from(MAX_TX_INV_IN_SENT_MESSAGE).expect("inventory cap fits in usize"); + + // A BlocksByHash request naming the protocol-max inventory count must not + // grant a retained-frame byte budget above the operational aggregate cap. + let blocks = LegacyRequestFrame::BlocksByHash(vec![block_hash(7); max_items]) + .encode_frame() + .expect("max-inventory blocks request encodes"); + let blocks_budget = + LegacyResponseBudget::from_request(blocks.message_type, &blocks.payload, limits) + .expect("budget derives from a max-inventory blocks request"); + assert!( + blocks_budget.max_bytes <= LEGACY_RESPONSE_MAX_AGGREGATE_BYTES, + "BlocksByHash retained-frame budget {} must be clamped to the aggregate cap {}", + blocks_budget.max_bytes, + LEGACY_RESPONSE_MAX_AGGREGATE_BYTES, + ); + + // The transaction-fetch path scales identically and must be clamped too. + let txs = LegacyRequestFrame::TransactionsById(vec![legacy_tx_id(7); max_items]) + .encode_frame() + .expect("max-inventory transactions request encodes"); + let txs_budget = LegacyResponseBudget::from_request(txs.message_type, &txs.payload, limits) + .expect("budget derives from a max-inventory transactions request"); + assert!( + txs_budget.max_bytes <= LEGACY_RESPONSE_MAX_AGGREGATE_BYTES, + "TransactionsById retained-frame budget {} must be clamped to the aggregate cap {}", + txs_budget.max_bytes, + LEGACY_RESPONSE_MAX_AGGREGATE_BYTES, + ); + + // The clamp must only remove the unbounded tail: a modest request still + // has to accept at least one full negotiated message, otherwise we would + // wrongly reject honest single-item responses. + let small = LegacyRequestFrame::BlocksByHash(vec![block_hash(1)]) + .encode_frame() + .expect("single-item blocks request encodes"); + let small_budget = + LegacyResponseBudget::from_request(small.message_type, &small.payload, limits) + .expect("budget derives from a single-item blocks request"); + assert!( + small_budget.max_bytes >= limits.max_message_bytes as usize, + "a single-item request must still permit one full response message; budget was {}", + small_budget.max_bytes, + ); + assert!( + small_budget.max_bytes <= LEGACY_RESPONSE_MAX_AGGREGATE_BYTES, + "even a single-item budget stays within the aggregate cap", + ); + } + + // SECURITY AUDIT (candidate claude-legacy-nil-response-nonfatal / + // subset-response-correlation-gossip-nil-response-nonfatal): SR-7 fail-closed. + // + // The outbound request stream worker decides connection-fatality from + // `LegacyResponseReadState::validate_frame`: `Fatal` => connection.close() + + // connection_token.cancel() (the peer is disconnected); `Ok`/`Local` => the + // peer stays connected and the request just returns an error. `validate_nil` + // accepts a `MSG_RESPONSE_NIL` sentinel for *every* request kind, so a peer + // that answers an inventory fetch (BlocksByHash / TransactionsById) or a Ping + // with a correct-id NIL passes the transport budget layer (worker returns + // Ok(frames)) and is NOT disconnected. Only the later `decode_response` layer + // rejects NIL for these kinds -- as an ordinary request-local error. The two + // layers disagree, so an unexpected/unsolicited response is tolerated instead + // of failing closed. + // + // This test asserts the SAFE behavior (the transport budget layer must reject + // NIL for inventory/Ping kinds as `Fatal`, so the worker disconnects). It + // currently FAILS, which is the reproduction. Do not weaken it to pass. + #[test] + fn nil_response_to_inventory_or_ping_request_is_not_fail_closed() { + let limits = test_connection_limits(); + let request_id = 99; + + let cases: [(LegacyRequestFrame, LegacyRequestKind); 3] = [ + ( + LegacyRequestFrame::BlocksByHash(vec![block_hash(1)]), + LegacyRequestKind::Blocks, + ), + ( + LegacyRequestFrame::TransactionsById(vec![legacy_tx_id(2)]), + LegacyRequestKind::Transactions, + ), + (LegacyRequestFrame::Ping, LegacyRequestKind::Ping), + ]; + + for (request, request_kind) in cases { + let request_frame = request.encode_frame().expect("request frame encodes"); + let budget = LegacyResponseBudget::from_request( + request_frame.message_type, + &request_frame.payload, + limits, + ) + .expect("budget derives from request"); + + // A hostile/buggy responder serializes Response::Nil with the real + // codec, addressed to our request id. + let nil_frames = LegacyResponseCodec::encode_response( + request_id, + Response::Nil, + limits.max_frame_bytes, + limits.max_message_bytes, + ) + .expect("nil response encodes"); + + // The higher decode layer DOES reject NIL for these kinds... + let decoded = + LegacyResponseCodec::decode_response(request_id, request_kind, nil_frames.clone()); + assert!( + decoded.is_err(), + "decode_response must reject a bare NIL for {request_kind:?}", + ); + + // ...but the transport budget layer -- the one that drives the + // fail-closed disconnect in the request stream worker -- must ALSO + // reject it as Fatal. It currently accepts it. + let mut state = LegacyResponseReadState::new(budget); + let mut validate = Ok(()); + for frame in &nil_frames { + validate = state.validate_frame(request_id, frame); + if validate.is_err() { + break; + } + } + let validate = validate.and_then(|()| state.finish()); + assert!( + matches!(validate, Err(OutboundRequestError::Fatal(_))), + "transport must fail closed (Fatal) on a NIL answer to {request_kind:?} so the \ + request stream worker disconnects the peer; got {validate:?}", + ); + } + } + + // SECURITY AUDIT (candidate claude-inbound-per-ip-cap-bypassed / + // codex-inbound-per-ip-cap-bypass): SR-4 admission. + // + // `accept_connection` used to register every inbound peer with + // `remote_ip = None`, and `register` only consults `active_by_ip` when + // `remote_ip` is `Some`, so the per-IP connection cap was enforced for native + // outbound dials (which pass a real IP) but entirely bypassed for inbound + // Router accepts: one source IP could authenticate as many distinct iroh node + // ids and fill the global connection budget despite `max_connections_per_ip` + // (default 1). The fix resolves the inbound peer's UDP source IP from the + // endpoint's node map at the accept site and passes it into `register`, so the + // per-IP cap now applies to Router-accepted connections too. + // + // This guard drives the real production `ProtocolHandler::accept` over a + // loopback iroh transport: two distinct authenticated identities dial from the + // same source IP (127.0.0.1) through the full native handshake. The per-IP cap + // is 1 while global admission keeps its default (well above 1), so the second + // identity can only be turned away by the per-IP cap, not the global gate. + // Before the fix both identities registered; now the second is rejected and + // its connection is closed, leaving exactly one registered peer. + #[tokio::test] + async fn inbound_accept_enforces_per_ip_cap() -> Result<(), BoxError> { + let _guard = zebra_test::init(); + + // The register-level invariant the accept path now relies on: with a real + // source IP, the per-IP cap rejects a second distinct identity with + // `ResourceLimit`. + async fn try_register( + supervisor: &ZakuraSupervisorHandle, + peer: &ZakuraPeerId, + remote_ip: Option, + ) -> ZakuraRegistration { + let (outbound_tx, _outbound_rx) = mpsc::channel(1); + let outbound_handle = ZakuraPeerHandle::new_for_tests(peer.clone(), outbound_tx); + supervisor + .register( + peer.clone(), + remote_ip, + [peer.as_bytes()[0]; TRANSCRIPT_HASH_BYTES], + outbound_handle, + CancellationToken::new(), + ZAKURA_CAP_LEGACY_GOSSIP | ZAKURA_CAP_HEADER_SYNC, + ) + .await + } + let ip: IpAddr = "203.0.113.7".parse().expect("test ip parses"); + let supervisor = ZakuraSupervisorHandle::new(1); + assert!( + matches!( + try_register(&supervisor, &test_peer(1), Some(ip)).await, + ZakuraRegistration::Registered { .. } + ), + "first identity from the IP registers", + ); + assert!( + matches!( + try_register(&supervisor, &test_peer(2), Some(ip)).await, + ZakuraRegistration::Rejected(ZakuraRejectReason::ResourceLimit) + ), + "a second distinct identity from the same IP must be Rejected(ResourceLimit) at cap 1", + ); + + // End-to-end: drive the production accept path with a per-IP cap of 1 and a + // strictly larger global cap so per-IP admission is what turns away the + // second same-IP identity. Wire the bound endpoint so the accept path can + // resolve the inbound source IP. + let limits = ZakuraLocalLimits::from_config(&Config::default()); + assert!( + limits.max_connections > 1, + "global admission cap must exceed the per-IP cap so the second same-IP identity is \ + turned away by the per-IP cap rather than the global gate", + ); + let server_ep = LocalEndpointFactory::with_transport_config(limits.transport_config()) + .endpoint(880) + .await?; + let supervisor = ZakuraSupervisorHandle::new(1); + let handler = ZakuraProtocolHandler::new( + supervisor.clone(), + Network::Mainnet, + ZakuraHandshakeConfig::for_network(&Network::Mainnet), + limits.clone(), + ) + .with_endpoint(server_ep.clone()); + let router = Router::builder(server_ep) + .accept(P2P_V2_ALPN, handler) + .spawn(); + // Iroh also binds a default IPv6 socket, so an unrestricted node address + // lets the two clients reach the server over different paths (e.g. one + // IPv4 loopback, one global IPv6) and therefore present different source + // IPs. Pin both dials to the server's IPv4 loopback address so they share + // one source IP (127.0.0.1) -- the single-source-IP shape of the finding. + let full_addr = router.endpoint().node_addr().initialized().await; + let loopback_addr = NodeAddr::new(full_addr.node_id).with_direct_addresses( + full_addr + .direct_addresses() + .copied() + .filter(|addr| addr.is_ipv4() && addr.ip().is_loopback()), + ); + assert!( + loopback_addr.direct_addresses().next().is_some(), + "server must advertise an IPv4 loopback direct address", + ); + let server_addr = loopback_addr; + + // Establish the QUIC connection only; the native handshake is driven + // separately so a per-IP rejection mid-handshake (the second identity) + // can be observed instead of aborting the test. + async fn connect_native( + server_addr: &NodeAddr, + seed: u64, + limits: &ZakuraLocalLimits, + ) -> Result<(Endpoint, Connection), BoxError> { + let endpoint = LocalEndpointFactory::with_transport_config(limits.transport_config()) + .endpoint(seed) + .await?; + endpoint.add_node_addr(server_addr.clone())?; + let connection = endpoint.connect(server_addr.clone(), P2P_V2_ALPN).await?; + Ok((endpoint, connection)) + } + async fn run_handshake( + endpoint: &Endpoint, + connection: &Connection, + limits: &ZakuraLocalLimits, + ) -> Result<(), BoxError> { + let config = ZakuraHandshakeConfig::for_network(&Config::default().network); + let local_peer_id = ZakuraPeerId::new(endpoint.node_id().as_bytes().to_vec())?; + run_native_initiator_handshake_without_trace( + connection, + limits, + &config, + &local_peer_id, + ) + .await?; + Ok(()) + } + + // First identity registers and claims the only per-IP slot. Hold the + // endpoint/connection so the peer stays registered for the second dial. + let (_ep1, _conn1) = connect_native(&server_addr, 881, &limits).await?; + run_handshake(&_ep1, &_conn1, &limits).await?; + let mut first_registered = 0; + for _ in 0..200 { + first_registered = supervisor.registered_ids().await.len(); + if first_registered >= 1 { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + assert_eq!( + first_registered, 1, + "the first inbound identity from the source IP must register (a real source IP was \ + resolved from the endpoint and counted against the per-IP cap)", + ); + + // Second distinct identity from the same source IP: the per-IP cap must + // reject its registration. The handshake may complete and then be closed, + // or be torn down mid-handshake by the rejection -- either way the server + // closes the connection with the resource-limit code and never registers a + // second peer. (Before the fix the accept passed remote_ip = None, so this + // identity registered and one source IP could exhaust the global budget.) + let (_ep2, conn2) = connect_native(&server_addr, 882, &limits).await?; + let _ = run_handshake(&_ep2, &conn2, &limits).await; + let mut rejected_close = false; + for _ in 0..400 { + if supervisor.registered_ids().await.len() >= 2 { + break; + } + if matches!( + conn2.close_reason(), + Some(iroh::endpoint::ConnectionError::ApplicationClosed(ref close)) + if close.error_code == VarInt::from_u32(ZAKURA_CLOSE_RESOURCE) + ) { + rejected_close = true; + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + let registered = supervisor.registered_ids().await.len(); + assert!( + rejected_close && registered == 1, + "a second distinct identity from the same source IP must be rejected by the per-IP \ + cap with a resource-limit close (resource_close={rejected_close}, \ + registered={registered}); before the fix the inbound accept passed remote_ip = None, \ + so both identities registered and one source IP could exhaust the connection budget", + ); + + router.shutdown().await?; + Ok(()) + } } diff --git a/zebra-network/src/zakura/header_sync/config.rs b/zebra-network/src/zakura/header_sync/config.rs index ee92cabc8e5..0dc545584f2 100644 --- a/zebra-network/src/zakura/header_sync/config.rs +++ b/zebra-network/src/zakura/header_sync/config.rs @@ -62,6 +62,11 @@ pub struct ZakuraHeaderSyncConfig { pub status_refresh_interval: Duration, /// Header-sync peer caps and queue limits owned by this reactor. pub peer_limits: ServicePeerLimits, + /// Accept full blocks delivered on Zakura full-block gossip paths. + /// + /// Disabling this keeps range-based header sync and legacy request/response + /// active while forcing block bodies to arrive through the block-sync stream. + pub accept_new_blocks: bool, /// Optional trusted header-sync anchor height. /// /// When unset, header sync starts from genesis. When set, [`anchor_hash`](Self::anchor_hash) @@ -81,6 +86,7 @@ impl Default for ZakuraHeaderSyncConfig { max_inflight_requests: DEFAULT_HS_MAX_INFLIGHT, status_refresh_interval: DEFAULT_HS_STATUS_REFRESH_INTERVAL, peer_limits: ServicePeerLimits::default(), + accept_new_blocks: true, anchor_height: None, anchor_hash: None, } @@ -130,7 +136,8 @@ pub fn header_sync_count_by_byte_budget(network: &Network, max_frame_bytes: u32) .unwrap_or(usize::MAX) .saturating_sub(FRAME_HEADER_BYTES); let payload_cap = MAX_HS_MESSAGE_BYTES.min(frame_payload_cap); - let header_bytes = header_sync_header_bytes_for_network(network); + let header_bytes = + header_sync_header_bytes_for_network(network).saturating_add(HEADER_SYNC_BODY_SIZE_BYTES); let count = payload_cap .saturating_sub(HEADER_SYNC_MESSAGE_TYPE_BYTES + HEADER_SYNC_COUNT_BYTES) / header_bytes; diff --git a/zebra-network/src/zakura/header_sync/error.rs b/zebra-network/src/zakura/header_sync/error.rs index 4b4f5b4f609..f8bbd545d36 100644 --- a/zebra-network/src/zakura/header_sync/error.rs +++ b/zebra-network/src/zakura/header_sync/error.rs @@ -36,6 +36,15 @@ pub enum HeaderSyncWireError { max: usize, }, + /// A locally constructed `Headers` message had a different number of size hints. + #[error("Zakura header-sync Headers body-size count {body_sizes} does not match header count {headers}")] + BodySizeCountMismatch { + /// Header count. + headers: usize, + /// Body-size hint count. + body_sizes: usize, + }, + /// An inbound `Headers` response did not match an in-flight request. #[error("unsolicited Zakura header-sync Headers response")] UnsolicitedHeaders, diff --git a/zebra-network/src/zakura/header_sync/events.rs b/zebra-network/src/zakura/header_sync/events.rs index 94a338a52fb..501cc06b0a0 100644 --- a/zebra-network/src/zakura/header_sync/events.rs +++ b/zebra-network/src/zakura/header_sync/events.rs @@ -1,6 +1,6 @@ use super::{config::*, error::*, validation::*, wire::*, *}; use crate::zakura::{ - HeaderSyncPeerSession, HeaderSyncServiceSummary, ServicePeerSnapshot, + FrontierUpdate, HeaderSyncPeerSession, HeaderSyncServiceSummary, ServicePeerSnapshot, ZakuraHeaderSyncCandidateState, }; @@ -11,6 +11,8 @@ pub struct HeaderSyncFrontiers { pub finalized_height: block::Height, /// Highest verified block body height, supplied by state. pub verified_block_tip: block::Height, + /// Hash at the highest verified block body height, supplied by state. + pub verified_block_hash: block::Hash, } /// Startup inputs for the dependency-neutral header-sync reactor. @@ -24,6 +26,8 @@ pub struct HeaderSyncStartup { pub frontiers: HeaderSyncFrontiers, /// Durable best header tip loaded from storage at startup. pub best_header_tip: Option<(block::Height, block::Hash)>, + /// Shared sync exchange frontier stream. + pub frontier_updates: Option>, /// Local stream-5 advertisement. pub config: ZakuraHeaderSyncConfig, /// Negotiated or local application frame cap for header-sync responses. @@ -57,6 +61,7 @@ impl HeaderSyncStartup { anchor, frontiers, best_header_tip, + frontier_updates: None, config, max_frame_bytes, request_timeout: DEFAULT_HS_REQUEST_TIMEOUT, @@ -253,6 +258,8 @@ pub enum HeaderSyncEvent { requested_count: u32, /// Bounded headers returned by state. headers: Vec>, + /// Advisory serialized body sizes, parallel to `headers`. + body_sizes: Vec, }, } @@ -277,6 +284,8 @@ pub enum HeaderSyncAction { start_height: block::Height, /// Headers to commit. This is an output payload, not reactor state. headers: Vec>, + /// Advisory serialized body sizes, parallel to `headers`. + body_sizes: Vec, /// Whether the range is expected to be finalized by checkpoint policy. finalized: bool, }, @@ -312,6 +321,20 @@ pub enum HeaderSyncAction { /// Last missing height. to: block::Height, }, + /// Notify production wiring that header sync advanced its best header target. + HeaderAdvanced { + /// New best-header target height. + height: block::Height, + /// New best-header target hash. + hash: block::Hash, + }, + /// Notify production wiring that header sync re-anchored its best header target. + HeaderReanchored { + /// Previous best-header target. + old: (block::Height, block::Hash), + /// New best-header target. + new: (block::Height, block::Hash), + }, /// Inform later block-pipeline wiring that a validated tip block arrived. NewBlockReceived { /// Source peer. diff --git a/zebra-network/src/zakura/header_sync/mod.rs b/zebra-network/src/zakura/header_sync/mod.rs index 564bf8e4131..200844b8ee2 100644 --- a/zebra-network/src/zakura/header_sync/mod.rs +++ b/zebra-network/src/zakura/header_sync/mod.rs @@ -34,6 +34,7 @@ use super::{ mod config; mod error; mod events; +mod pipe; mod reactor; mod scheduler; mod service; diff --git a/zebra-network/src/zakura/header_sync/pipe.rs b/zebra-network/src/zakura/header_sync/pipe.rs new file mode 100644 index 00000000000..10d2df32ad5 --- /dev/null +++ b/zebra-network/src/zakura/header_sync/pipe.rs @@ -0,0 +1,679 @@ +//! header_sync/pipe.rs — the per-peer header-sync pipe (stream 5). +//! +//! THE PHASE-2 DAG SLICE IS THIS DIAGRAM. The code below is a mechanical +//! transcription; the [`PIPE_SHAPE`] const is the inspectable, drift-checked +//! copy of it. +//! +//! queued(GetHeaders) ─▶ command(record expected) ─▶ expected_headers.push_back +//! recv ─▶ guard ─┬─ Headers ─▶ expected_headers.pop_front ─▶ decode ─▶ forward(WireMessage) +//! └─ Control ───────────────────────────────▶ decode ─▶ forward(WireMessage) +//! +//! Phase 2 moves request/response correlation out of +//! [`HeaderSyncPeerSession`] and into [`HsLocal`]. The shared scheduler still +//! decides when to ask a peer for headers, but it sends that decision to this +//! peer-owned pipe as a command after the outbound `GetHeaders` is queued +//! successfully. The pipe prioritizes and drains those commands before inbound +//! frames so a response cannot beat its local expectation. This retires the +//! session mutex without changing the reactor's synthetic `WireMessage` test +//! path. + +use std::{collections::VecDeque, sync::Arc}; + +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::{events::*, scheduler::*, service::HeaderSyncPeerCommand, wire::*, *}; +use crate::zakura::{ + Edge, Flow, FramedRecv, Node, NodeKind, Pipe, PipeCx, PipeShape, SinkReject, ZakuraPeerId, +}; + +pub(super) struct HsLocal { + /// Plain peer-local response expectations, owned by this pipe task. + expected_headers: VecDeque, + /// Commands from shared scheduling state into this peer-local pipe. + commands: mpsc::UnboundedReceiver, + /// Pre-decode rate gate for inbound `NewBlock` floods. + /// + /// `NewBlock` is the only stream-5 message that deserializes a full + /// `Arc` (up to `MAX_HS_MESSAGE_BYTES`) directly from the wire. The + /// reactor's semantic `inbound_new_block` meter only fires *after* that + /// decode, so an authenticated peer could otherwise force one full-block + /// deserialization per frame before being metered. This gate enforces the + /// same minimum interval *before* decode so excess `NewBlock` frames are + /// dropped without ever reaching `Block::zcash_deserialize`. + new_block_meter: RateMeter, +} + +impl HsLocal { + /// Build per-peer local state around this peer's stream-5 session. + pub(super) fn new( + commands: mpsc::UnboundedReceiver, + new_block_min_interval: Duration, + ) -> Self { + Self { + expected_headers: VecDeque::new(), + commands, + new_block_meter: RateMeter::new(new_block_min_interval), + } + } + + /// Take one pre-decode `NewBlock` token. `false` means this frame arrived + /// faster than the minimum interval and must be dropped before decode. + fn admit_new_block(&mut self) -> bool { + self.new_block_meter.try_take(Instant::now()) + } + + fn pop_expected_headers_response(&mut self) -> Option { + self.expected_headers.pop_front() + } + + /// Restore a solicited-response expectation that was popped for decode but + /// whose decoded `Headers` event could not be handed to the reactor (the + /// bounded `events` queue was full or closed). It goes back to the *front* so + /// FIFO order is preserved and the reactor's still-outstanding range stays + /// correlated, instead of leaving the expectation silently consumed. + fn restore_expected_headers(&mut self, expected: ExpectedHeadersResponse) { + self.expected_headers.push_front(expected); + } + + fn handle_command(&mut self, command: HeaderSyncPeerCommand) { + match command { + HeaderSyncPeerCommand::RecordExpectedHeaders(expected) => { + self.expected_headers.push_back(expected); + } + } + } + + fn drain_ready_commands(&mut self) { + while let Ok(command) = self.commands.try_recv() { + self.handle_command(command); + } + } +} + +/// Shared environment handed to every header-sync pipe. +/// +/// Phase 1's environment is just the cloneable reactor handle: the decode stage +/// forwards each decoded message (or decode failure) to the unchanged reactor +/// over this handle. Cross-peer shared core state arrives in Phase 2. +#[derive(Clone)] +pub(super) struct HsEnv { + /// Handle used to forward inbound wire events to the header-sync reactor. + handle: HeaderSyncHandle, +} + +impl HsEnv { + /// Wrap a cloneable reactor handle as the pipe's shared environment. + pub(super) fn new(handle: HeaderSyncHandle) -> Self { + Self { handle } + } +} + +/// The Phase-2 header-sync pipe DAG slice, as checked documentation. +pub(super) const PIPE_SHAPE: PipeShape = PipeShape { + service: "header-sync", + nodes: &[ + Node { + id: "guard", + kind: NodeKind::Guard, + }, + Node { + id: "decode", + kind: NodeKind::Decode, + }, + Node { + id: "correlate", + kind: NodeKind::Mutate, + }, + Node { + id: "emit", + kind: NodeKind::Emit, + }, + ], + edges: &[ + Edge { + from: "guard", + to: "correlate", + on: "Headers", + }, + Edge { + from: "guard", + to: "decode", + on: "Control", + }, + Edge { + from: "correlate", + to: "decode", + on: "Expected", + }, + Edge { + from: "decode", + to: "emit", + on: "Ok", + }, + ], +}; + +/// Executable transcription of [`PIPE_SHAPE`] — the production entry function. +/// +/// The guard already admitted this frame (oversize-only) before `run_inbound` +/// is reached, so this is the `Headers|Control → correlate → decode → emit` tail. It +/// delegates to the single [`deliver`] implementation with the peer-owned +/// expected-response value, so the production pipe and the test/recorder +/// `deliver_frame` path can never diverge on *what* they decode or emit. +/// +/// The two callers differ only in how they treat a closed reactor queue, which +/// reproduces the old per-caller handling exactly: the production sink logged +/// the `SinkReject::Local` and continued the loop, so `run_inbound` maps that +/// one case to a debug log plus [`Flow::Done`] (which [`run_peer`] treats as +/// "continue"). Protocol rejects pass straight through and tear the peer down. +/// +/// One addition over the old per-caller handling: when a *solicited* `Headers` +/// response hits that local-reject path, the expectation popped before decode is +/// restored to [`HsLocal`] so reactor queue saturation cannot silently consume it +/// and strand the still-outstanding range. +pub(super) fn run_inbound(cx: &mut PipeCx<'_, HsLocal, HsEnv>, frame: Frame) -> Flow<()> { + // Pre-decode `NewBlock` rate gate: a `NewBlock` frame that arrives inside the + // per-peer minimum interval is dropped *before* the full `Arc` is + // deserialized, so a flood cannot force repeated full-block decode ahead of + // the reactor's semantic meter. Throttling (drop, keep the peer) matches the + // session guard's back-pressure outcome and the reactor's cheap + // dedup-without-scoring policy, so honest re-floods are not penalized; the + // first frame in each window still reaches the reactor, preserving + // first-offense malformed/spam disconnects. + if u8::try_from(frame.message_type).ok() == Some(MSG_HS_NEW_BLOCK) + && !cx.local.admit_new_block() + { + metrics::counter!("sync.header.tip.new_block.predecode_throttled").increment(1); + return Flow::Done; + } + + let expected = (u8::try_from(frame.message_type).ok() == Some(MSG_HS_HEADERS)) + .then(|| cx.local.pop_expected_headers_response()) + .flatten(); + match deliver(&cx.env.handle, expected, cx.peer_id.clone(), frame) { + Flow::Reject(SinkReject::Local(error)) => { + // The reactor `events` queue was full or closed, so this decoded frame + // could not be delivered locally. For a *solicited* `Headers` response + // the expectation was already popped before decode, so restore it: the + // reactor's matching range is still outstanding, and a consumed-but- + // undelivered expectation would otherwise lose the response entirely + // (recoverable only by the request timeout) and desynchronize the + // peer-local FIFO from that outstanding range. Restoring keeps the pipe + // in the same state as a request still awaiting its response, which the + // timeout/retry machinery already handles correctly. + if let Some(expected) = expected { + cx.local.restore_expected_headers(expected); + } + tracing::debug!( + ?error, + peer_id = ?cx.peer_id, + "header-sync stream could not deliver frame locally" + ); + Flow::Done + } + other => other, + } +} + +/// The single inbound decode/branch/forward stage, shared by both paths. +/// +/// This is the one decode implementation reachable from: +/// +/// - the production pipe's [`run_inbound`] (with `Some(session)` so a `Headers` +/// response is correlated against the peer's outstanding `GetHeaders`), and +/// - [`HeaderSyncService::deliver_frame`](super::service::HeaderSyncService) (the +/// test/recorder path, which passes `None` so a `Headers` response with no +/// outstanding request is rejected as `UnsolicitedHeaders`). +/// +/// It is a faithful port of the old `deliver_header_sync_frame`: the same events +/// fire on the same conditions, mapped onto [`Flow`]: +/// +/// - a successful forward to the reactor ⇒ [`Flow::Continue`], +/// - the old `SinkReject::Protocol` cases ⇒ [`Flow::Reject`] with a `Protocol` +/// reason (fatal — disconnect the peer), and +/// - the old `SinkReject::Local` "queue closed" case ⇒ [`Flow::Reject`] with a +/// `Local` reason. Each caller then maps `Local` to its old behavior: +/// `run_inbound` logs and continues, while `deliver_frame` returns it to the +/// registry as `Err(SinkReject::Local(_))`. +pub(super) fn deliver( + handle: &HeaderSyncHandle, + expected: Option, + peer_id: ZakuraPeerId, + frame: Frame, +) -> Flow<()> { + if u8::try_from(frame.message_type).ok() == Some(MSG_HS_HEADERS) { + let Some(expected) = expected else { + let error = Arc::new(HeaderSyncWireError::UnsolicitedHeaders); + let _ = handle.try_send(HeaderSyncEvent::WireProtocolFailure { + peer: peer_id.clone(), + reason: HeaderSyncMisbehavior::UnsolicitedHeaders, + error: error.clone(), + }); + let protocol_error = + std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); + return Flow::Reject(SinkReject::protocol(protocol_error)); + }; + + let msg = match HeaderSyncMessage::decode_frame( + frame, + HeaderSyncDecodeContext::for_headers_response(expected, expected.count), + ) { + Ok(msg) => msg, + Err(error) => { + let protocol_error = + std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); + let _ = handle.try_send(HeaderSyncEvent::WireProtocolFailure { + peer: peer_id.clone(), + reason: HeaderSyncMisbehavior::MalformedMessage, + error: Arc::new(error), + }); + return Flow::Reject(SinkReject::protocol(protocol_error)); + } + }; + + return forward(handle, HeaderSyncEvent::WireMessage { peer: peer_id, msg }); + } + + let msg = match decode_control_frame(frame) { + Ok(msg) => msg, + Err(error) => { + let protocol_error = + std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); + let _ = handle.try_send(HeaderSyncEvent::WireDecodeFailed { + peer: peer_id, + error: Arc::new(error), + }); + return Flow::Reject(SinkReject::protocol(protocol_error)); + } + }; + + forward(handle, HeaderSyncEvent::WireMessage { peer: peer_id, msg }) +} + +/// Run one peer-owned header-sync pipe until stream close, cancellation, or reject. +/// +/// Correlation-ordering invariant: a `RecordExpectedHeaders` command is enqueued +/// synchronously the moment its outbound `GetHeaders` is queued, so it is already +/// in this peer's command channel before any network response can return (a round +/// trip is orders of magnitude slower than a local enqueue). The loop drains +/// ready commands before every inbound frame, and the `biased` select prefers the +/// command channel over `recv`, so the expectation is always popped into +/// `HsLocal.expected_headers` before the matching `Headers` frame is decoded — +/// never the reverse, which would reject a solicited response as +/// `UnsolicitedHeaders`. +pub(super) async fn run_peer( + mut pipe: Pipe, + mut recv: FramedRecv, + cancel: CancellationToken, +) -> Result<(), SinkReject> { + enum Input { + Frame(Frame), + Command(HeaderSyncPeerCommand), + Done, + } + + loop { + pipe.local_mut().drain_ready_commands(); + + let input = { + let local = pipe.local_mut(); + tokio::select! { + biased; + () = cancel.cancelled() => Input::Done, + command = local.commands.recv() => match command { + Some(command) => Input::Command(command), + None => Input::Done, + }, + frame = recv.recv() => match frame { + Some(frame) => Input::Frame(frame), + None => Input::Done, + }, + } + }; + + match input { + Input::Done => return Ok(()), + Input::Frame(frame) => { + pipe.local_mut().drain_ready_commands(); + match pipe.run_one(frame) { + Flow::Continue(()) | Flow::Done => {} + Flow::Reject(reject) => return Err(reject), + } + } + Input::Command(command) => pipe.local_mut().handle_command(command), + } + } +} + +/// Forward a successfully decoded inbound event to the reactor. +/// +/// A closed reactor queue is a local, non-fatal condition for the peer: the old +/// `deliver_header_sync_frame` returned `SinkReject::local` here, so this returns +/// [`Flow::Reject`] with a `Local` reason. Callers decide whether to continue or +/// surface it (see [`deliver`]). +fn forward(handle: &HeaderSyncHandle, event: HeaderSyncEvent) -> Flow<()> { + match handle.try_send(event) { + Ok(()) => Flow::Continue(()), + Err(error) => Flow::Reject(SinkReject::local(format!( + "header-sync queue closed: {error}" + ))), + } +} + +/// Decode a non-`Headers` (control) frame. +/// +/// `Headers` frames need the peer's outstanding-request context and are handled +/// in [`deliver`]; a `Headers` frame reaching this path has no correlated +/// request, so it is rejected as `UnsolicitedHeaders` exactly as the old +/// `decode_header_sync_frame` did. +fn decode_control_frame(frame: Frame) -> Result { + if u8::try_from(frame.message_type).ok() == Some(MSG_HS_HEADERS) { + return Err(HeaderSyncWireError::UnsolicitedHeaders); + } + + HeaderSyncMessage::decode_frame(frame, HeaderSyncDecodeContext::control()) +} + +#[cfg(test)] +mod tests { + use tokio::sync::watch; + + use super::*; + use crate::zakura::{ServicePeerSnapshot, ZakuraHeaderSyncCandidateState}; + + const FRAME_FORKS: [&str; 2] = ["Headers", "Control"]; + + fn peer() -> ZakuraPeerId { + ZakuraPeerId::new(vec![5; 32]).expect("test peer id is within bounds") + } + + /// Build a `HeaderSyncHandle` whose bounded `events` queue the test can drain. + /// The watch frontiers are never read on the inbound decode path, so dummy + /// values suffice. + fn test_handle() -> (HeaderSyncHandle, mpsc::Receiver) { + let (events, events_rx) = mpsc::channel(16); + let (lifecycle, _lifecycle_rx) = mpsc::unbounded_channel(); + let (_tip_tx, tip) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let (_peers_tx, peers) = watch::channel(ServicePeerSnapshot::default()); + let (_candidates_tx, candidates) = + watch::channel(ZakuraHeaderSyncCandidateState::default()); + ( + HeaderSyncHandle { + events, + lifecycle, + tip, + peers, + candidates, + }, + events_rx, + ) + } + + /// Build a `HeaderSyncHandle` whose bounded `events` queue is already full, + /// so the next `try_send` from the pipe fails with `Full`. The receiver is + /// returned (and must be kept alive) so the failure is `Full`, not `Closed`. + fn saturated_events_handle() -> (HeaderSyncHandle, mpsc::Receiver) { + let (events, events_rx) = mpsc::channel(1); + events + .try_send(HeaderSyncEvent::PeerDisconnected(peer())) + .expect("the single events slot is free"); + let (lifecycle, _lifecycle_rx) = mpsc::unbounded_channel(); + let (_tip_tx, tip) = watch::channel((block::Height(0), block::Hash([0; 32]))); + let (_peers_tx, peers) = watch::channel(ServicePeerSnapshot::default()); + let (_candidates_tx, candidates) = + watch::channel(ZakuraHeaderSyncCandidateState::default()); + ( + HeaderSyncHandle { + events, + lifecycle, + tip, + peers, + candidates, + }, + events_rx, + ) + } + + fn headers_frame(payload: Vec) -> Frame { + Frame { + message_type: u16::from(MSG_HS_HEADERS), + flags: 0, + payload, + } + } + + /// A `Headers` frame with no recorded expectation is unsolicited: it reports + /// `UnsolicitedHeaders` misbehavior and rejects the peer, before any decode. + #[test] + fn deliver_unsolicited_headers_rejects_without_expectation() { + let (handle, mut events) = test_handle(); + + let flow = deliver(&handle, None, peer(), headers_frame(Vec::new())); + + assert!(matches!(flow, Flow::Reject(SinkReject::Protocol(_)))); + match events.try_recv() { + Ok(HeaderSyncEvent::WireProtocolFailure { reason, .. }) => { + assert!(matches!(reason, HeaderSyncMisbehavior::UnsolicitedHeaders)); + } + other => panic!("expected WireProtocolFailure(UnsolicitedHeaders), got {other:?}"), + } + } + + /// With a recorded expectation, the same `Headers` frame is *correlated* and + /// decoded: a malformed payload now reports `MalformedMessage`, not + /// `UnsolicitedHeaders`, proving the expectation was consumed before decode. + #[test] + fn deliver_correlated_headers_decodes_against_expectation() { + let (handle, mut events) = test_handle(); + let expected = ExpectedHeadersResponse::new(block::Height(1), 1).expect("count is valid"); + + let flow = deliver(&handle, Some(expected), peer(), headers_frame(Vec::new())); + + assert!(matches!(flow, Flow::Reject(SinkReject::Protocol(_)))); + match events.try_recv() { + Ok(HeaderSyncEvent::WireProtocolFailure { reason, .. }) => { + assert!(matches!(reason, HeaderSyncMisbehavior::MalformedMessage)); + } + other => panic!("expected WireProtocolFailure(MalformedMessage), got {other:?}"), + } + } + + /// The peer-local correlation queue is FIFO and is filled by draining ready + /// commands. This is the invariant `run_peer` relies on: an expectation + /// recorded by a `RecordExpectedHeaders` command is drained and available to + /// pop before the matching `Headers` response is processed. + #[test] + fn local_correlation_queue_drains_commands_in_fifo_order() { + let (commands_tx, commands_rx) = mpsc::unbounded_channel(); + let mut local = HsLocal::new(commands_rx, DEFAULT_HS_INBOUND_NEW_BLOCK_MIN_INTERVAL); + + let first = ExpectedHeadersResponse::new(block::Height(1), 1).expect("count is valid"); + let second = ExpectedHeadersResponse::new(block::Height(2), 2).expect("count is valid"); + commands_tx + .send(HeaderSyncPeerCommand::RecordExpectedHeaders(first)) + .expect("pipe is alive"); + commands_tx + .send(HeaderSyncPeerCommand::RecordExpectedHeaders(second)) + .expect("pipe is alive"); + + // Nothing is available until the pipe drains its ready commands. + assert_eq!(local.pop_expected_headers_response(), None); + local.drain_ready_commands(); + + assert_eq!(local.pop_expected_headers_response(), Some(first)); + assert_eq!(local.pop_expected_headers_response(), Some(second)); + assert_eq!(local.pop_expected_headers_response(), None); + } + + /// A `NewBlock` flood is throttled *before* full-block decode: the first + /// frame in a window is decoded and forwarded to the reactor, but a second + /// distinct frame inside the per-peer minimum interval is dropped before + /// `Block::zcash_deserialize` runs, so nothing reaches the reactor and the + /// peer is kept (`Flow::Done`). This proves the amplification gap is closed — + /// without the pre-decode gate the second full block is deserialized and + /// forwarded too. + #[test] + fn new_block_flood_is_throttled_before_decode() { + use zebra_chain::serialization::ZcashDeserializeInto; + use zebra_test::vectors::{BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES}; + + let (handle, mut events) = test_handle(); + let (_commands_tx, commands_rx) = mpsc::unbounded_channel(); + + let block_one: Arc = Arc::new( + BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into() + .expect("block 1 vector parses"), + ); + let block_two: Arc = Arc::new( + BLOCK_MAINNET_2_BYTES + .zcash_deserialize_into() + .expect("block 2 vector parses"), + ); + let frame_one = HeaderSyncMessage::NewBlock(block_one.clone()) + .encode_frame() + .expect("new block frame encodes"); + let frame_two = HeaderSyncMessage::NewBlock(block_two.clone()) + .encode_frame() + .expect("new block frame encodes"); + + let mut pipe = Pipe::new( + peer(), + HsLocal::new(commands_rx, DEFAULT_HS_INBOUND_NEW_BLOCK_MIN_INTERVAL), + HsEnv::new(handle), + crate::zakura::SessionGuard::oversize_only(MAX_HS_MESSAGE_BYTES as u32), + run_inbound, + &PIPE_SHAPE, + ); + + // First flood frame: admitted, decoded, and forwarded to the reactor. + assert!(matches!(pipe.run_one(frame_one), Flow::Continue(()))); + match events.try_recv() { + Ok(HeaderSyncEvent::WireMessage { + msg: HeaderSyncMessage::NewBlock(block), + .. + }) => assert_eq!(block.hash(), block_one.hash()), + other => panic!("expected first NewBlock to be forwarded, got {other:?}"), + } + + // Second distinct flood frame inside the interval is dropped before + // decode: the peer is kept and nothing reaches the reactor. + assert!(matches!(pipe.run_one(frame_two), Flow::Done)); + assert!( + matches!(events.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "second NewBlock must be throttled before decode, not forwarded" + ); + } + + /// Under reactor `events`-queue saturation, a valid *solicited* `Headers` + /// response must not silently consume its peer-local expectation. The pipe + /// pops the expectation before decode; when the decoded response cannot be + /// delivered to the full reactor queue, the pipe logs and continues + /// (`Flow::Done`) — but the popped expectation is restored to the FIFO so the + /// reactor's still-outstanding range stays correlated. Without the fix the + /// expectation is consumed and lost, stranding the range until the request + /// timeout and desynchronizing the peer-local FIFO from the outstanding range. + #[test] + fn saturated_events_queue_restores_solicited_expectation() { + use zebra_chain::serialization::ZcashDeserializeInto; + use zebra_test::vectors::BLOCK_MAINNET_1_BYTES; + + // Keep `_events_rx` alive so the saturated queue rejects with `Full` + // (a live receiver), not `Closed`. + let (handle, _events_rx) = saturated_events_handle(); + let (commands_tx, commands_rx) = mpsc::unbounded_channel(); + + let expected = ExpectedHeadersResponse::new(block::Height(1), 1).expect("count is valid"); + commands_tx + .send(HeaderSyncPeerCommand::RecordExpectedHeaders(expected)) + .expect("pipe is alive"); + + // A syntactically valid one-header solicited response: it decodes against + // the expectation and reaches the reactor forward, where the full queue + // turns it into a local reject. + let block_one: Arc = Arc::new( + BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into() + .expect("block 1 vector parses"), + ); + let solicited_headers = HeaderSyncMessage::Headers { + headers: vec![block_one.header.clone()], + body_sizes: vec![0], + } + .encode_frame() + .expect("headers frame encodes"); + + let mut pipe = Pipe::new( + peer(), + HsLocal::new(commands_rx, DEFAULT_HS_INBOUND_NEW_BLOCK_MIN_INTERVAL), + HsEnv::new(handle), + crate::zakura::SessionGuard::oversize_only(MAX_HS_MESSAGE_BYTES as u32), + run_inbound, + &PIPE_SHAPE, + ); + // Drain the recorded expectation into `HsLocal`, mirroring `run_peer`'s + // pre-frame command drain so the `Headers` frame is correlated. + pipe.local_mut().drain_ready_commands(); + + // The decoded response cannot be delivered (events queue is full); the + // pipe logs and continues, exactly as production does. + assert!(matches!(pipe.run_one(solicited_headers), Flow::Done)); + + // The popped expectation must be restored so the still-outstanding range + // stays correlated. Without the fix the expectation is gone (returns None). + assert_eq!( + pipe.local_mut().pop_expected_headers_response(), + Some(expected), + "a solicited Headers response dropped on reactor queue saturation must restore its expectation" + ); + } + + #[test] + fn pipe_shape_matches_runtime() { + // (a) The declared shape is internally consistent. + PIPE_SHAPE + .validate() + .expect("header-sync PIPE_SHAPE edges name only real nodes"); + + // (b) Phase 2's real runtime fork is still frame-shape based: + // `Headers` needs peer-local request correlation, while all other + // stream-5 messages decode as `Control` and are forwarded to the + // compatibility reactor for semantic dispatch. + let frame_forks: Vec<&str> = PIPE_SHAPE + .edges + .iter() + .filter(|edge| edge.from == "guard") + .map(|edge| edge.on) + .collect(); + + assert_eq!( + frame_forks.len(), + FRAME_FORKS.len(), + "guard has exactly the runtime frame-shape forks" + ); + for fork in FRAME_FORKS { + assert!( + frame_forks.contains(&fork), + "guard edge missing for runtime fork {fork}" + ); + } + + // (c) `Headers` responses correlate before decode; all decoded messages + // terminate at the single forward/emit stage. + assert!( + PIPE_SHAPE + .edges + .iter() + .any(|edge| edge.from == "correlate" && edge.to == "decode"), + "headers responses correlate before decode" + ); + assert!( + PIPE_SHAPE + .nodes + .iter() + .any(|node| node.id == "emit" && matches!(node.kind, NodeKind::Emit)), + "the pipe terminates at a single `emit` node" + ); + } +} diff --git a/zebra-network/src/zakura/header_sync/reactor.rs b/zebra-network/src/zakura/header_sync/reactor.rs index 28208d38235..54581940fce 100644 --- a/zebra-network/src/zakura/header_sync/reactor.rs +++ b/zebra-network/src/zakura/header_sync/reactor.rs @@ -1,9 +1,17 @@ use super::{config::*, error::*, events::*, scheduler::*, state::*, validation::*, wire::*, *}; use crate::zakura::{ - HeaderSyncServiceSummary, ServiceAdmissionDecision, ServicePeerDirection, ServicePeerSnapshot, - ZakuraHeaderSyncCandidateState, + FrontierChange, FrontierUpdate, HeaderSyncServiceSummary, ServiceAdmissionDecision, + ServicePeerDirection, ServicePeerSnapshot, ZakuraHeaderSyncCandidateState, }; +/// Upper bound on how long the reactor will wait to enqueue a data-plane action +/// before abandoning it. The bounded `actions` channel is normally drained by +/// the action driver almost immediately; this deadline only trips when that +/// driver is genuinely stalled on backend/verifier work, and it keeps a stalled +/// driver from wedging the reactor's control plane — peer-lifecycle draining, +/// request timeouts, and above all misbehavior disconnects. +const ACTION_SEND_TIMEOUT: Duration = Duration::from_secs(5); + /// Spawn a header-sync reactor and return its handle plus action stream. pub fn spawn_header_sync_reactor( startup: HeaderSyncStartup, @@ -15,7 +23,7 @@ pub fn spawn_header_sync_reactor( ), HeaderSyncStartError, > { - let state = HeaderSyncState::new(&startup)?; + let state = HeaderSyncCore::new(&startup)?; let (events_tx, events_rx) = mpsc::channel(128); let (lifecycle_tx, lifecycle_rx) = mpsc::unbounded_channel(); let (actions_tx, actions_rx) = mpsc::channel(128); @@ -52,7 +60,7 @@ pub fn spawn_header_sync_reactor( #[derive(Debug)] pub(super) struct HeaderSyncReactor { startup: HeaderSyncStartup, - state: HeaderSyncState, + state: HeaderSyncCore, events: mpsc::Receiver, lifecycle: mpsc::UnboundedReceiver, actions: mpsc::Sender, @@ -63,14 +71,14 @@ pub(super) struct HeaderSyncReactor { impl HeaderSyncReactor { async fn run(mut self) { + let mut frontier_updates = self.startup.frontier_updates.clone(); + let mut frontier_updates_open = frontier_updates.is_some(); if self.startup.range_state_actions_enabled { let _ = self - .actions - .send(HeaderSyncAction::QueryBestHeaderTip) + .dispatch_action(HeaderSyncAction::QueryBestHeaderTip) .await; let _ = self - .actions - .send(HeaderSyncAction::QueryMissingBlockBodies { + .dispatch_action(HeaderSyncAction::QueryMissingBlockBodies { from: next_height(self.state.verified_block_tip) .unwrap_or(self.state.verified_block_tip), limit: DEFAULT_HS_RANGE, @@ -97,6 +105,23 @@ impl HeaderSyncReactor { }; self.handle_event(event).await; } + changed = async { + match frontier_updates.as_mut() { + Some(frontier_updates) => frontier_updates.changed().await, + None => std::future::pending().await, + } + }, if frontier_updates_open => { + match changed { + Ok(()) => { + let frontier_updates = frontier_updates + .as_mut() + .expect("frontier update receiver exists while frontier_updates_open is true"); + let update = *frontier_updates.borrow_and_update(); + self.handle_frontier_update(update).await; + } + Err(_) => frontier_updates_open = false, + } + } _ = ticks.tick() => { self.handle_timeouts().await; } @@ -105,6 +130,7 @@ impl HeaderSyncReactor { } async fn handle_event(&mut self, event: HeaderSyncEvent) { + self.trace_event_received(&event); match event { HeaderSyncEvent::PeerConnected(session) => self.handle_peer_connected(session).await, HeaderSyncEvent::PeerDisconnected(peer) => self.handle_peer_disconnected(peer), @@ -180,11 +206,13 @@ impl HeaderSyncReactor { start_height, requested_count, headers, + body_sizes, } => self.handle_header_range_response_ready( peer, start_height, requested_count, headers, + body_sizes, ), } } @@ -212,6 +240,23 @@ impl HeaderSyncReactor { } } + async fn handle_frontier_update(&mut self, update: FrontierUpdate) { + match update.change { + FrontierChange::Snapshot + | FrontierChange::VerifiedGrow + | FrontierChange::VerifiedReset => { + let frontier = update.frontier; + self.handle_state_frontiers_changed(HeaderSyncFrontiers { + finalized_height: frontier.finalized.height, + verified_block_tip: frontier.verified_body.height, + verified_block_hash: frontier.verified_body.hash, + }) + .await; + } + FrontierChange::HeaderAdvanced | FrontierChange::HeaderReanchored => {} + } + } + fn admitted_count(&self, direction: ServicePeerDirection) -> usize { self.state .peers @@ -340,17 +385,34 @@ impl HeaderSyncReactor { } self.state.parked_peers.remove(&peer); + self.state.schedule.forget_peer(&peer); + let status_refresh_interval = self.startup.status_refresh_interval; self.state .peers .entry(peer.clone()) .and_modify(|peer_state| { peer_state.session = session.clone(); peer_state.direction = direction; + // A new transport replaces the old one; its remote has received + // no status yet, so the initial status below must always be sent. + // Outstanding requests and inbound serving counts are also + // session-local: responses for the old stream cannot satisfy + // work sent on this fresh stream. + peer_state.received_status = false; + peer_state.reset_sent_status(); + peer_state.outstanding.clear(); + peer_state.late_covered_responses = 0; + peer_state.served_headers_inflight = 0; + peer_state.meters = HeaderSyncPeerMeters::new( + status_refresh_interval, + DEFAULT_HS_INBOUND_STATUS_MIN_INTERVAL, + DEFAULT_HS_INBOUND_NEW_BLOCK_MIN_INTERVAL, + ); }) .or_insert_with(|| { PeerHeaderState::new( session, - self.state.anchor.0, + self.state.anchor, self.startup.config.advertised_max_headers_per_response(), self.startup.config.advertised_max_inflight_requests(), self.startup.status_refresh_interval, @@ -376,7 +438,7 @@ impl HeaderSyncReactor { async fn handle_full_block_committed(&mut self, height: block::Height, hash: block::Hash) { self.state.pending_new_blocks.remove(&hash); let _ = self.state.seen.insert(hash); - self.state.verified_block_tip = self.state.verified_block_tip.max(height); + self.update_verified_block_tip(height, hash); self.state.schedule.mark_height_covered(height); self.cancel_covered_outstanding(); if height > self.state.best_header_tip { @@ -400,7 +462,7 @@ impl HeaderSyncReactor { return; } - self.state.verified_block_tip = self.state.verified_block_tip.max(height); + self.update_verified_block_tip(height, hash); self.state.schedule.mark_height_covered(height); self.cancel_covered_outstanding(); if height > self.state.best_header_tip { @@ -496,6 +558,10 @@ impl HeaderSyncReactor { async fn handle_state_frontiers_changed(&mut self, frontiers: HeaderSyncFrontiers) { self.state.finalized_height = frontiers.finalized_height; self.state.verified_block_tip = frontiers.verified_block_tip; + self.state.verified_block_hash = frontiers.verified_block_hash; + if self.state.best_header_tip <= self.state.verified_block_tip { + self.state.stale_anchor.reset(); + } self.schedule().await; } @@ -581,12 +647,19 @@ impl HeaderSyncReactor { start_height: block::Height, requested_count: u32, headers: Vec>, + body_sizes: Vec, ) { let Some(peer_state) = self.state.peers.get_mut(&peer) else { return; }; + if validate_body_sizes_len(headers.len(), body_sizes.len()).is_err() { + peer_state.finish_serving_headers(); + return; + } let returned_count = u32::try_from(headers.len()).unwrap_or(u32::MAX); - let send_result = peer_state.session.try_send_headers(headers); + let send_result = peer_state + .session + .try_send_headers_with_sizes(headers, body_sizes); peer_state.finish_serving_headers(); match send_result { @@ -622,12 +695,16 @@ impl HeaderSyncReactor { let Some(peer_state) = self.state.peers.get_mut(&peer) else { return; }; - if !peer_state.inbound_status.try_take(Instant::now()) { + let advances_advertised_tip = status.tip_height > peer_state.advertised_tip; + let status_token_available = + peer_state.meters.inbound_status.try_take(Instant::now()); + if !advances_advertised_tip && !status_token_available { self.report_misbehavior(peer, HeaderSyncMisbehavior::StatusSpam) .await; return; } peer_state.advertised_tip = status.tip_height; + peer_state.advertised_hash = status.tip_hash; peer_state.anchor = status.anchor_height; peer_state.max_headers_per_response = clamp_advertised_range(status.max_headers_per_response); @@ -639,8 +716,11 @@ impl HeaderSyncReactor { self.trace_status_received(&peer, status); self.schedule().await; } - HeaderSyncMessage::Headers(headers) => { - self.handle_headers(peer, headers).await; + HeaderSyncMessage::Headers { + headers, + body_sizes, + } => { + self.handle_headers(peer, headers, body_sizes).await; } HeaderSyncMessage::GetHeaders { start_height, @@ -654,6 +734,22 @@ impl HeaderSyncReactor { } } + fn restore_outstanding_after_late_covered_response( + &mut self, + peer: &ZakuraPeerId, + outstanding: OutstandingRange, + ) -> bool { + let Some(peer_state) = self.state.peers.get_mut(peer) else { + return false; + }; + if !peer_state.take_late_covered_response() { + return false; + } + peer_state.restore_oldest_outstanding(outstanding); + metrics::counter!("sync.header.response.late_covered_dropped").increment(1); + true + } + async fn handle_get_headers( &mut self, peer: ZakuraPeerId, @@ -690,15 +786,13 @@ impl HeaderSyncReactor { return; } - if self - .actions - .send(HeaderSyncAction::QueryHeadersByHeightRange { + if !self + .dispatch_action(HeaderSyncAction::QueryHeadersByHeightRange { peer: peer.clone(), start: start_height, count, }) .await - .is_err() { if let Some(peer_state) = self.state.peers.get_mut(&peer) { peer_state.finish_serving_headers(); @@ -740,6 +834,7 @@ impl HeaderSyncReactor { .peers .get_mut(&peer) .expect("peer exists because it was checked before validation") + .meters .inbound_new_block .try_take(Instant::now()) { @@ -770,16 +865,14 @@ impl HeaderSyncReactor { let inserted = self.state.pending_new_blocks.insert(hash); debug_assert!(inserted, "pending acceptance was checked before insert"); - if self - .actions - .send(HeaderSyncAction::NewBlockReceived { + if !self + .dispatch_action(HeaderSyncAction::NewBlockReceived { peer, height, hash, block, }) .await - .is_err() { self.state.pending_new_blocks.remove(&hash); } @@ -804,7 +897,12 @@ impl HeaderSyncReactor { } #[tracing::instrument(skip(self, headers))] - async fn handle_headers(&mut self, peer: ZakuraPeerId, headers: Vec>) { + async fn handle_headers( + &mut self, + peer: ZakuraPeerId, + headers: Vec>, + body_sizes: Vec, + ) { metrics::counter!("sync.header.response.received").increment(1); let Some(peer_state) = self.state.peers.get_mut(&peer) else { self.report_misbehavior(peer, HeaderSyncMisbehavior::UnsolicitedHeaders) @@ -825,6 +923,7 @@ impl HeaderSyncReactor { self.handle_headers_for_outstanding( peer, headers, + body_sizes, outstanding, peer_max_headers_per_response, in_flight_count, @@ -836,10 +935,19 @@ impl HeaderSyncReactor { &mut self, peer: ZakuraPeerId, headers: Vec>, + body_sizes: Vec, outstanding: OutstandingRange, peer_max_headers_per_response: u32, in_flight_count: usize, ) { + if validate_body_sizes_len(headers.len(), body_sizes.len()).is_err() { + self.report_misbehavior(peer, HeaderSyncMisbehavior::MalformedMessage) + .await; + self.state.schedule.retry(outstanding.range); + self.schedule().await; + return; + } + if headers.is_empty() { self.record_advisory_unconfirmed(&peer); let deadline = Instant::now() + self.empty_headers_retry_delay(); @@ -892,17 +1000,55 @@ impl HeaderSyncReactor { outstanding.expected_max_count, ), }; - if validate_header_range_links(outstanding.range.anchor_hash, &headers).is_err() { + if let Err(error) = validate_header_range_links(outstanding.range.anchor_hash, &headers) { + debug!( + ?peer, + ?error, + anchor_hash = ?outstanding.range.anchor_hash, + start_height = ?outstanding.range.start_height, + count = ?header_count, + "Zakura header-sync rejected header range links" + ); + self.trace_range_validation_rejected( + &peer, + outstanding.range, + header_count, + "link", + header_sync_wire_error_kind(&error), + ); + if matches!(error, HeaderSyncWireError::FirstHeaderDoesNotLink) + && self.restore_outstanding_after_late_covered_response(&peer, outstanding) + { + return; + } + if self + .handle_possible_stale_anchor_link_failure(&peer, outstanding.range, &error) + .await + { + self.schedule().await; + return; + } self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) .await; self.state.schedule.retry(outstanding.range); self.schedule().await; return; } - if validate_headers_stateless(headers.clone(), validation_context) - .await - .is_err() - { + if let Err(error) = validate_headers_stateless(headers.clone(), validation_context).await { + debug!( + ?peer, + ?error, + start_height = ?outstanding.range.start_height, + count = ?header_count, + "Zakura header-sync rejected stateless header range" + ); + self.trace_range_validation_rejected( + &peer, + outstanding.range, + header_count, + "stateless", + header_sync_wire_error_kind(&error), + ); self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) .await; self.state.schedule.retry(outstanding.range); @@ -921,6 +1067,13 @@ impl HeaderSyncReactor { if end_height != outstanding.range.end_height() || self.startup.network.checkpoint_list().hash(end_height) != Some(last_hash) { + self.trace_range_validation_rejected( + &peer, + outstanding.range, + header_count, + "checkpoint", + "checkpoint_hash_mismatch", + ); self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) .await; self.state.schedule.retry(outstanding.range); @@ -938,17 +1091,59 @@ impl HeaderSyncReactor { outstanding.range, ); let _ = self - .actions - .send(HeaderSyncAction::CommitHeaderRange { + .dispatch_action(HeaderSyncAction::CommitHeaderRange { peer, anchor: outstanding.range.anchor_hash, start_height: outstanding.range.start_height, headers, + body_sizes, finalized: outstanding.range.finalized, }) .await; } + async fn handle_possible_stale_anchor_link_failure( + &mut self, + peer: &ZakuraPeerId, + range: RangeRequest, + error: &HeaderSyncWireError, + ) -> bool { + if !matches!(error, HeaderSyncWireError::FirstHeaderDoesNotLink) + || range.priority != RangePriority::Forward + || range.finalized + || self.state.best_header_tip <= self.state.verified_block_tip + { + self.state.stale_anchor.reset(); + return false; + } + + self.state.stale_anchor.record(peer.clone()); + metrics::counter!("sync.header.stale_anchor.link_failure").increment(1); + + if !self.state.stale_anchor.should_reanchor() { + self.state.schedule.clear_assignment(range); + self.state.schedule.retry(range); + return true; + } + + self.reanchor_to_verified_block_tip().await; + true + } + + async fn reanchor_to_verified_block_tip(&mut self) { + let height = self.state.verified_block_tip; + let hash = self.state.verified_block_hash; + metrics::counter!("sync.header.stale_anchor.reanchored").increment(1); + + self.state.stale_anchor.reset(); + self.state.schedule.clear_forward(); + self.state + .pending_commits + .retain(|_, range| range.priority != RangePriority::Forward); + self.cancel_forward_outstanding(); + self.publish_best_tip_reanchored(height, hash).await; + } + async fn handle_timeouts(&mut self) { let now = Instant::now(); let mut timed_out = Vec::new(); @@ -1059,15 +1254,27 @@ impl HeaderSyncReactor { } } - fn send_status(&self, peer: &ZakuraPeerId) { - let Some(peer_state) = self.state.peers.get(peer) else { - return; - }; - metrics::counter!("sync.header.peer.status.sent").increment(1); + fn send_status(&mut self, peer: &ZakuraPeerId) { let status = self.local_status(); + // Suppress a status identical to the last one we sent this peer over its + // current session: it advances nothing and the peer's inbound status + // rate limiter would treat the redundant message as spam. + match self.state.peers.get_mut(peer) { + Some(peer_state) if peer_state.status_differs_from_last_sent(status) => { + peer_state.record_sent_status(status); + } + Some(_) => { + metrics::counter!("sync.header.peer.status.suppressed_redundant").increment(1); + return; + } + None => return, + } + metrics::counter!("sync.header.peer.status.sent").increment(1); self.trace_status_sent(peer, status); - if let Err(error) = peer_state.session.try_send_status(status) { - tracing::debug!(?peer, ?error, "failed to queue Zakura header-sync Status"); + if let Some(peer_state) = self.state.peers.get(peer) { + if let Err(error) = peer_state.session.try_send_status(status) { + tracing::debug!(?peer, ?error, "failed to queue Zakura header-sync Status"); + } } #[cfg(test)] let _ = self.actions.try_send(HeaderSyncAction::SendMessage { @@ -1082,10 +1289,40 @@ impl HeaderSyncReactor { metrics::gauge!("sync.header.best_tip.height").set(height.0 as f64); self.trace_frontier_advanced(height, hash); let _ = self.tip.send((height, hash)); + let _ = self + .dispatch_action(HeaderSyncAction::HeaderAdvanced { height, hash }) + .await; + self.publish_candidate_state(); + self.broadcast_status_refresh().await; + } + + async fn publish_best_tip_reanchored(&mut self, height: block::Height, hash: block::Hash) { + let old = (self.state.best_header_tip, self.state.best_header_hash); + self.state.best_header_tip = height; + self.state.best_header_hash = hash; + metrics::gauge!("sync.header.best_tip.height").set(height.0 as f64); + self.trace_frontier_reanchored(height, hash); + let _ = self.tip.send((height, hash)); + let _ = self + .dispatch_action(HeaderSyncAction::HeaderReanchored { + old, + new: (height, hash), + }) + .await; self.publish_candidate_state(); self.broadcast_status_refresh().await; } + fn update_verified_block_tip(&mut self, height: block::Height, hash: block::Hash) { + if height > self.state.verified_block_tip { + self.state.verified_block_tip = height; + self.state.verified_block_hash = hash; + } + if self.state.best_header_tip <= self.state.verified_block_tip { + self.state.stale_anchor.reset(); + } + } + async fn broadcast_status_refresh(&mut self) { let now = Instant::now(); let status = self.local_status(); @@ -1093,7 +1330,20 @@ impl HeaderSyncReactor { .state .peers .iter_mut() - .filter_map(|(peer_id, peer)| peer.unsolicited.try_take(now).then(|| peer_id.clone())) + .filter_map(|(peer_id, peer)| { + // Never re-send a peer a status identical to its last one: the + // peer's inbound rate limiter would treat it as spam. A redundant + // refresh is dropped without spending the peer's status budget. + if !peer.status_differs_from_last_sent(status) { + metrics::counter!("sync.header.peer.status.suppressed_redundant").increment(1); + return None; + } + if !peer.meters.unsolicited.try_take(now) { + return None; + } + peer.record_sent_status(status); + Some(peer_id.clone()) + }) .collect(); for peer in peer_ids { @@ -1123,8 +1373,7 @@ impl HeaderSyncReactor { .set(count_between(from, self.state.best_header_tip) as f64); self.trace_missing_bodies(from, self.state.best_header_tip); let _ = self - .actions - .send(HeaderSyncAction::BodyGaps { + .dispatch_action(HeaderSyncAction::BodyGaps { from, to: self.state.best_header_tip, }) @@ -1132,17 +1381,258 @@ impl HeaderSyncReactor { } } + /// Hand a data-plane action to the action driver without letting a slow or + /// stalled driver wedge the reactor. A full channel is awaited only up to + /// [`ACTION_SEND_TIMEOUT`]; past that the action is dropped so the reactor + /// keeps draining peer-lifecycle events, request timeouts, and misbehavior + /// disconnects. Returns `true` only if the action was accepted. + async fn dispatch_action(&self, action: HeaderSyncAction) -> bool { + self.trace_action_dispatched(&action); + match time::timeout(ACTION_SEND_TIMEOUT, self.actions.send(action)).await { + Ok(Ok(())) => true, + // Receiver dropped: the driver is gone, treat like a send failure. + Ok(Err(_)) => false, + // Driver stalled past the deadline: drop the action and stay live. + Err(_) => { + metrics::counter!("sync.header.action.send_timeout").increment(1); + false + } + } + } + async fn report_misbehavior(&mut self, peer: ZakuraPeerId, reason: HeaderSyncMisbehavior) { if let Some(peer_state) = self.state.peers.get_mut(&peer) { peer_state.misbehavior = peer_state.misbehavior.saturating_add(1); + // Prioritized, non-blocking disconnect: tear down the offending + // peer's header-sync session locally so a saturated or stalled + // action channel can never delay it. The supervised session + // teardown also emits a `PeerDisconnected` lifecycle event that + // cleans up this reactor's per-peer state. + peer_state.session.cancel_token().cancel(); } metrics::counter!("sync.header.peer.disconnect").increment(1); self.trace_peer_violation(&peer, reason); self.trace_peer_disconnect_requested(&peer, reason); - let _ = self - .actions - .send(HeaderSyncAction::Misbehavior { peer, reason }) - .await; + // Best-effort supervisor notification for cross-service scoring/full + // disconnect. Never block the reactor waiting for channel capacity: the + // local session cancel above already removed the peer from this service. + let action = HeaderSyncAction::Misbehavior { peer, reason }; + self.trace_action_dispatched(&action); + if self.actions.try_send(action).is_err() { + metrics::counter!("sync.header.peer.disconnect.action_dropped").increment(1); + } + } + + fn trace_event_received(&self, event: &HeaderSyncEvent) { + self.emit_trace(hs_trace::HEADER_EVENT_RECEIVED, |row| match event { + HeaderSyncEvent::PeerConnected(session) => { + insert_optional_str(row, hs_trace::KIND, Some("peer_connected")); + insert_peer(row, hs_trace::PEER, session.peer_id()); + } + HeaderSyncEvent::PeerDisconnected(peer) => { + insert_optional_str(row, hs_trace::KIND, Some("peer_disconnected")); + insert_peer(row, hs_trace::PEER, peer); + } + HeaderSyncEvent::AdvisoryHeaderSummary { peer, summary } => { + insert_optional_str(row, hs_trace::KIND, Some("advisory_header_summary")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, summary.best_height); + } + HeaderSyncEvent::FullBlockCommitted { height, hash, .. } => { + insert_optional_str(row, hs_trace::KIND, Some("full_block_committed")); + insert_height(row, hs_trace::HEIGHT, *height); + insert_hash(row, hs_trace::HASH, *hash); + } + HeaderSyncEvent::NewBlockAccepted { + peer, height, hash, .. + } => { + insert_optional_str(row, hs_trace::KIND, Some("new_block_accepted")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, *height); + insert_hash(row, hs_trace::HASH, *hash); + } + HeaderSyncEvent::NewBlockDuplicate { peer, height, hash } => { + insert_optional_str(row, hs_trace::KIND, Some("new_block_duplicate")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, *height); + insert_hash(row, hs_trace::HASH, *hash); + } + HeaderSyncEvent::NewBlockRejected { peer, hash } => { + insert_optional_str(row, hs_trace::KIND, Some("new_block_rejected")); + insert_peer(row, hs_trace::PEER, peer); + insert_hash(row, hs_trace::HASH, *hash); + } + HeaderSyncEvent::WireMessage { peer, msg } => { + insert_optional_str(row, hs_trace::KIND, Some("wire_message")); + insert_optional_str(row, hs_trace::REASON, Some(header_sync_message_label(msg))); + insert_peer(row, hs_trace::PEER, peer); + trace_header_sync_message_fields(row, msg); + } + HeaderSyncEvent::WireDecodeFailed { peer, .. } => { + insert_optional_str(row, hs_trace::KIND, Some("wire_decode_failed")); + insert_peer(row, hs_trace::PEER, peer); + } + HeaderSyncEvent::WireProtocolFailure { peer, reason, .. } => { + insert_optional_str(row, hs_trace::KIND, Some("wire_protocol_failure")); + insert_optional_str( + row, + hs_trace::REASON, + Some(misbehavior_reason_label(*reason)), + ); + insert_peer(row, hs_trace::PEER, peer); + } + HeaderSyncEvent::StateFrontiersChanged(frontiers) => { + insert_optional_str(row, hs_trace::KIND, Some("state_frontiers_changed")); + insert_height(row, "finalized_height", frontiers.finalized_height); + insert_height(row, "verified_block_tip", frontiers.verified_block_tip); + } + HeaderSyncEvent::HeaderRangeCommitted { + start_height, + tip_height, + tip_hash, + } => { + insert_optional_str(row, hs_trace::KIND, Some("header_range_committed")); + insert_height(row, hs_trace::RANGE_START, *start_height); + insert_u64( + row, + hs_trace::RANGE_COUNT, + u64::from(count_between(*start_height, *tip_height)), + ); + insert_height(row, hs_trace::HEIGHT, *tip_height); + insert_hash(row, hs_trace::HASH, *tip_hash); + } + HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind, + } => { + insert_optional_str(row, hs_trace::KIND, Some("header_range_commit_failed")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, *start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(*count)); + insert_optional_str( + row, + hs_trace::REASON, + Some(commit_failure_reason_label(*kind)), + ); + } + HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height, + requested_count, + returned_count, + } => { + insert_optional_str(row, hs_trace::KIND, Some("header_range_response_finished")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, *start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(*returned_count)); + insert_u64(row, hs_trace::EXPECTED_COUNT, u64::from(*requested_count)); + } + HeaderSyncEvent::HeaderRangeResponseReady { + peer, + start_height, + requested_count, + headers, + .. + } => { + insert_optional_str(row, hs_trace::KIND, Some("header_range_response_ready")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, *start_height); + insert_u64(row, hs_trace::RANGE_COUNT, headers.len() as u64); + insert_u64(row, hs_trace::EXPECTED_COUNT, u64::from(*requested_count)); + } + }); + } + + fn trace_action_dispatched(&self, action: &HeaderSyncAction) { + self.emit_trace(hs_trace::HEADER_ACTION_DISPATCHED, |row| match action { + #[cfg(test)] + HeaderSyncAction::SendMessage { peer, msg } => { + insert_optional_str(row, hs_trace::KIND, Some("send_message")); + insert_optional_str(row, hs_trace::REASON, Some(header_sync_message_label(msg))); + insert_peer(row, hs_trace::PEER, peer); + trace_header_sync_message_fields(row, msg); + } + #[cfg(test)] + HeaderSyncAction::ForwardNewBlock { + source, + peer, + height, + hash, + .. + } => { + insert_optional_str(row, hs_trace::KIND, Some("forward_new_block")); + if let Some(source) = source { + insert_peer(row, hs_trace::SOURCE_PEER, source); + } + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, *height); + insert_hash(row, hs_trace::HASH, *hash); + } + HeaderSyncAction::Misbehavior { peer, reason } => { + insert_optional_str(row, hs_trace::KIND, Some("misbehavior")); + insert_peer(row, hs_trace::PEER, peer); + insert_optional_str( + row, + hs_trace::REASON, + Some(misbehavior_reason_label(*reason)), + ); + } + HeaderSyncAction::NewBlockReceived { + peer, height, hash, .. + } => { + insert_optional_str(row, hs_trace::KIND, Some("new_block_received")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, *height); + insert_hash(row, hs_trace::HASH, *hash); + } + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + insert_optional_str(row, hs_trace::KIND, Some("query_headers_by_height_range")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, *start); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(*count)); + } + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + .. + } => { + insert_optional_str(row, hs_trace::KIND, Some("commit_header_range")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, *start_height); + insert_u64(row, hs_trace::RANGE_COUNT, headers.len() as u64); + } + HeaderSyncAction::QueryBestHeaderTip => { + insert_optional_str(row, hs_trace::KIND, Some("query_best_header_tip")); + } + HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { + insert_optional_str(row, hs_trace::KIND, Some("query_missing_block_bodies")); + insert_height(row, hs_trace::RANGE_START, *from); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(*limit)); + } + HeaderSyncAction::BodyGaps { from, to } => { + insert_optional_str(row, hs_trace::KIND, Some("body_gaps")); + insert_height(row, hs_trace::RANGE_START, *from); + insert_u64( + row, + hs_trace::RANGE_COUNT, + u64::from(count_between(*from, *to)), + ); + } + HeaderSyncAction::HeaderAdvanced { height, hash } => { + insert_optional_str(row, hs_trace::KIND, Some("header_advanced")); + insert_height(row, hs_trace::HEIGHT, *height); + insert_hash(row, hs_trace::HASH, *hash); + } + HeaderSyncAction::HeaderReanchored { old, new } => { + insert_optional_str(row, hs_trace::KIND, Some("header_reanchored")); + insert_height(row, hs_trace::HEIGHT, new.0); + insert_hash(row, hs_trace::HASH, new.1); + insert_height(row, hs_trace::RANGE_START, old.0); + } + }); } fn trace_status_sent(&self, peer: &ZakuraPeerId, status: HeaderSyncStatus) { @@ -1250,6 +1740,31 @@ impl HeaderSyncReactor { }); } + fn trace_range_validation_rejected( + &self, + peer: &ZakuraPeerId, + range: RangeRequest, + count: u32, + validation_stage: &'static str, + error_kind: &'static str, + ) { + self.emit_trace(hs_trace::HEADER_RANGE_REJECTED, |row| { + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::RANGE_START, range.start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(count)); + insert_hash(row, hs_trace::ANCHOR_HASH, range.anchor_hash); + insert_optional_str(row, hs_trace::VALIDATION_STAGE, Some(validation_stage)); + insert_optional_str(row, hs_trace::ERROR_KIND, Some(error_kind)); + insert_optional_str( + row, + hs_trace::REASON, + Some(misbehavior_reason_label( + HeaderSyncMisbehavior::InvalidRange, + )), + ); + }); + } + fn trace_new_block_received( &self, peer: &ZakuraPeerId, @@ -1328,6 +1843,13 @@ impl HeaderSyncReactor { }); } + fn trace_frontier_reanchored(&self, height: block::Height, hash: block::Hash) { + self.emit_trace(hs_trace::HEADER_FRONTIER_REANCHORED, |row| { + insert_height(row, hs_trace::HEIGHT, height); + insert_hash(row, hs_trace::HASH, hash); + }); + } + fn trace_missing_bodies(&self, from: block::Height, to: block::Height) { self.emit_trace(hs_trace::HEADER_MISSING_BODIES_REPORTED, |row| { insert_height(row, hs_trace::RANGE_START, from); @@ -1380,6 +1902,47 @@ impl HeaderSyncReactor { } } } + + fn cancel_forward_outstanding(&mut self) { + for peer in self.state.peers.values_mut() { + let mut index = 0; + while index < peer.outstanding.len() { + if peer.outstanding[index].range.priority == RangePriority::Forward { + peer.outstanding.remove(index); + peer.late_covered_responses = peer.late_covered_responses.saturating_add(1); + } else { + index += 1; + } + } + } + } +} + +fn header_sync_wire_error_kind(error: &HeaderSyncWireError) -> &'static str { + match error { + HeaderSyncWireError::OversizedPayload { .. } => "oversized_payload", + HeaderSyncWireError::HeaderCountLimit { .. } => "header_count_limit", + HeaderSyncWireError::BodySizeCountMismatch { .. } => "body_size_count_mismatch", + HeaderSyncWireError::UnsolicitedHeaders => "unsolicited_headers", + HeaderSyncWireError::ZeroHeaderRequestCount => "zero_header_request_count", + HeaderSyncWireError::HeightOutOfRange(_) => "height_out_of_range", + HeaderSyncWireError::UnknownMessageType(_) => "unknown_message_type", + HeaderSyncWireError::UnknownFrameMessageType(_) => "unknown_frame_message_type", + HeaderSyncWireError::UnsupportedFlags(_) => "unsupported_flags", + HeaderSyncWireError::MismatchedFrameMessageType { .. } => "mismatched_frame_message_type", + HeaderSyncWireError::TrailingBytes => "trailing_bytes", + HeaderSyncWireError::NonContiguousHeaders => "non_contiguous_headers", + HeaderSyncWireError::FirstHeaderDoesNotLink => "first_header_does_not_link", + HeaderSyncWireError::WrongEquihashSolutionSize => "wrong_equihash_solution_size", + HeaderSyncWireError::InvalidDifficultyThreshold => "invalid_difficulty_threshold", + HeaderSyncWireError::DifficultyFilter { .. } => "difficulty_filter", + HeaderSyncWireError::NumericOverflow(_) => "numeric_overflow", + HeaderSyncWireError::Io(_) => "io", + HeaderSyncWireError::Serialization(_) => "serialization", + HeaderSyncWireError::Time(_) => "time", + HeaderSyncWireError::Equihash(_) => "equihash", + HeaderSyncWireError::BlockingTask(_) => "blocking_task", + } } fn header_sync_candidate_target(best_header_tip: block::Height) -> block::Height { @@ -1399,3 +1962,51 @@ fn node_id_from_header_peer_id(peer: &ZakuraPeerId) -> Option { let bytes: [u8; 32] = peer.as_bytes().try_into().ok()?; NodeId::from_bytes(&bytes).ok() } + +fn trace_header_sync_message_fields( + row: &mut serde_json::Map, + msg: &HeaderSyncMessage, +) { + match msg { + HeaderSyncMessage::Status(status) => { + insert_height(row, hs_trace::HEIGHT, status.tip_height); + insert_hash(row, hs_trace::HASH, status.tip_hash); + insert_height(row, hs_trace::RANGE_START, status.anchor_height); + insert_u64( + row, + hs_trace::ADVERTISED_CAP, + u64::from(status.max_headers_per_response), + ); + insert_u64( + row, + hs_trace::IN_FLIGHT_COUNT, + u64::from(status.max_inflight_requests), + ); + } + HeaderSyncMessage::Headers { headers, .. } => { + insert_u64(row, hs_trace::RANGE_COUNT, headers.len() as u64); + } + HeaderSyncMessage::GetHeaders { + start_height, + count, + } => { + insert_height(row, hs_trace::RANGE_START, *start_height); + insert_u64(row, hs_trace::RANGE_COUNT, u64::from(*count)); + } + HeaderSyncMessage::NewBlock(block) => { + insert_hash(row, hs_trace::HASH, block.hash()); + if let Some(height) = block.coinbase_height() { + insert_height(row, hs_trace::HEIGHT, height); + } + } + } +} + +fn header_sync_message_label(msg: &HeaderSyncMessage) -> &'static str { + match msg { + HeaderSyncMessage::Status(_) => "status", + HeaderSyncMessage::Headers { .. } => "headers", + HeaderSyncMessage::GetHeaders { .. } => "get_headers", + HeaderSyncMessage::NewBlock(_) => "new_block", + } +} diff --git a/zebra-network/src/zakura/header_sync/scheduler.rs b/zebra-network/src/zakura/header_sync/scheduler.rs index fd6e4e47d7c..91dd49b1e56 100644 --- a/zebra-network/src/zakura/header_sync/scheduler.rs +++ b/zebra-network/src/zakura/header_sync/scheduler.rs @@ -155,12 +155,19 @@ impl RangeScheduler { for peers in self.assigned.values_mut() { peers.remove(peer); } + self.assigned.retain(|_, peers| !peers.is_empty()); } pub(super) fn clear_assignment(&mut self, range: RangeRequest) { self.assigned.remove(&range); } + pub(super) fn clear_forward(&mut self) { + self.forward.clear(); + self.assigned + .retain(|range, _| range.priority != RangePriority::Forward); + } + pub(super) fn mark_height_covered(&mut self, height: block::Height) { self.mark_covered_interval(CoveredRange { start: height, diff --git a/zebra-network/src/zakura/header_sync/service.rs b/zebra-network/src/zakura/header_sync/service.rs index 1501e0d42e2..2acf9d7f76b 100644 --- a/zebra-network/src/zakura/header_sync/service.rs +++ b/zebra-network/src/zakura/header_sync/service.rs @@ -1,16 +1,14 @@ -use std::{ - collections::VecDeque, - sync::{Arc, Mutex as StdMutex}, -}; +use std::sync::Arc; use tokio::{sync::mpsc, task}; use tokio_util::sync::CancellationToken; -use super::{events::*, validation::*, wire::*, *}; +use super::{events::*, pipe::*, wire::*, *}; use crate::zakura::{ - BoxRunFuture, Frame, FramedRecv, FramedSend, OrderedSendError, Peer, PeerStreamSession, - Service, ServicePeerDirection, Sink, SinkReject, Stream, StreamMode, ZakuraPeerId, - ZakuraSupervisorHandle, ZAKURA_CAP_HEADER_SYNC, + handle_pipe_exit, spawn_supervised_pipe, BoxRunFuture, Flow, Frame, FramedRecv, FramedSend, + OrderedSendError, Peer, PeerStreamSession, Pipe, Service, ServicePeerDirection, SessionGuard, + Sink, SinkReject, Stream, StreamMode, ZakuraPeerId, ZakuraSupervisorHandle, + ZAKURA_CAP_HEADER_SYNC, }; const HEADER_SYNC_SERVICE_STREAMS: [Stream; 1] = [Stream { @@ -42,16 +40,21 @@ pub struct HeaderSyncPeerSession { struct HeaderSyncPeerSessionInner { send: FramedSend, cancel_token: CancellationToken, - expected_headers: StdMutex>, + commands: Option>, } impl HeaderSyncPeerSession { - pub(crate) fn new(session: &PeerStreamSession, direction: ServicePeerDirection) -> Self { - Self::from_parts_with_direction( + fn new_with_commands( + session: &PeerStreamSession, + direction: ServicePeerDirection, + commands: mpsc::UnboundedSender, + ) -> Self { + Self::from_parts_with_direction_and_commands( session.peer_id().clone(), direction, session.sender(), session.cancel_token(), + Some(commands), ) } @@ -70,6 +73,17 @@ impl HeaderSyncPeerSession { direction: ServicePeerDirection, send: FramedSend, cancel_token: CancellationToken, + ) -> Self { + Self::from_parts_with_direction_and_commands(peer_id, direction, send, cancel_token, None) + } + + #[cfg(test)] + fn from_parts_with_direction_and_commands( + peer_id: ZakuraPeerId, + direction: ServicePeerDirection, + send: FramedSend, + cancel_token: CancellationToken, + commands: Option>, ) -> Self { Self { peer_id, @@ -77,17 +91,18 @@ impl HeaderSyncPeerSession { inner: Arc::new(HeaderSyncPeerSessionInner { send, cancel_token, - expected_headers: StdMutex::new(VecDeque::new()), + commands, }), } } #[cfg(not(test))] - fn from_parts_with_direction( + fn from_parts_with_direction_and_commands( peer_id: ZakuraPeerId, direction: ServicePeerDirection, send: FramedSend, cancel_token: CancellationToken, + commands: Option>, ) -> Self { Self { peer_id, @@ -95,7 +110,7 @@ impl HeaderSyncPeerSession { inner: Arc::new(HeaderSyncPeerSessionInner { send, cancel_token, - expected_headers: StdMutex::new(VecDeque::new()), + commands, }), } } @@ -128,17 +143,20 @@ impl HeaderSyncPeerSession { ) -> Result<(), OrderedSendError> { let expected = ExpectedHeadersResponse::new(start_height, count) .map_err(|error| OrderedSendError::Encode(Box::new(error)))?; - let mut expected_headers = self - .inner - .expected_headers - .lock() - .map_err(|_| OrderedSendError::Closed)?; + if let Some(commands) = &self.inner.commands { + self.try_send_message(HeaderSyncMessage::GetHeaders { + start_height, + count, + })?; + return commands + .send(HeaderSyncPeerCommand::RecordExpectedHeaders(expected)) + .map_err(|_| OrderedSendError::Closed); + } + self.try_send_message(HeaderSyncMessage::GetHeaders { start_height, count, - })?; - expected_headers.push_back(expected); - Ok(()) + }) } /// Send a typed header range response. @@ -146,7 +164,20 @@ impl HeaderSyncPeerSession { &self, headers: Vec>, ) -> Result<(), OrderedSendError> { - self.try_send_message(HeaderSyncMessage::Headers(headers)) + let body_sizes = vec![0; headers.len()]; + self.try_send_headers_with_sizes(headers, body_sizes) + } + + /// Send a typed header range response with one advisory body-size hint per header. + pub fn try_send_headers_with_sizes( + &self, + headers: Vec>, + body_sizes: Vec, + ) -> Result<(), OrderedSendError> { + self.try_send_message(HeaderSyncMessage::Headers { + headers, + body_sizes, + }) } /// Send a typed full tip block announcement. @@ -164,14 +195,13 @@ impl HeaderSyncPeerSession { Err(mpsc::error::TrySendError::Closed(_frame)) => Err(OrderedSendError::Closed), } } +} - fn pop_expected_headers_response(&self) -> Option { - self.inner - .expected_headers - .lock() - .expect("header-sync expected-response mutex is never poisoned") - .pop_front() - } +/// Commands from shared scheduling state into one peer-owned header-sync pipe. +#[derive(Debug)] +pub(super) enum HeaderSyncPeerCommand { + /// Record an expected `Headers` response after `GetHeaders` was queued. + RecordExpectedHeaders(ExpectedHeadersResponse), } /// Pump actor actions that can be satisfied at the transport/service seam. @@ -240,7 +270,9 @@ pub(crate) async fn drive_header_sync_actions( } HeaderSyncAction::QueryBestHeaderTip | HeaderSyncAction::QueryMissingBlockBodies { .. } - | HeaderSyncAction::BodyGaps { .. } => {} + | HeaderSyncAction::BodyGaps { .. } + | HeaderSyncAction::HeaderAdvanced { .. } + | HeaderSyncAction::HeaderReanchored { .. } => {} } } } @@ -294,22 +326,68 @@ impl Service for HeaderSyncService { send, peer.service_cancel_token(), ); + // The sink loop parks on the service token (a child of the connection + // token) exactly as the old `HeaderSyncSink::run` select did. The + // connection token is cancelled only on a protocol reject below, never on + // a normal/parked exit — parking one service must not tear down the + // shared connection that other services (discovery, block-sync) ride on. let service_cancel_token = session.cancel_token(); let connection_cancel_token = peer.cancel_token(); - let header_sync_session = HeaderSyncPeerSession::new(&session, peer.direction); + let (commands_tx, commands_rx) = mpsc::unbounded_channel(); + let header_sync_session = + HeaderSyncPeerSession::new_with_commands(&session, peer.direction, commands_tx); let _ = self .header_sync .send_lifecycle(HeaderSyncEvent::PeerConnected(header_sync_session.clone())); - spawn_header_sync_sink( - peer_id, - session, - header_sync_session, - self.header_sync.clone(), - service_cancel_token, - connection_cancel_token, + let (_session_peer, _stream_kind, recv, _send, _session_cancel) = session.into_parts(); + + // Phase 2 keeps request/response correlation in `HsLocal`: after the + // session queues an outbound `GetHeaders`, the peer-owned pipe records + // the expected `Headers` response in plain local state. + let pipe = Pipe::new( + peer_id.clone(), + HsLocal::new(commands_rx, DEFAULT_HS_INBOUND_NEW_BLOCK_MIN_INTERVAL), + HsEnv::new(self.header_sync.clone()), + SessionGuard::oversize_only(header_sync_guard_max_bytes()), + run_inbound, + &PIPE_SHAPE, ); + // The pipe future reproduces the old sink's connection handling: a + // protocol reject (the only way `run_peer` returns `Err`, since + // `run_inbound` maps a closed-queue `Local` to a benign continue) + // cancels the *connection*, matching the old + // `connection_cancel_token.cancel()` on `SinkReject::Protocol`. A normal + // or parked exit leaves the connection alone. + let pipe_cancel_token = service_cancel_token.clone(); + let protocol_connection_cancel_token = connection_cancel_token.clone(); + let pipe = async move { + handle_pipe_exit( + "header-sync", + &protocol_connection_cancel_token, + run_peer(pipe, recv, pipe_cancel_token).await, + ); + }; + + // The supervised teardown runs on every exit path — normal return, + // protocol reject, or panic. It cancels this peer's *service* token + // (idempotent; already cancelled on a park/protocol exit) and sends + // `PeerDisconnected`. Sending it from teardown is the latent-bug fix: the + // old sink only sent `PeerDisconnected` on the normal return path, so a + // panicking task leaked the peer's reactor state. + let teardown_handle = self.header_sync.clone(); + let teardown_peer = peer_id.clone(); + let on_teardown = move || { + let _ = + teardown_handle.send_lifecycle(HeaderSyncEvent::PeerDisconnected(teardown_peer)); + }; + let panic_connection_cancel_token = connection_cancel_token.clone(); + let on_panic = move || panic_connection_cancel_token.cancel(); + + // Reuse the single supervised launcher; let the returned handle drop to + // detach the task (the `PipeTeardown` still runs on every exit path). + spawn_supervised_pipe(peer_id, service_cancel_token, on_teardown, on_panic, pipe); } fn remove_peer(&self, peer: &ZakuraPeerId) { @@ -328,10 +406,30 @@ impl Service for HeaderSyncService { return Ok(()); } - deliver_header_sync_frame(&self.header_sync, None, peer_id, frame) + // The test/recorder path has no peer session, so a `Headers` response + // with no outstanding request is rejected as `UnsolicitedHeaders`. A + // `Local` reject (closed reactor queue) is surfaced to the registry + // exactly as the old `deliver_header_sync_frame` returned it. + match deliver(&self.header_sync, None, peer_id, frame) { + Flow::Continue(()) | Flow::Done => Ok(()), + Flow::Reject(reject) => Err(reject), + } } } +/// Service-level oversize cap for the header-sync guard. +/// +/// Matches the decode stage's `MAX_HS_MESSAGE_BYTES` threshold so the guard +/// rejects nothing the decode stage would have admitted; the transport already +/// caps frames at this payload size before they reach the service, so this is a +/// defense-in-depth bound that never changes which events fire. +fn header_sync_guard_max_bytes() -> u32 { + // `MAX_HS_MESSAGE_BYTES` is a 2 MiB protocol constant that fits in `u32`; + // the `const` assertion in `wire.rs` keeps it below the local message cap. + u32::try_from(MAX_HS_MESSAGE_BYTES) + .expect("MAX_HS_MESSAGE_BYTES is a 2 MiB constant that fits in u32") +} + /// Testkit/no-reactor mode records stream-5 inbound frames without running header sync. #[derive(Debug)] pub(crate) struct HeaderSyncPassthroughService { @@ -411,91 +509,6 @@ impl Service for HeaderSyncPassthroughService { } } -fn spawn_header_sync_sink( - peer_id: ZakuraPeerId, - session: PeerStreamSession, - header_sync_session: HeaderSyncPeerSession, - header_sync: HeaderSyncHandle, - service_cancel_token: CancellationToken, - connection_cancel_token: CancellationToken, -) { - task::spawn(async move { - let (_session_peer, _stream_kind, recv, _send, _session_cancel) = session.into_parts(); - let sink = Box::new(HeaderSyncSink { - peer_id: peer_id.clone(), - header_sync: header_sync.clone(), - session: header_sync_session, - cancel_token: service_cancel_token.clone(), - }); - - let result = sink.run(recv).await; - - match result { - Ok(()) => {} - Err(SinkReject::Protocol(error)) => { - tracing::debug!( - ?error, - ?peer_id, - "header-sync stream rejected protocol-invalid frame" - ); - connection_cancel_token.cancel(); - } - Err(SinkReject::Local(error)) => { - tracing::debug!( - ?error, - ?peer_id, - "header-sync stream could not deliver frame locally" - ); - } - } - - let _ = header_sync.send_lifecycle(HeaderSyncEvent::PeerDisconnected(peer_id)); - }); -} - -#[derive(Debug)] -struct HeaderSyncSink { - peer_id: ZakuraPeerId, - header_sync: HeaderSyncHandle, - session: HeaderSyncPeerSession, - cancel_token: CancellationToken, -} - -impl Sink for HeaderSyncSink { - fn run(self: Box, mut recv: FramedRecv) -> BoxRunFuture<'static, Result<(), SinkReject>> { - Box::pin(async move { - loop { - let frame = tokio::select! { - _ = self.cancel_token.cancelled() => return Ok(()), - frame = recv.recv() => { - let Some(frame) = frame else { - return Ok(()); - }; - frame - } - }; - - match deliver_header_sync_frame( - &self.header_sync, - Some(&self.session), - self.peer_id.clone(), - frame, - ) { - Ok(()) => {} - Err(SinkReject::Protocol(error)) => return Err(SinkReject::Protocol(error)), - Err(SinkReject::Local(error)) => { - tracing::debug!( - ?error, - peer_id = ?self.peer_id, - "header-sync stream could not deliver frame locally" - ); - } - } - } - }) - } -} - #[derive(Debug)] struct HeaderSyncPassthroughSink { peer_id: ZakuraPeerId, @@ -536,71 +549,3 @@ impl Sink for HeaderSyncPassthroughSink { }) } } - -fn deliver_header_sync_frame( - header_sync: &HeaderSyncHandle, - session: Option<&HeaderSyncPeerSession>, - peer_id: ZakuraPeerId, - frame: Frame, -) -> Result<(), SinkReject> { - if u8::try_from(frame.message_type).ok() == Some(MSG_HS_HEADERS) { - let Some(expected) = session.and_then(HeaderSyncPeerSession::pop_expected_headers_response) - else { - let error = Arc::new(HeaderSyncWireError::UnsolicitedHeaders); - let _ = header_sync.try_send(HeaderSyncEvent::WireProtocolFailure { - peer: peer_id.clone(), - reason: HeaderSyncMisbehavior::UnsolicitedHeaders, - error: error.clone(), - }); - let protocol_error = - std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); - return Err(SinkReject::protocol(protocol_error)); - }; - - let msg = match HeaderSyncMessage::decode_frame( - frame, - HeaderSyncDecodeContext::for_headers_response(expected, expected.count), - ) { - Ok(msg) => msg, - Err(error) => { - let protocol_error = - std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); - let _ = header_sync.try_send(HeaderSyncEvent::WireProtocolFailure { - peer: peer_id.clone(), - reason: HeaderSyncMisbehavior::MalformedMessage, - error: Arc::new(error), - }); - return Err(SinkReject::protocol(protocol_error)); - } - }; - - return header_sync - .try_send(HeaderSyncEvent::WireMessage { peer: peer_id, msg }) - .map_err(|error| SinkReject::local(format!("header-sync queue closed: {error}"))); - } - - let msg = match decode_header_sync_frame(frame) { - Ok(msg) => msg, - Err(error) => { - let protocol_error = - std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()); - let _ = header_sync.try_send(HeaderSyncEvent::WireDecodeFailed { - peer: peer_id, - error: Arc::new(error), - }); - return Err(SinkReject::protocol(protocol_error)); - } - }; - - header_sync - .try_send(HeaderSyncEvent::WireMessage { peer: peer_id, msg }) - .map_err(|error| SinkReject::local(format!("header-sync queue closed: {error}"))) -} - -fn decode_header_sync_frame(frame: Frame) -> Result { - if u8::try_from(frame.message_type).ok() == Some(MSG_HS_HEADERS) { - return Err(HeaderSyncWireError::UnsolicitedHeaders); - } - - HeaderSyncMessage::decode_frame(frame, HeaderSyncDecodeContext::control()) -} diff --git a/zebra-network/src/zakura/header_sync/state.rs b/zebra-network/src/zakura/header_sync/state.rs index f06e6257e50..79061b8ec11 100644 --- a/zebra-network/src/zakura/header_sync/state.rs +++ b/zebra-network/src/zakura/header_sync/state.rs @@ -6,12 +6,15 @@ use crate::zakura::{ pub(super) const HEADER_SYNC_ADVISORY_BACKOFF_FAILURES: u32 = 2; pub(super) const HEADER_SYNC_ADVISORY_BACKOFF: Duration = Duration::from_secs(60); pub(super) const HEADER_SYNC_ADVISORY_TTL: Duration = DEFAULT_LIVE_SERVICE_SUMMARY_TTL; +pub(super) const HEADER_SYNC_STALE_ANCHOR_LINK_FAILURES: u32 = 3; +pub(super) const HEADER_SYNC_STALE_ANCHOR_DISTINCT_PEERS: usize = 2; #[derive(Clone, Debug)] -pub(super) struct HeaderSyncState { +pub(super) struct HeaderSyncCore { pub(super) anchor: (block::Height, block::Hash), pub(super) finalized_height: block::Height, pub(super) verified_block_tip: block::Height, + pub(super) verified_block_hash: block::Hash, pub(super) best_header_tip: block::Height, pub(super) best_header_hash: block::Hash, pub(super) peers: HashMap, @@ -21,9 +24,10 @@ pub(super) struct HeaderSyncState { pub(super) schedule: RangeScheduler, pub(super) pending_commits: HashMap, pub(super) advisory: HashMap, + pub(super) stale_anchor: StaleAnchorFailures, } -impl HeaderSyncState { +impl HeaderSyncCore { pub(super) fn new(startup: &HeaderSyncStartup) -> Result { validate_anchor(&startup.network, startup.anchor)?; let (best_header_tip, best_header_hash) = startup.best_header_tip.unwrap_or(startup.anchor); @@ -32,6 +36,7 @@ impl HeaderSyncState { anchor: startup.anchor, finalized_height: startup.frontiers.finalized_height, verified_block_tip: startup.frontiers.verified_block_tip, + verified_block_hash: startup.frontiers.verified_block_hash, best_header_tip, best_header_hash, peers: HashMap::new(), @@ -41,6 +46,7 @@ impl HeaderSyncState { schedule: RangeScheduler::new(), pending_commits: HashMap::new(), advisory: HashMap::new(), + stale_anchor: StaleAnchorFailures::default(), }) } @@ -115,6 +121,29 @@ impl HeaderSyncState { } } +#[derive(Clone, Debug, Default)] +pub(super) struct StaleAnchorFailures { + pub(super) count: u32, + pub(super) peers: HashSet, +} + +impl StaleAnchorFailures { + pub(super) fn record(&mut self, peer: ZakuraPeerId) { + self.count = self.count.saturating_add(1); + self.peers.insert(peer); + } + + pub(super) fn should_reanchor(&self) -> bool { + self.count >= HEADER_SYNC_STALE_ANCHOR_LINK_FAILURES + && self.peers.len() >= HEADER_SYNC_STALE_ANCHOR_DISTINCT_PEERS + } + + pub(super) fn reset(&mut self) { + self.count = 0; + self.peers.clear(); + } +} + #[derive(Copy, Clone, Debug)] pub(super) struct HeaderSyncAdvisoryPeerState { pub(super) summary: HeaderSyncServiceSummary, @@ -168,15 +197,18 @@ pub(super) struct PeerHeaderState { pub(super) session: HeaderSyncPeerSession, pub(super) direction: ServicePeerDirection, pub(super) advertised_tip: block::Height, + pub(super) advertised_hash: block::Hash, pub(super) anchor: block::Height, pub(super) max_headers_per_response: u32, pub(super) max_inflight_requests: u16, pub(super) received_status: bool, + /// The most recent status sent to this peer over its current session, if + /// any. Used to suppress re-sending an identical, non-tip-advancing status, + /// which the peer's inbound rate limiter would otherwise treat as spam. + pub(super) last_sent_status: Option, pub(super) outstanding: Vec, pub(super) late_covered_responses: usize, - pub(super) unsolicited: RateMeter, - pub(super) inbound_status: RateMeter, - pub(super) inbound_new_block: RateMeter, + pub(super) meters: HeaderSyncPeerMeters, pub(super) served_headers_inflight: u16, pub(super) misbehavior: u32, } @@ -184,7 +216,7 @@ pub(super) struct PeerHeaderState { impl PeerHeaderState { pub(super) fn new( session: HeaderSyncPeerSession, - anchor: block::Height, + anchor: (block::Height, block::Hash), local_range: u32, local_inflight: u16, status_refresh_interval: Duration, @@ -194,16 +226,20 @@ impl PeerHeaderState { Self { direction: session.direction(), session, - advertised_tip: anchor, - anchor, + advertised_tip: anchor.0, + advertised_hash: anchor.1, + anchor: anchor.0, max_headers_per_response: clamp_advertised_range(local_range), max_inflight_requests: local_inflight.clamp(1, LOCAL_MAX_HS_INFLIGHT_PER_PEER), received_status: false, + last_sent_status: None, outstanding: Vec::new(), late_covered_responses: 0, - unsolicited: RateMeter::new(status_refresh_interval), - inbound_status: RateMeter::new(inbound_status_min_interval), - inbound_new_block: RateMeter::new(inbound_new_block_min_interval), + meters: HeaderSyncPeerMeters::new( + status_refresh_interval, + inbound_status_min_interval, + inbound_new_block_min_interval, + ), served_headers_inflight: 0, misbehavior: 0, } @@ -219,6 +255,10 @@ impl PeerHeaderState { (!self.outstanding.is_empty()).then(|| self.outstanding.remove(0)) } + pub(super) fn restore_oldest_outstanding(&mut self, outstanding: OutstandingRange) { + self.outstanding.insert(0, outstanding); + } + pub(super) fn take_late_covered_response(&mut self) -> bool { if self.late_covered_responses == 0 { return false; @@ -227,6 +267,28 @@ impl PeerHeaderState { true } + /// Whether `status` differs from the most recent status sent to this peer + /// over its current session. A status identical to the last one we sent is + /// redundant — the peer cannot learn anything from it and its inbound status + /// rate limiter would treat it as spam — so callers suppress it. + pub(super) fn status_differs_from_last_sent(&self, status: HeaderSyncStatus) -> bool { + self.last_sent_status != Some(status) + } + + /// Records `status` as the most recent status sent to this peer, so a later + /// identical status can be suppressed by [`Self::status_differs_from_last_sent`]. + pub(super) fn record_sent_status(&mut self, status: HeaderSyncStatus) { + self.last_sent_status = Some(status); + } + + /// Forgets the last status sent to this peer so the next one is always sent. + /// Called when a fresh session replaces the peer's transport: the new + /// channel's remote has received no status yet and gates serving us on it, + /// so the initial status must go out regardless of its contents. + pub(super) fn reset_sent_status(&mut self) { + self.last_sent_status = None; + } + pub(super) fn try_start_serving_headers(&mut self, local_inflight_cap: u16) -> bool { if self.served_headers_inflight >= local_inflight_cap { return false; @@ -240,6 +302,27 @@ impl PeerHeaderState { } } +#[derive(Clone, Debug)] +pub(super) struct HeaderSyncPeerMeters { + pub(super) unsolicited: RateMeter, + pub(super) inbound_status: RateMeter, + pub(super) inbound_new_block: RateMeter, +} + +impl HeaderSyncPeerMeters { + pub(super) fn new( + status_refresh_interval: Duration, + inbound_status_min_interval: Duration, + inbound_new_block_min_interval: Duration, + ) -> Self { + Self { + unsolicited: RateMeter::new(status_refresh_interval), + inbound_status: RateMeter::new(inbound_status_min_interval), + inbound_new_block: RateMeter::new(inbound_new_block_min_interval), + } + } +} + #[derive(Copy, Clone, Debug)] pub(super) struct OutstandingRange { pub(super) range: RangeRequest, diff --git a/zebra-network/src/zakura/header_sync/tests.rs b/zebra-network/src/zakura/header_sync/tests.rs index c9cdc1c55c2..d6ecf03f05f 100644 --- a/zebra-network/src/zakura/header_sync/tests.rs +++ b/zebra-network/src/zakura/header_sync/tests.rs @@ -1,7 +1,8 @@ use super::*; use super::{config::*, error::*, events::*, reactor::*, validation::*, wire::*}; use crate::zakura::{ - testkit::TraceCapture, HeaderSyncServiceSummary, ServicePeerDirection, ServicePeerLimits, + testkit::{TraceCapture, TraceValue}, + HeaderSyncServiceSummary, ServicePeerDirection, ServicePeerLimits, }; use chrono::Duration; use metrics::{ @@ -114,6 +115,24 @@ fn mainnet_header(bytes: &[u8]) -> Arc { mainnet_block(bytes).header.clone() } +fn headers_message(headers: Vec>) -> HeaderSyncMessage { + let body_sizes = vec![0; headers.len()]; + HeaderSyncMessage::Headers { + headers, + body_sizes, + } +} + +fn headers_message_with_sizes( + headers: Vec>, + body_sizes: Vec, +) -> HeaderSyncMessage { + HeaderSyncMessage::Headers { + headers, + body_sizes, + } +} + async fn validate_headers_stateless_after_equihash_acceptance( headers: Vec>, context: HeaderSyncValidationContext<'_>, @@ -247,6 +266,7 @@ fn startup_for( HeaderSyncFrontiers { finalized_height: anchor.0, verified_block_tip: anchor.0, + verified_block_hash: anchor.1, }, best_header_tip, ZakuraHeaderSyncConfig::default(), @@ -267,6 +287,7 @@ fn startup_new_is_passive_until_local_hooks_are_wired() { HeaderSyncFrontiers { finalized_height: anchor.0, verified_block_tip: anchor.0, + verified_block_hash: anchor.1, }, Some(anchor), ZakuraHeaderSyncConfig::default(), @@ -297,6 +318,7 @@ async fn peer_caps_reject_full_without_status_or_misbehavior_and_free_on_remove( HeaderSyncFrontiers { finalized_height: anchor.0, verified_block_tip: anchor.0, + verified_block_hash: anchor.1, }, Some(anchor), ZakuraHeaderSyncConfig { @@ -440,7 +462,7 @@ async fn advisory_summary_status_mismatch_uses_status_without_misbehavior_and_ba .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(Vec::new()), + msg: headers_message(Vec::new()), }) .await .unwrap(); @@ -508,7 +530,7 @@ async fn advisory_backoff_is_pruned_on_peer_disconnected() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(Vec::new()), + msg: headers_message(Vec::new()), }) .await .unwrap(); @@ -547,6 +569,7 @@ async fn admission_failure_after_advisory_selection_creates_no_outstanding_range HeaderSyncFrontiers { finalized_height: anchor.0, verified_block_tip: anchor.0, + verified_block_hash: anchor.1, }, Some(anchor), ZakuraHeaderSyncConfig { @@ -634,6 +657,7 @@ async fn next_non_query_action(actions: &mut mpsc::Receiver) - HeaderSyncAction::QueryBestHeaderTip | HeaderSyncAction::QueryMissingBlockBodies { .. } | HeaderSyncAction::QueryHeadersByHeightRange { .. } + | HeaderSyncAction::HeaderAdvanced { .. } ) { return action; } @@ -651,6 +675,27 @@ async fn next_query_headers_action( } } +async fn next_outbound_get_headers( + actions: &mut mpsc::Receiver, +) -> (ZakuraPeerId, block::Height, u32) { + loop { + match next_non_query_action(actions).await { + HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + }, + } => return (peer, start_height, count), + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("unexpected misbehavior from {peer:?}: {reason:?}") + } + _ => {} + } + } +} + async fn assert_no_commit_or_misbehavior(actions: &mut mpsc::Receiver) { while let Ok(Some(action)) = tokio::time::timeout(std::time::Duration::from_millis(50), actions.recv()).await @@ -698,6 +743,27 @@ async fn advertise_tip( tip_height: block::Height, max_headers_per_response: u32, max_inflight_requests: u16, +) { + advertise_tip_with_hash( + fixture, + peer_id, + anchor_height, + tip_height, + block::Hash([9; 32]), + max_headers_per_response, + max_inflight_requests, + ) + .await; +} + +async fn advertise_tip_with_hash( + fixture: &ReactorFixture, + peer_id: ZakuraPeerId, + anchor_height: block::Height, + tip_height: block::Height, + tip_hash: block::Hash, + max_headers_per_response: u32, + max_inflight_requests: u16, ) { fixture .handle @@ -705,7 +771,7 @@ async fn advertise_tip( peer: peer_id, msg: HeaderSyncMessage::Status(HeaderSyncStatus { tip_height, - tip_hash: block::Hash([9; 32]), + tip_hash, anchor_height, max_headers_per_response, max_inflight_requests, @@ -748,7 +814,18 @@ fn codec_round_trips_get_headers() { #[test] fn codec_round_trips_headers_with_bounded_vector() { let headers = vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]; - let message = HeaderSyncMessage::Headers(headers); + let message = headers_message_with_sizes(headers, vec![123_456]); + + let encoded = message.encode().unwrap(); + let decoded = HeaderSyncMessage::decode(&encoded, headers_context(1, 1)).unwrap(); + + assert_eq!(decoded, message); +} + +#[test] +fn codec_round_trips_headers_with_unknown_body_size_sentinel() { + let headers = vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]; + let message = headers_message_with_sizes(headers, vec![0]); let encoded = message.encode().unwrap(); let decoded = HeaderSyncMessage::decode(&encoded, headers_context(1, 1)).unwrap(); @@ -787,6 +864,36 @@ fn codec_rejects_unknown_message_types_and_trailing_bytes() { )); } +#[test] +fn headers_codec_rejects_body_size_mismatch_truncation_and_trailing_bytes() { + let headers = vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]; + let message = headers_message_with_sizes(headers.clone(), vec![100]); + + assert!(matches!( + headers_message_with_sizes(headers.clone(), vec![100, 200]).encode(), + Err(HeaderSyncWireError::BodySizeCountMismatch { + headers: 1, + body_sizes: 2, + }) + )); + + let mut truncated_mid_size = message.encode().unwrap(); + truncated_mid_size.pop(); + assert!(HeaderSyncMessage::decode(&truncated_mid_size, headers_context(1, 1)).is_err()); + + let mut truncated_mid_header = vec![MSG_HS_HEADERS]; + truncated_mid_header.write_u32::(1).unwrap(); + truncated_mid_header.extend_from_slice(&[0; 8]); + assert!(HeaderSyncMessage::decode(&truncated_mid_header, headers_context(1, 1)).is_err()); + + let mut with_trailing = message.encode().unwrap(); + with_trailing.push(0); + assert!(matches!( + HeaderSyncMessage::decode(&with_trailing, headers_context(1, 1)), + Err(HeaderSyncWireError::TrailingBytes) + )); +} + #[test] fn frame_decode_rejects_oversized_payload_length_before_allocating() { let mut bytes = Vec::new(); @@ -829,13 +936,19 @@ fn decode_rejects_header_counts_over_contract_caps() { fn headers_codec_does_not_use_legacy_160_header_cap() { let header = mainnet_header(&BLOCK_MAINNET_1_BYTES); let headers = vec![header; 161]; - let message = HeaderSyncMessage::Headers(headers); + let message = headers_message(headers); let encoded = message.encode().unwrap(); let decoded = HeaderSyncMessage::decode(&encoded, headers_context(161, 161)).unwrap(); match decoded { - HeaderSyncMessage::Headers(headers) => assert_eq!(headers.len(), 161), + HeaderSyncMessage::Headers { + headers, + body_sizes, + } => { + assert_eq!(headers.len(), 161); + assert_eq!(body_sizes, vec![0; 161]); + } _ => panic!("decoded message must be Headers"), } } @@ -862,6 +975,7 @@ fn advertised_defaults_and_clamping_match_design() { let config = ZakuraHeaderSyncConfig::default(); assert_eq!(config.max_headers_per_response, DEFAULT_HS_RANGE); assert_eq!(config.max_inflight_requests, DEFAULT_HS_MAX_INFLIGHT); + assert!(config.accept_new_blocks); assert_eq!( ZakuraHeaderSyncConfig { max_inflight_requests: u16::MAX, @@ -905,7 +1019,7 @@ fn header_serialized_sizes_are_exact_and_message_cap_has_headroom() { let default_response_bytes = HEADER_SYNC_MESSAGE_TYPE_BYTES + HEADER_SYNC_COUNT_BYTES - + COMMON_HEADER_BYTES * DEFAULT_HS_RANGE as usize; + + (COMMON_HEADER_BYTES + HEADER_SYNC_BODY_SIZE_BYTES) * DEFAULT_HS_RANGE as usize; assert!(default_response_bytes < MAX_HS_MESSAGE_BYTES); assert!(MAX_HS_MESSAGE_BYTES < LOCAL_MAX_MESSAGE_BYTES as usize); } @@ -924,7 +1038,7 @@ fn request_and_serving_counts_are_clamped_by_byte_budget() { vec![mainnet_header(&BLOCK_MAINNET_1_BYTES); usize::try_from(count).unwrap() + 100]; let headers = truncate_headers_to_byte_budget(headers, &Network::Mainnet, LOCAL_MAX_MESSAGE_BYTES); - let encoded = HeaderSyncMessage::Headers(headers).encode().unwrap(); + let encoded = headers_message(headers).encode().unwrap(); assert!(encoded.len() <= MAX_HS_MESSAGE_BYTES); assert!(encoded.len() + FRAME_HEADER_BYTES <= LOCAL_MAX_MESSAGE_BYTES as usize); @@ -940,6 +1054,7 @@ async fn reactor_starts_from_storage_frontiers_and_publishes_watch() { HeaderSyncFrontiers { finalized_height: block::Height(2), verified_block_tip: block::Height(5), + verified_block_hash: block::Hash([5; 32]), }, Some(best), ZakuraHeaderSyncConfig::default(), @@ -1337,7 +1452,7 @@ async fn incoming_headers_match_outstanding_before_commit() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), }) .await .unwrap(); @@ -1399,7 +1514,7 @@ async fn headers_over_outstanding_contract_reports_response_too_long_without_flo .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(vec![ + msg: headers_message(vec![ mainnet_header(&BLOCK_MAINNET_1_BYTES), mainnet_header(&BLOCK_MAINNET_2_BYTES), ]), @@ -1471,7 +1586,7 @@ async fn matching_headers_are_statelessly_validated_before_commit() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(vec![ + msg: headers_message(vec![ mainnet_header(&BLOCK_MAINNET_1_BYTES), Arc::new(bad_second), ]), @@ -1569,7 +1684,7 @@ async fn peer_disconnect_removes_outstanding_requests_for_that_peer() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]), + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]), }) .await .unwrap(); @@ -1680,7 +1795,7 @@ async fn covered_hedged_outstanding_ranges_do_not_commit_twice() { .handle .send(HeaderSyncEvent::WireMessage { peer: second_peer, - msg: HeaderSyncMessage::Headers(Vec::new()), + msg: headers_message(Vec::new()), }) .await .unwrap(); @@ -1688,6 +1803,95 @@ async fn covered_hedged_outstanding_ranges_do_not_commit_twice() { assert_no_commit_or_misbehavior(&mut fixture.actions).await; } +#[tokio::test(flavor = "current_thread")] +async fn late_covered_response_does_not_reanchor_newer_outstanding_range() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(74); + let committed_hash = block::Hash([1; 32]); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(2), + 1, + 1, + ) + .await; + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 1, + }, + } if peer == peer_id => break, + _ => {} + } + } + + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: block::Height(1), + tip_height: block::Height(1), + tip_hash: committed_hash, + }) + .await + .unwrap(); + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height: block::Height(2), + count: 1, + }, + } if peer == peer_id => break, + _ => {} + } + } + + while tokio::time::timeout(std::time::Duration::from_millis(10), fixture.actions.recv()) + .await + .is_ok() + {} + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id, + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]), + }) + .await + .unwrap(); + + while let Ok(Some(action)) = + tokio::time::timeout(std::time::Duration::from_millis(50), fixture.actions.recv()).await + { + match action { + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::GetHeaders { .. }, + .. + } + | HeaderSyncAction::HeaderReanchored { .. } + | HeaderSyncAction::Misbehavior { .. } => { + panic!("late covered response must not trigger a new action: {action:?}") + } + _ => {} + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn local_commit_failure_retries_without_peer_misbehavior() { let checkpoint_hash = block::Hash::from(mainnet_header(&BLOCK_MAINNET_3_BYTES).as_ref()); @@ -1719,7 +1923,7 @@ async fn local_commit_failure_retries_without_peer_misbehavior() { .handle .send(HeaderSyncEvent::WireMessage { peer: first_peer.clone(), - msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), }) .await .unwrap(); @@ -1833,6 +2037,158 @@ async fn material_tip_advance_sends_rate_limited_unsolicited_status() { assert_eq!(status_count, 1); } +#[test] +fn peer_state_suppresses_redundant_status_until_session_reset() { + let (send, _recv) = crate::zakura::framed_channel(32); + let session = HeaderSyncPeerSession::from_parts_with_direction( + peer(80), + ServicePeerDirection::Inbound, + send, + CancellationToken::new(), + ); + let mut peer_state = super::state::PeerHeaderState::new( + session, + (block::Height(0), block::Hash([0; 32])), + DEFAULT_HS_RANGE, + DEFAULT_HS_MAX_INFLIGHT, + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(1), + ); + + let status = HeaderSyncStatus { + tip_height: block::Height(5), + tip_hash: block::Hash([5; 32]), + ..HeaderSyncStatus::default() + }; + + // Nothing has been sent yet, so the first status is always new. + assert!(peer_state.status_differs_from_last_sent(status)); + peer_state.record_sent_status(status); + + // An identical status is redundant and must be suppressed. + assert!(!peer_state.status_differs_from_last_sent(status)); + + // A tip-advancing status differs and is sent. + let advanced = HeaderSyncStatus { + tip_height: block::Height(6), + ..status + }; + assert!(peer_state.status_differs_from_last_sent(advanced)); + + // A same-height hash change (e.g. a reorg at the tip) also differs. + let reorged = HeaderSyncStatus { + tip_hash: block::Hash([9; 32]), + ..status + }; + assert!(peer_state.status_differs_from_last_sent(reorged)); + + // Replacing the session forgets the last status, so an identical status is + // resent — a fresh channel's remote has not received it and gates serving on it. + peer_state.reset_sent_status(); + assert!(peer_state.status_differs_from_last_sent(status)); +} + +#[tokio::test(flavor = "current_thread")] +async fn reconnect_resends_initial_status_after_session_reset() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(72); + + // First connect: the peer receives its initial status. + connect_peer(&fixture, peer_id.clone()).await; + assert!(matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); + + // Reconnecting installs a fresh session at the same frontier. Even though the + // status is byte-identical to the one already sent, the new channel's remote + // has not received it, so it must be resent rather than suppressed. + connect_peer(&fixture, peer_id.clone()).await; + assert!(matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn reconnect_clears_session_bound_outstanding_ranges() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_id = peer(73); + + connect_peer(&fixture, peer_id.clone()).await; + assert!(matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(5), + 1, + 1, + ) + .await; + assert!(matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 1, + }, + } if peer == peer_id + )); + + connect_peer(&fixture, peer_id.clone()).await; + assert!(matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + msg: HeaderSyncMessage::Status(_), + .. + } + )); + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(5), + 1, + 1, + ) + .await; + assert!(matches!( + next_non_query_action(&mut fixture.actions).await, + HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::GetHeaders { + start_height: block::Height(1), + count: 1, + }, + } if peer == peer_id + )); +} + #[tokio::test(flavor = "current_thread")] async fn full_block_committed_covers_outstanding_height() { let network = regtest_network(); @@ -1874,11 +2230,18 @@ async fn full_block_committed_covers_outstanding_height() { }) .await .unwrap(); + match next_action(&mut fixture.actions).await { + HeaderSyncAction::HeaderAdvanced { height, hash } => { + assert_eq!(height, block::Height(1)); + assert_eq!(hash, block::Hash([1; 32])); + } + action => panic!("full block commit must publish a header advance, got {action:?}"), + } fixture .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id, - msg: HeaderSyncMessage::Headers(Vec::new()), + msg: headers_message(Vec::new()), }) .await .unwrap(); @@ -2319,29 +2682,148 @@ async fn rapid_status_updates_and_new_block_spam_report_disconnect() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn new_block_spam_does_not_poison_seen_cache() { +async fn rapid_advancing_status_updates_are_not_spam() { let network = Network::Mainnet; - let first_block = mainnet_block(&BLOCK_MAINNET_1_BYTES); - let second_block = mainnet_block(&BLOCK_MAINNET_2_BYTES); - let second_hash = second_block.hash(); let mut fixture = spawn_test_reactor(startup_for( network.clone(), (block::Height(0), network.genesis_hash()), None, )); - let spam_peer = peer(56); - let honest_peer = peer(57); - let destination = peer(58); - - for peer_id in [spam_peer.clone(), honest_peer.clone(), destination] { - connect_peer(&fixture, peer_id).await; - } + let status_peer = peer(55); + connect_peer(&fixture, status_peer.clone()).await; - fixture - .handle - .send(HeaderSyncEvent::WireMessage { - peer: spam_peer.clone(), - msg: HeaderSyncMessage::NewBlock(first_block), + advertise_tip( + &fixture, + status_peer.clone(), + block::Height(0), + block::Height(1), + DEFAULT_HS_RANGE, + 1, + ) + .await; + advertise_tip( + &fixture, + status_peer, + block::Height(0), + block::Height(2), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(100), + fixture.actions.recv(), + ) + .await + { + if let HeaderSyncAction::Misbehavior { reason, .. } = action { + panic!("advancing status update was reported as {reason:?}"); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn same_height_hash_churn_is_status_spam() { + let network = Network::Mainnet; + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let status_peer = peer(59); + connect_peer(&fixture, status_peer.clone()).await; + + advertise_tip_with_hash( + &fixture, + status_peer.clone(), + block::Height(0), + block::Height(1), + block::Hash([1; 32]), + DEFAULT_HS_RANGE, + 1, + ) + .await; + advertise_tip_with_hash( + &fixture, + status_peer.clone(), + block::Height(0), + block::Height(1), + block::Hash([2; 32]), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + loop { + if let HeaderSyncAction::Misbehavior { peer, reason } = + next_non_query_action(&mut fixture.actions).await + { + assert_eq!(peer, status_peer); + assert_eq!(reason, HeaderSyncMisbehavior::StatusSpam); + break; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn same_height_hash_change_with_token_is_accepted() { + let network = Network::Mainnet; + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let status_peer = peer(60); + connect_peer(&fixture, status_peer.clone()).await; + + advertise_tip_with_hash( + &fixture, + status_peer, + block::Height(0), + block::Height(0), + block::Hash([3; 32]), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(100), + fixture.actions.recv(), + ) + .await + { + if let HeaderSyncAction::Misbehavior { reason, .. } = action { + panic!("same-height status update with a token was reported as {reason:?}"); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn new_block_spam_does_not_poison_seen_cache() { + let network = Network::Mainnet; + let first_block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let second_block = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let second_hash = second_block.hash(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let spam_peer = peer(56); + let honest_peer = peer(57); + let destination = peer(58); + + for peer_id in [spam_peer.clone(), honest_peer.clone(), destination] { + connect_peer(&fixture, peer_id).await; + } + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: spam_peer.clone(), + msg: HeaderSyncMessage::NewBlock(first_block), }) .await .unwrap(); @@ -2661,6 +3143,71 @@ async fn inbound_get_headers_over_cap_disconnects_without_state_read() { } } +#[tokio::test(flavor = "current_thread")] +async fn rejected_non_linking_range_traces_link_stage_and_error_kind() { + let network = regtest_network(); + let anchor = (block::Height(0), network.genesis_hash()); + let mut capture = + TraceCapture::for_test("rejected_non_linking_range_traces_link_stage_and_error_kind") + .unwrap(); + let mut startup = startup_for(network, anchor, Some(anchor)); + startup.trace = ZakuraTrace::new(capture.tracer(), "01"); + let mut fixture = spawn_test_reactor(startup); + let peer_id = peer(64); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + anchor.0, + block::Height(1), + DEFAULT_HS_RANGE, + 1, + ) + .await; + let (served_peer, start_height, count) = next_outbound_get_headers(&mut fixture.actions).await; + assert_eq!(served_peer, peer_id); + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 1); + + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: peer_id.clone(), + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_2_BYTES)]), + }) + .await + .unwrap(); + + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::Misbehavior { peer, reason } => { + assert_eq!(peer, peer_id); + assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + } + action => panic!("unexpected action: {action:?}"), + } + + capture.flush().await; + let reader = capture.reader().unwrap(); + let header_sync = reader.table(HEADER_SYNC_TABLE.table()); + let anchor_hash = format!("{}", anchor.1); + header_sync.assert_row( + hs_trace::HEADER_RANGE_REJECTED, + &[ + (hs_trace::RANGE_START, TraceValue::U64(1)), + (hs_trace::RANGE_COUNT, TraceValue::U64(1)), + (hs_trace::ANCHOR_HASH, TraceValue::Str(&anchor_hash)), + (hs_trace::VALIDATION_STAGE, TraceValue::Str("link")), + ( + hs_trace::ERROR_KIND, + TraceValue::Str("first_header_does_not_link"), + ), + ], + ); + + let _ = capture.finish().await.unwrap(); +} + #[tokio::test(flavor = "current_thread")] async fn header_sync_jsonl_trace_captures_status_range_dedup_and_disconnect() { let network = Network::Mainnet; @@ -2800,7 +3347,7 @@ async fn header_sync_metrics_record_status_range_new_block_dedup_and_disconnect( .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_4_BYTES)]), }) .await .unwrap(); @@ -2875,7 +3422,7 @@ async fn unsolicited_headers_are_misbehavior_but_empty_headers_retry() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(Vec::new()), + msg: headers_message(Vec::new()), }) .await .unwrap(); @@ -2912,7 +3459,7 @@ async fn unsolicited_headers_are_misbehavior_but_empty_headers_retry() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(Vec::new()), + msg: headers_message(Vec::new()), }) .await .unwrap(); @@ -2954,6 +3501,147 @@ async fn committed_range_updates_best_tip_watch_and_does_not_advance_finality() assert_ne!(fixture.handle.best_header_tip().0, block::Height(0)); } +#[tokio::test(flavor = "current_thread")] +async fn forward_link_wedge_reanchors_to_verified_tip_without_banning() { + let network = regtest_network(); + let verified = (block::Height(0), network.genesis_hash()); + let stranded_tip = (block::Height(3), block::Hash([3; 32])); + let mut startup = HeaderSyncStartup::new( + network.clone(), + verified, + HeaderSyncFrontiers { + finalized_height: verified.0, + verified_block_tip: verified.0, + verified_block_hash: verified.1, + }, + Some(stranded_tip), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.range_state_actions_enabled = true; + let mut fixture = spawn_test_reactor(startup); + let mut tip = fixture.handle.subscribe_tip(); + let peers = [peer(61), peer(62)]; + + for peer_id in peers.iter().cloned() { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + verified.0, + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + } + + for _ in 0..3 { + let (served_peer, start_height, count) = + next_outbound_get_headers(&mut fixture.actions).await; + assert_eq!(start_height, block::Height(4)); + assert_eq!(count, 1); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: served_peer, + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]), + }) + .await + .unwrap(); + } + + tip.changed().await.unwrap(); + assert_eq!(*tip.borrow(), verified); + assert_eq!(fixture.handle.best_header_tip(), verified); + + let expected_start = verified.0.next().expect("genesis has a successor"); + let mut saw_reanchor_action = false; + for _ in 0..8 { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::HeaderReanchored { old, new } => { + assert_eq!(old, stranded_tip); + assert_eq!(new, verified); + saw_reanchor_action = true; + } + HeaderSyncAction::SendMessage { + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count: _, + }, + .. + } if saw_reanchor_action && start_height == expected_start => { + assert_no_commit_or_misbehavior(&mut fixture.actions).await; + return; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("unexpected misbehavior from {peer:?}: {reason:?}"); + } + _ => {} + } + } + panic!("after re-anchor, header sync did not emit the reanchor action and request forward from the verified tip"); +} + +#[tokio::test(flavor = "current_thread")] +async fn single_peer_forward_link_failures_do_not_reanchor_globally() { + let network = regtest_network(); + let verified = (block::Height(0), network.genesis_hash()); + let stranded_tip = (block::Height(3), block::Hash([3; 32])); + let mut startup = HeaderSyncStartup::new( + network.clone(), + verified, + HeaderSyncFrontiers { + finalized_height: verified.0, + verified_block_tip: verified.0, + verified_block_hash: verified.1, + }, + Some(stranded_tip), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.range_state_actions_enabled = true; + let mut fixture = spawn_test_reactor(startup); + let mut tip = fixture.handle.subscribe_tip(); + let peer_id = peer(63); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + verified.0, + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + for _ in 0..3 { + let (served_peer, start_height, count) = + next_outbound_get_headers(&mut fixture.actions).await; + assert_eq!(start_height, block::Height(4)); + assert_eq!(count, 1); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: served_peer, + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_1_BYTES)]), + }) + .await + .unwrap(); + } + + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), tip.changed()) + .await + .is_err(), + "one peer alone must not lower the global header frontier" + ); + assert_eq!(fixture.handle.best_header_tip(), stranded_tip); + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + #[tokio::test(flavor = "current_thread")] async fn forward_genesis_backfill_reaches_checkpoint_before_finalized_commit() { let headers = [ @@ -3000,7 +3688,7 @@ async fn forward_genesis_backfill_reaches_checkpoint_before_finalized_commit() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(headers.to_vec()), + msg: headers_message(headers.to_vec()), }) .await .unwrap(); @@ -3064,7 +3752,7 @@ async fn truncated_finalized_backfill_is_rejected_before_commit() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(headers[..2].to_vec()), + msg: headers_message(headers[..2].to_vec()), }) .await .unwrap(); @@ -3121,7 +3809,7 @@ async fn backward_checkpoint_backfill_accepts_linking_run_as_finalized() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(headers.to_vec()), + msg: headers_message(headers.to_vec()), }) .await .unwrap(); @@ -3179,7 +3867,7 @@ async fn checkpoint_backfill_rejects_non_contiguous_run_before_commit() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(vec![ + msg: headers_message(vec![ mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES), mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES), mainnet_header(&BLOCK_MAINNET_GENESIS_BYTES), @@ -3231,7 +3919,7 @@ async fn header_response_that_does_not_link_to_anchor_is_misbehavior_before_comm .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(vec![mainnet_header(&BLOCK_MAINNET_2_BYTES)]), + msg: headers_message(vec![mainnet_header(&BLOCK_MAINNET_2_BYTES)]), }) .await .unwrap(); @@ -3288,7 +3976,7 @@ async fn checkpoint_backfill_rejects_checkpoint_hash_mismatch_before_commit() { .handle .send(HeaderSyncEvent::WireMessage { peer: peer_id.clone(), - msg: HeaderSyncMessage::Headers(headers.to_vec()), + msg: headers_message(headers.to_vec()), }) .await .unwrap(); @@ -3479,6 +4167,41 @@ async fn stateless_validation_rejects_wrong_solution_size_for_network() { )); } +#[test] +fn regtest_header_validation_accepts_common_and_short_solution_sizes() { + let regtest = Network::new_regtest(Default::default()); + let common_sized = mainnet_header(&BLOCK_MAINNET_1_BYTES); + let mut short_sized = *common_sized; + short_sized.solution = Solution::Regtest([0; 36]); + + validate_solution_sizes(std::slice::from_ref(&common_sized), ®test) + .expect("regtest accepts Zebra-mined common-size solutions"); + validate_solution_sizes(&[Arc::new(short_sized)], ®test) + .expect("regtest accepts short regtest solutions"); + assert!(matches!( + validate_solution_sizes(&[Arc::new(short_sized)], &Network::Mainnet), + Err(HeaderSyncWireError::WrongEquihashSolutionSize) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn regtest_stateless_validation_skips_pow_filter() { + let regtest = Network::new_regtest(Default::default()); + let mut header = *mainnet_header(&BLOCK_MAINNET_1_BYTES); + header.difficulty_threshold = + CompactDifficulty::from_bytes_in_display_order(&[0x01, 0x01, 0x00, 0x00]).unwrap(); + let context = HeaderSyncValidationContext { + network: ®test, + now: Utc::now(), + start_height: block::Height(1), + decode_context: headers_context(1, DEFAULT_HS_RANGE), + }; + + validate_headers_stateless(vec![Arc::new(header)], context) + .await + .expect("regtest header sync leaves PoW enforcement to block verification"); +} + #[tokio::test(flavor = "current_thread")] async fn pow_validation_does_not_monopolize_the_runtime_thread() { use std::sync::atomic::{AtomicUsize, Ordering}; @@ -3531,3 +4254,88 @@ fn hostile_vectors_are_rejected_for_allocation_and_unsolicited_headers() { Err(HeaderSyncWireError::UnsolicitedHeaders) )); } + +/// Regression for `claude-sync-reactor-action-backpressure-stalls-disconnect`: +/// when the bounded 128-slot action channel is saturated and the action driver +/// is stalled, the reactor must still disconnect a misbehaving peer promptly via +/// a prioritized, non-blocking path, instead of awaiting `actions.send` and +/// delaying the disconnect until the queue drains. +#[tokio::test] +async fn misbehavior_disconnect_is_prompt_when_action_channel_is_saturated() { + let network = Network::Mainnet; + let anchor = (block::Height(0), network.genesis_hash()); + let mut startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: anchor.0, + verified_block_tip: anchor.0, + verified_block_hash: anchor.1, + }, + Some(anchor), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + // Keep the test deterministic: no scheduling/state actions, so the only + // actions enqueued are the ones we drive below. + startup.range_state_actions_enabled = false; + let fixture = spawn_test_reactor(startup); + + // Connect the peer we will later flag as misbehaving and keep its session + // cancellation token so we can observe the local disconnect. + let probe = peer(7); + let probe_cancel = + connect_peer_with_direction(&fixture, probe.clone(), ServicePeerDirection::Inbound).await; + + // `anchor_height > tip_height` is an `InvalidStatus` misbehavior, evaluated + // before any per-peer state lookup, so it works for both an unknown filler + // peer and the connected probe peer. + let invalid_status = HeaderSyncMessage::Status(HeaderSyncStatus { + tip_height: block::Height(0), + tip_hash: block::Hash([0; 32]), + anchor_height: block::Height(1), + max_headers_per_response: 1, + max_inflight_requests: 1, + }); + + // Saturate the bounded action channel and never drain it (the production + // action driver is "stalled"). Misbehavior events from an unrelated, unknown + // peer enqueue `Misbehavior` actions until the channel is full. A per-send + // timeout keeps the test from hanging if the (unfixed) reactor wedges on a + // blocking `actions.send` and stops draining the bounded events queue. + let filler = peer(200); + for _ in 0..400 { + let send = fixture.handle.send(HeaderSyncEvent::WireMessage { + peer: filler.clone(), + msg: invalid_status.clone(), + }); + if tokio::time::timeout(std::time::Duration::from_millis(200), send) + .await + .is_err() + { + break; + } + } + + // Flag the connected probe peer as misbehaving while the action channel is + // saturated. Sending the event is best-effort: an unfixed, wedged reactor + // will never accept it, but the disconnect must still never arrive there. + let _ = tokio::time::timeout( + std::time::Duration::from_millis(200), + fixture.handle.send(HeaderSyncEvent::WireMessage { + peer: probe.clone(), + msg: invalid_status.clone(), + }), + ) + .await; + + // The disconnect must land well under `ACTION_SEND_TIMEOUT` and without + // anyone draining the action channel — proving a prioritized, non-blocking + // disconnect path rather than backpressured delivery. + tokio::time::timeout(std::time::Duration::from_secs(1), probe_cancel.cancelled()) + .await + .expect( + "a misbehaving peer must be disconnected promptly even when the action channel is \ + saturated and the action driver is stalled", + ); +} diff --git a/zebra-network/src/zakura/header_sync/validation.rs b/zebra-network/src/zakura/header_sync/validation.rs index ae595410992..a1d2bb9e985 100644 --- a/zebra-network/src/zakura/header_sync/validation.rs +++ b/zebra-network/src/zakura/header_sync/validation.rs @@ -161,7 +161,7 @@ pub async fn validate_headers_stateless( validate_internal_continuity(&headers)?; validate_header_times(&headers, context.now, context.start_height)?; validate_solution_sizes(&headers, context.network)?; - validate_pow_spawn_blocking(headers).await + validate_pow_spawn_blocking(headers, context.network).await } /// Check that a header range links to its anchor and is internally contiguous. @@ -191,7 +191,7 @@ pub async fn validate_new_block_stateless( let header = block.header.clone(); validate_header_times(std::slice::from_ref(&header), now, height)?; validate_solution_sizes(std::slice::from_ref(&header), network)?; - validate_pow_spawn_blocking(vec![header]).await + validate_pow_spawn_blocking(vec![header], network).await } pub(super) fn validate_header_count( @@ -245,7 +245,9 @@ pub(super) fn validate_solution_sizes( .is_some_and(|parameters| parameters.is_regtest()); for header in headers { match (expect_regtest, header.solution) { - (true, equihash::Solution::Regtest(_)) | (false, equihash::Solution::Common(_)) => {} + (true, equihash::Solution::Regtest(_)) + | (true, equihash::Solution::Common(_)) + | (false, equihash::Solution::Common(_)) => {} _ => return Err(HeaderSyncWireError::WrongEquihashSolutionSize), } } @@ -254,13 +256,22 @@ pub(super) fn validate_solution_sizes( pub(super) async fn validate_pow_spawn_blocking( headers: Vec>, + network: &Network, ) -> Result<(), HeaderSyncWireError> { - tokio::task::spawn_blocking(move || validate_pow_blocking(&headers)).await? + let skip_pow_filter = network + .parameters() + .is_some_and(|parameters| parameters.is_regtest()); + tokio::task::spawn_blocking(move || validate_pow_blocking(&headers, skip_pow_filter)).await? } pub(super) fn validate_pow_blocking( headers: &[Arc], + skip_pow_filter: bool, ) -> Result<(), HeaderSyncWireError> { + if skip_pow_filter { + return Ok(()); + } + for header in headers { header.solution.check(header)?; let hash = block::Hash::from(header.as_ref()); @@ -302,6 +313,19 @@ pub(super) fn validate_headers_len(len: usize, max: usize) -> Result<(), HeaderS Ok(()) } +pub(super) fn validate_body_sizes_len( + headers: usize, + body_sizes: usize, +) -> Result<(), HeaderSyncWireError> { + if headers != body_sizes { + return Err(HeaderSyncWireError::BodySizeCountMismatch { + headers, + body_sizes, + }); + } + Ok(()) +} + pub(super) fn clamp_advertised_range(value: u32) -> u32 { value.clamp(1, MAX_HS_RANGE) } diff --git a/zebra-network/src/zakura/header_sync/wire.rs b/zebra-network/src/zakura/header_sync/wire.rs index cadbd977ab6..ce25b2e2119 100644 --- a/zebra-network/src/zakura/header_sync/wire.rs +++ b/zebra-network/src/zakura/header_sync/wire.rs @@ -3,7 +3,10 @@ use super::{config::*, error::*, validation::*, *}; /// Zakura stream kind reserved for native header sync. pub const ZAKURA_STREAM_HEADER_SYNC: u16 = 5; /// Version of the native header-sync stream. -pub const ZAKURA_HEADER_SYNC_STREAM_VERSION: u16 = 1; +/// +/// Version 2 intentionally breaks stream-5 compatibility before header sync is +/// deployed: `Headers` now carries one advisory body-size hint per header. +pub const ZAKURA_HEADER_SYNC_STREAM_VERSION: u16 = 2; /// Peer status advertisement. pub const MSG_HS_STATUS: u8 = 1; @@ -25,6 +28,7 @@ pub const DEFAULT_HS_MAX_INFLIGHT: u16 = 10; pub(super) const HEADER_SYNC_MESSAGE_TYPE_BYTES: usize = 1; pub(super) const HEADER_SYNC_COUNT_BYTES: usize = 4; +pub(super) const HEADER_SYNC_BODY_SIZE_BYTES: usize = 4; pub(super) const COMMON_HEADER_BYTES: usize = 1_487; pub(super) const REGTEST_HEADER_BYTES: usize = 177; pub(super) const HEADER_SYNC_FANOUT: usize = 3; @@ -44,7 +48,7 @@ const _: () = assert!(MAX_HS_MESSAGE_BYTES < LOCAL_MAX_MESSAGE_BYTES as usize); const _: () = assert!( HEADER_SYNC_MESSAGE_TYPE_BYTES + HEADER_SYNC_COUNT_BYTES - + COMMON_HEADER_BYTES * (DEFAULT_HS_RANGE as usize) + + (COMMON_HEADER_BYTES + HEADER_SYNC_BODY_SIZE_BYTES) * (DEFAULT_HS_RANGE as usize) < MAX_HS_MESSAGE_BYTES ); @@ -60,8 +64,15 @@ pub enum HeaderSyncMessage { /// Requested header count. count: u32, }, - /// A bounded contiguous header run. - Headers(Vec>), + /// A bounded contiguous header run with one advisory body-size hint per header. + /// + /// A `0` size means "unknown"; the hint is not consensus data. + Headers { + /// Headers in ascending height order. + headers: Vec>, + /// Advisory serialized body sizes, parallel to `headers`. + body_sizes: Vec, + }, /// Full block tip-flood payload. NewBlock(Arc), } @@ -72,7 +83,7 @@ impl HeaderSyncMessage { match self { Self::Status(_) => MSG_HS_STATUS, Self::GetHeaders { .. } => MSG_HS_GET_HEADERS, - Self::Headers(_) => MSG_HS_HEADERS, + Self::Headers { .. } => MSG_HS_HEADERS, Self::NewBlock(_) => MSG_HS_NEW_BLOCK, } } @@ -91,11 +102,16 @@ impl HeaderSyncMessage { write_height(&mut bytes, *start_height)?; bytes.write_u32::(*count)?; } - Self::Headers(headers) => { + Self::Headers { + headers, + body_sizes, + } => { validate_headers_len(headers.len(), usize_from_u32(MAX_HS_RANGE, "headers cap")?)?; + validate_body_sizes_len(headers.len(), body_sizes.len())?; bytes.write_u32::(u32_from_usize(headers.len(), "headers count")?)?; - for header in headers { + for (header, body_size) in headers.iter().zip(body_sizes) { header.zcash_serialize(&mut bytes)?; + bytes.write_u32::(*body_size)?; } } Self::NewBlock(block) => { @@ -142,10 +158,15 @@ impl HeaderSyncMessage { }; validate_headers_len(count, max_headers)?; let mut headers = Vec::with_capacity(count); + let mut body_sizes = Vec::with_capacity(count); for _ in 0..count { headers.push(Arc::new(block::Header::zcash_deserialize(&mut reader)?)); + body_sizes.push(reader.read_u32::()?); + } + Self::Headers { + headers, + body_sizes, } - Self::Headers(headers) } MSG_HS_NEW_BLOCK => { Self::NewBlock(Arc::new(block::Block::zcash_deserialize(&mut reader)?)) diff --git a/zebra-network/src/zakura/legacy_gossip.rs b/zebra-network/src/zakura/legacy_gossip.rs index cb7458f8911..b4bf905a258 100644 --- a/zebra-network/src/zakura/legacy_gossip.rs +++ b/zebra-network/src/zakura/legacy_gossip.rs @@ -40,10 +40,11 @@ use crate::{ }; use super::{ - trace::peer_label as trace_peer_label, BoxRunFuture, Frame, FramedSend, OrderedSendError, Peer, - RequestResponseService, Service as ZakuraService, SinkReject, Stream, StreamMode, - ZakuraPeerHandle, ZakuraPeerId, ZakuraSupervisorHandle, ZakuraTrace, FRAME_HEADER_BYTES, - LEGACY_REQUEST_TABLE, LOCAL_MAX_CONTROL_FRAME_BYTES, ZAKURA_CAP_LEGACY_GOSSIP, + spawn_supervised_peer_task, trace::peer_label as trace_peer_label, BoxRunFuture, Frame, + FramedSend, OrderedSendError, Peer, RequestResponseService, Service as ZakuraService, + SinkReject, Stream, StreamMode, ZakuraPeerHandle, ZakuraPeerId, ZakuraSupervisorHandle, + ZakuraTrace, FRAME_HEADER_BYTES, LEGACY_REQUEST_TABLE, LOCAL_MAX_CONTROL_FRAME_BYTES, + ZAKURA_CAP_LEGACY_GOSSIP, }; /// Zakura stream kind reserved for legacy gossip compatibility. @@ -94,6 +95,18 @@ const LEGACY_REQUEST_IN_FLIGHT_LIMIT: usize = 64; const LEGACY_GOSSIP_SERVICE_TIMEOUT: Duration = Duration::from_secs(30); const DEFAULT_FIRST_SEEN_TTL: Duration = Duration::from_secs(10 * 60); const DEFAULT_FIRST_SEEN_CAPACITY: usize = 50_000; +/// A failed gossip inbound attempt that consumed at least this long is treated as +/// "expensive" (a slow/backpressured/timed-out service), and the same inventory is +/// placed in a short cooldown. The first-seen cache is only updated after a +/// *successful* call, so without this an authenticated peer could replay the same +/// valid advertisement while the inbound service is slow/erroring and make the +/// serial gossip worker re-pay the full 30s readiness/call budget for every +/// duplicate. Fast failures stay below this threshold and remain immediately +/// retryable, so a transient blip does not drop the advertisement. +const LEGACY_GOSSIP_EXPENSIVE_ATTEMPT: Duration = Duration::from_secs(1); +/// How long an expensive failed attempt suppresses duplicate copies of the same +/// inventory before a genuine re-advertisement may retry. +const LEGACY_GOSSIP_DUPLICATE_COOLDOWN: Duration = Duration::from_secs(30); const LEGACY_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); const SOURCE_INVENTORY_MISSING_RETRIES: usize = 8; const SOURCE_INVENTORY_MISSING_RETRY_DELAY: Duration = Duration::from_millis(500); @@ -104,6 +117,18 @@ const LEGACY_REQUEST_READY_TIMEOUT: Duration = Duration::from_secs(10); /// fetch on the legacy peer set forever, starving the Zakura path. const DUAL_STACK_LEGACY_INVENTORY_TIMEOUT: Duration = Duration::from_secs(3); const LEGACY_RESPONSE_CHUNK_BYTES: usize = 512 * 1024; +/// Maximum cumulative response payload bytes the inbound responder will buffer +/// for a single legacy request before aborting. +/// +/// A peer can name up to `MAX_TX_INV_IN_SENT_MESSAGE` block/transaction hashes +/// on one request; without an aggregate cap `encode_response` would serialize +/// and retain the entire multi-frame `Vec` (worst case +/// `MAX_TX_INV_IN_SENT_MESSAGE * MAX_PROTOCOL_MESSAGE_LEN`, tens of GiB) before +/// the first byte is written. The outbound reader already enforces a symmetric +/// per-response cap (`LegacyResponseBudget`); this is the responder-side mirror. +/// Sized well above any single response the local service emits (zebrad caps +/// `getdata` at ~1 MiB) yet far below memory-risk thresholds. +const LEGACY_RESPONSE_MAX_AGGREGATE_BYTES: usize = 8 * MAX_PROTOCOL_MESSAGE_LEN; const REQUEST_ID_BYTES: usize = 8; const RESPONSE_CHUNK_HEADER_BYTES: usize = REQUEST_ID_BYTES + 1; const NO_STOP_HASH: block::Hash = block::Hash([0; 32]); @@ -471,8 +496,14 @@ impl LegacyResponseCodec { request_id: u64, response: Response, max_frame_bytes: u32, + max_message_bytes: u32, ) -> Result, LegacyGossipError> { let mut frames = Vec::new(); + // Bound the cumulative response so a peer that requests many available + // blocks/transactions cannot force us to serialize and retain an + // unbounded `Vec` before the first byte is written. The budget is + // shared across every frame of this response and aborts encoding early. + let mut budget = ResponseEncodeBudget::default(); match response { Response::Blocks(blocks) => { let mut missing = Vec::new(); @@ -481,9 +512,11 @@ impl LegacyResponseCodec { InventoryResponse::Available((block, _)) => { push_chunked_response( &mut frames, + &mut budget, MSG_RESPONSE_BLOCK, request_id, max_frame_bytes, + max_message_bytes, block.zcash_serialize_to_vec()?, )?; } @@ -491,7 +524,11 @@ impl LegacyResponseCodec { } } if !missing.is_empty() { - frames.push(missing_blocks_frame(request_id, missing)?); + push_response_frame( + &mut frames, + &mut budget, + missing_blocks_frame(request_id, missing)?, + )?; } } Response::Transactions(transactions) => { @@ -501,9 +538,11 @@ impl LegacyResponseCodec { InventoryResponse::Available((transaction, _)) => { push_chunked_response( &mut frames, + &mut budget, MSG_RESPONSE_TRANSACTION, request_id, max_frame_bytes, + max_message_bytes, transaction.transaction.zcash_serialize_to_vec()?, )?; } @@ -511,29 +550,50 @@ impl LegacyResponseCodec { } } if !missing.is_empty() { - frames.push(missing_transactions_frame(request_id, missing)?); + push_response_frame( + &mut frames, + &mut budget, + missing_transactions_frame(request_id, missing)?, + )?; } } Response::BlockHashes(hashes) => { // FindBlocks should already be service-capped; overflowing the wire cap is a bug. - frames.push(block_hashes_frame(request_id, hashes)?); + push_response_frame( + &mut frames, + &mut budget, + block_hashes_frame(request_id, hashes)?, + )?; } Response::BlockHeaders(headers) => { // FindHeaders is protocol-capped by MAX_HEADERS_PER_MESSAGE; reject overflow. - frames.push(block_headers_frame(request_id, headers)?); + push_response_frame( + &mut frames, + &mut budget, + block_headers_frame(request_id, headers)?, + )?; } Response::TransactionIds(ids) => { // Mempools can exceed one legacy inv response, so advertise the first capped page. - frames.push(transaction_ids_frame( - request_id, - truncate_to_inventory_cap(ids)?, - )?); + push_response_frame( + &mut frames, + &mut budget, + transaction_ids_frame(request_id, truncate_to_inventory_cap(ids)?)?, + )?; } Response::Pong(_) => { - frames.push(id_only_frame(MSG_RESPONSE_PONG, request_id)); + push_response_frame( + &mut frames, + &mut budget, + id_only_frame(MSG_RESPONSE_PONG, request_id), + )?; } Response::Nil => { - frames.push(id_only_frame(MSG_RESPONSE_NIL, request_id)); + push_response_frame( + &mut frames, + &mut budget, + id_only_frame(MSG_RESPONSE_NIL, request_id), + )?; } response => return Err(LegacyGossipError::UnexpectedResponse(response.command())), } @@ -849,18 +909,62 @@ fn ensure_header_count(count: usize) -> Result<(), LegacyGossipError> { Ok(()) } +/// Tracks the cumulative size of an encoded legacy response so the inbound +/// responder aborts a single request's response early instead of buffering an +/// unbounded `Vec` before the first byte is written. The outbound reader +/// enforces a symmetric per-response cap via `LegacyResponseBudget`. +#[derive(Default)] +struct ResponseEncodeBudget { + bytes: usize, +} + +impl ResponseEncodeBudget { + /// Account for one buffered response frame's payload, rejecting the whole + /// response once cumulative payload bytes exceed the responder aggregate + /// budget. + fn account(&mut self, payload_len: usize) -> Result<(), LegacyGossipError> { + self.bytes = self.bytes.saturating_add(payload_len); + if self.bytes > LEGACY_RESPONSE_MAX_AGGREGATE_BYTES { + return Err(LegacyGossipError::ResponseAggregateBudget(self.bytes)); + } + Ok(()) + } +} + +/// Push one fully-built response frame, charging it against the aggregate +/// budget first so an over-budget response aborts before it is retained. +fn push_response_frame( + frames: &mut Vec, + budget: &mut ResponseEncodeBudget, + frame: Frame, +) -> Result<(), LegacyGossipError> { + budget.account(frame.payload.len())?; + frames.push(frame); + Ok(()) +} + fn push_chunked_response( frames: &mut Vec, + budget: &mut ResponseEncodeBudget, message_type: u16, request_id: u64, max_frame_bytes: u32, + max_message_bytes: u32, bytes: Vec, ) -> Result<(), LegacyGossipError> { if bytes.len() > MAX_PROTOCOL_MESSAGE_LEN { return Err(LegacyGossipError::OversizedResponse(bytes.len())); } - let max_payload_bytes = usize::try_from(max_frame_bytes)?.saturating_sub(FRAME_HEADER_BYTES); + // Size each chunk frame against the *effective* outbound cap: the smaller of + // the negotiated frame payload cap (`max_frame_bytes - FRAME_HEADER_BYTES`) + // and the peer's negotiated `max_message_bytes`. The handshake clamps the two + // caps independently, so sizing against the frame cap alone produces chunk + // frames whose payload exceeds the peer's accepted message cap, which the + // peer (and our own `write_response_frame`) reject as oversize. + let max_payload_bytes = usize::try_from(max_frame_bytes)? + .saturating_sub(FRAME_HEADER_BYTES) + .min(usize::try_from(max_message_bytes)?); let max_chunk_bytes = max_payload_bytes .checked_sub(RESPONSE_CHUNK_HEADER_BYTES) .ok_or(LegacyGossipError::OversizedResponse(bytes.len()))?; @@ -879,6 +983,10 @@ fn push_chunked_response( payload.extend_from_slice(&request_id.to_le_bytes()); payload.push(u8::from(index + 1 == chunks.len())); payload.extend_from_slice(chunk); + // Charge each chunk against the shared budget so a response that + // aggregates many available items aborts mid-encode rather than after + // the whole `Vec` has been materialized. + budget.account(payload.len())?; frames.push(Frame { message_type, flags: 0, @@ -1174,6 +1282,14 @@ impl LegacyGossipOutbound { .remove(peer); } + #[cfg(test)] + fn contains(&self, peer: &ZakuraPeerId) -> bool { + self.sessions + .lock() + .expect("legacy gossip outbound mutex is never poisoned") + .contains_key(peer) + } + fn remember_latest_block(&self, frame: &LegacyGossipFrame) { if let LegacyGossipFrame::AdvertiseBlock(hash) = frame { *self @@ -1219,6 +1335,32 @@ impl LegacyGossipOutbound { } } +#[cfg(test)] +fn legacy_gossip_recv_loop_panic_target() -> &'static StdMutex> { + static TARGET: OnceLock>> = OnceLock::new(); + TARGET.get_or_init(Default::default) +} + +#[cfg(test)] +fn arm_legacy_gossip_recv_loop_panic(peer: ZakuraPeerId) { + *legacy_gossip_recv_loop_panic_target() + .lock() + .expect("legacy gossip recv-loop panic target mutex is never poisoned") = Some(peer); +} + +#[cfg(test)] +fn should_panic_legacy_gossip_recv_loop(peer: &ZakuraPeerId) -> bool { + let mut target = legacy_gossip_recv_loop_panic_target() + .lock() + .expect("legacy gossip recv-loop panic target mutex is never poisoned"); + if target.as_ref() == Some(peer) { + *target = None; + true + } else { + false + } +} + /// Typed ordered legacy gossip sender for one peer. #[derive(Clone, Debug)] pub struct LegacyGossipPeerSession { @@ -1948,12 +2090,20 @@ fn bounded_u64(value: usize) -> u64 { #[derive(Clone, Debug)] struct LegacyGossipForwarder { broadcast: ZakuraGossipBroadcast, + /// Short-lived dedup of inventory already handed to the inbound service but not + /// yet confirmed `mark_seen`. Bounds duplicate work when the service is slow, + /// not ready, or erroring (see [`LEGACY_GOSSIP_DUPLICATE_COOLDOWN`]). + attempt_cooldown: FirstSeenCache, } impl LegacyGossipForwarder { fn new(supervisor: ZakuraSupervisorHandle) -> Self { Self { broadcast: ZakuraGossipBroadcast::new(supervisor), + attempt_cooldown: FirstSeenCache::new( + DEFAULT_FIRST_SEEN_CAPACITY, + LEGACY_GOSSIP_DUPLICATE_COOLDOWN, + ), } } @@ -1961,6 +2111,22 @@ impl LegacyGossipForwarder { self.broadcast.unseen(frame).await } + /// Return the subset of `frame`'s inventory not currently suppressed by a recent + /// expensive failed attempt. Read-only: the cooldown is populated only by + /// [`Self::note_failed_attempt`], so fast failures remain immediately retryable. + async fn fresh_attempt(&self, frame: &LegacyGossipFrame) -> Option { + self.attempt_cooldown.unseen(frame).await + } + + /// Record `frame`'s inventory in the cooldown when a failed attempt consumed at + /// least [`LEGACY_GOSSIP_EXPENSIVE_ATTEMPT`], so queued duplicates skip re-paying + /// a slow readiness/call. Cheap failures are left immediately retryable. + async fn note_failed_attempt(&self, frame: &LegacyGossipFrame, elapsed: Duration) { + if elapsed >= LEGACY_GOSSIP_EXPENSIVE_ATTEMPT { + self.attempt_cooldown.record_seen(frame).await; + } + } + async fn mark_seen(&self, frame: &LegacyGossipFrame) { self.broadcast.mark_seen(frame).await; } @@ -2059,6 +2225,7 @@ impl LegacyGossipSink { stream_kind: u16, request_id: u64, max_frame_bytes: u32, + max_message_bytes: u32, frame: Frame, ) -> BoxRunFuture<'a, Result, SinkReject>> { Box::pin(async move { @@ -2100,8 +2267,13 @@ impl LegacyGossipSink { response.command(), &response, ); - LegacyResponseCodec::encode_response(request_id, response, max_frame_bytes) - .map_err(SinkReject::local) + LegacyResponseCodec::encode_response( + request_id, + response, + max_frame_bytes, + max_message_bytes, + ) + .map_err(SinkReject::local) }) } } @@ -2126,51 +2298,80 @@ impl ZakuraService for LegacyGossipSink { let session = LegacyGossipPeerSession::new(peer_id.clone(), send); outbound.insert(session.clone()); - tokio::spawn({ - let outbound = outbound.clone(); - let session = session.clone(); - async move { - if let Err(error) = outbound.replay_latest_block_to_peer(session).await { - debug!(?error, "latest Zakura block gossip replay failed"); + let replay_task_peer_id = peer_id.clone(); + let replay_panic_peer_id = replay_task_peer_id.clone(); + let replay_panic_outbound = outbound.clone(); + let replay_panic_cancel = cancel_token.clone(); + spawn_supervised_peer_task( + replay_task_peer_id, + || {}, + move || { + replay_panic_cancel.cancel(); + replay_panic_outbound.remove(&replay_panic_peer_id); + }, + { + let outbound = outbound.clone(); + let session = session.clone(); + async move { + if let Err(error) = outbound.replay_latest_block_to_peer(session).await { + debug!(?error, "latest Zakura block gossip replay failed"); + } } - } - }); + }, + ); - tokio::spawn(async move { - loop { - let frame = tokio::select! { - _ = cancel_token.cancelled() => { - outbound.remove(&peer_id); - return; - } - frame = recv.recv() => { - let Some(frame) = frame else { + let recv_task_peer_id = peer_id.clone(); + let recv_panic_peer_id = recv_task_peer_id.clone(); + let recv_panic_outbound = outbound.clone(); + let recv_panic_cancel = cancel_token.clone(); + spawn_supervised_peer_task( + recv_task_peer_id, + || {}, + move || { + recv_panic_cancel.cancel(); + recv_panic_outbound.remove(&recv_panic_peer_id); + }, + async move { + loop { + let frame = tokio::select! { + _ = cancel_token.cancelled() => { outbound.remove(&peer_id); return; - }; - frame - } - }; + } + frame = recv.recv() => { + let Some(frame) = frame else { + outbound.remove(&peer_id); + return; + }; + frame + } + }; - match Self::enqueue_gossip_frame(&inbound_tx, peer_id.clone(), frame) { - Ok(()) => {} - Err(SinkReject::Protocol(error)) => { - debug!( - ?error, - ?peer_id, - "legacy gossip stream rejected protocol-invalid frame" - ); - cancel_token.cancel(); - outbound.remove(&peer_id); - return; + #[cfg(test)] + if should_panic_legacy_gossip_recv_loop(&peer_id) { + panic!("injected legacy gossip recv-loop panic after state registration"); } - Err(SinkReject::Local(error)) => { - debug!(?error, ?peer_id, "legacy gossip inbound queue closed"); - return; + + match Self::enqueue_gossip_frame(&inbound_tx, peer_id.clone(), frame) { + Ok(()) => {} + Err(SinkReject::Protocol(error)) => { + debug!( + ?error, + ?peer_id, + "legacy gossip stream rejected protocol-invalid frame" + ); + cancel_token.cancel(); + outbound.remove(&peer_id); + return; + } + Err(SinkReject::Local(error)) => { + debug!(?error, ?peer_id, "legacy gossip inbound queue closed"); + return; + } } } - } - }); + }, + ); } fn remove_peer(&self, peer: &ZakuraPeerId) { @@ -2198,9 +2399,17 @@ impl RequestResponseService for LegacyGossipSink { stream_kind: u16, request_id: u64, max_frame_bytes: u32, + max_message_bytes: u32, frame: Frame, ) -> BoxRunFuture<'a, Result, SinkReject>> { - self.request(peer_id, stream_kind, request_id, max_frame_bytes, frame) + self.request( + peer_id, + stream_kind, + request_id, + max_frame_bytes, + max_message_bytes, + frame, + ) } } @@ -2269,11 +2478,25 @@ async fn handle_legacy_gossip( let Some(unseen_frame) = forwarder.unseen(&gossip.frame).await else { return; }; + // Skip inventory whose recent service attempt was expensive but did not succeed. + // The not-ready, call-error, and call-timeout paths below all return without + // `mark_seen`, so without this an authenticated peer could replay the same valid + // advertisement while the inbound service is slow/erroring and make the serial + // worker re-pay the full 30s+30s readiness/call budget for every queued + // duplicate. Fast failures are not recorded, so a transient blip stays retryable. + let Some(unseen_frame) = forwarder.fresh_attempt(&unseen_frame).await else { + debug!("legacy gossip duplicate suppressed after recent expensive attempt"); + return; + }; let request = unseen_frame.clone().into_request(gossip.peer_id.clone()); + let started = Instant::now(); let ready = timeout(LEGACY_GOSSIP_SERVICE_TIMEOUT, inbound.ready()).await; let Ok(Ok(service)) = ready else { debug!("legacy gossip inbound service was not ready"); + forwarder + .note_failed_attempt(&unseen_frame, started.elapsed()) + .await; return; }; @@ -2281,10 +2504,16 @@ async fn handle_legacy_gossip( Ok(Ok(_)) => {} Ok(Err(error)) => { debug!(?error, "legacy gossip inbound service call failed"); + forwarder + .note_failed_attempt(&unseen_frame, started.elapsed()) + .await; return; } Err(_) => { debug!("legacy gossip inbound service call timed out"); + forwarder + .note_failed_attempt(&unseen_frame, started.elapsed()) + .await; return; } } @@ -2306,28 +2535,53 @@ async fn handle_legacy_request( { let command = request.frame.to_string(); let peer_id = request.peer_id.clone(); + let request_id = request.request_id; let request_kind = request.frame.kind(); + let mut response_tx = request.response_tx; let Some(legacy_request) = request.frame.into_service_request() else { // Inbound legacy Ping is handled locally; the requester measures round-trip time. - let _ = request.response_tx.send(Ok(Response::Pong(Duration::ZERO))); + let _ = response_tx.send(Ok(Response::Pong(Duration::ZERO))); return; }; trace_legacy_request_start( &trace, "inbound.request", Some(&peer_id), - request.request_id, + request_id, request_kind, request_kind.message_type(), ); - let ready = timeout(LEGACY_REQUEST_TIMEOUT, inbound.ready()).await; - let result = match ready { - Ok(Ok(service)) => timeout(LEGACY_REQUEST_TIMEOUT, service.call(legacy_request)) - .await - .map_err(|_| -> BoxError { format!("{command} inbound service timed out").into() }) - .and_then(|response| response), - Ok(Err(error)) => Err(error), - Err(_) => Err(format!("{command} inbound service readiness timed out").into()), + + let service_work = async move { + let ready = timeout(LEGACY_REQUEST_TIMEOUT, inbound.ready()).await; + match ready { + Ok(Ok(service)) => timeout(LEGACY_REQUEST_TIMEOUT, service.call(legacy_request)) + .await + .map_err(|_| -> BoxError { format!("{command} inbound service timed out").into() }) + .and_then(|response| response), + Ok(Err(error)) => Err(error), + Err(_) => Err(format!("{command} inbound service readiness timed out").into()), + } + }; + + // Tie the spawned handler's lifetime to the request stream. The request-stream + // side (`LegacyGossipSink::request`) waits only LEGACY_REQUEST_TIMEOUT for the + // oneshot result and then drops the receiver; the peer/connection going away + // drops it too. In either case `closed()` resolves, so abort the in-flight + // service work and release the permit instead of letting this orphaned handler + // keep one of LEGACY_REQUEST_IN_FLIGHT_LIMIT permits (and keep doing backend + // work) for up to another full readiness + service-call timeout. + let result = tokio::select! { + biased; + () = response_tx.closed() => { + debug!( + ?peer_id, + request_id, + "legacy request handler aborted: response receiver dropped before service completed" + ); + return; + } + result = service_work => result, }; if let Err(error) = &result { @@ -2335,16 +2589,16 @@ async fn handle_legacy_request( &trace, "inbound.error", Some(&peer_id), - request.request_id, + request_id, request_kind.command(), error.to_string(), ); } - if request.response_tx.send(result).is_err() { + if response_tx.send(result).is_err() { debug!( - peer_id = ?request.peer_id, - "legacy request response receiver dropped before service completed" + ?peer_id, + request_id, "legacy request response receiver dropped before service completed" ); } } @@ -2530,6 +2784,9 @@ pub enum LegacyGossipError { /// A response exceeded the protocol message size. #[error("oversized legacy response: {0} bytes")] OversizedResponse(usize), + /// The encoded response would exceed the responder-side aggregate budget. + #[error("legacy response exceeded responder aggregate budget: {0} bytes")] + ResponseAggregateBudget(usize), /// The legacy service returned an unexpected response variant. #[error("unexpected legacy response: {0}")] UnexpectedResponse(&'static str), @@ -2820,6 +3077,50 @@ mod tests { } } + /// Always ready, but every `call` future is pending forever. Models a slow or + /// backpressured inbound service whose work outlives the request-stream timeout. + #[derive(Clone, Debug)] + struct NeverCompletesService; + + impl Service for NeverCompletesService { + type Response = Response; + type Error = BoxError; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _request: Request) -> Self::Future { + std::future::pending().boxed() + } + } + + /// Always ready, but every `call` future is pending forever and counts + /// invocations. Models a slow/backpressured inbound service whose calls hit + /// `LEGACY_GOSSIP_SERVICE_TIMEOUT`, so `handle_legacy_gossip` returns without + /// `mark_seen`, exposing how many times duplicate gossip frames reach the + /// expensive readiness/call path. + #[derive(Clone, Debug)] + struct CountingPendingService { + calls: Arc, + } + + impl Service for CountingPendingService { + type Response = Response; + type Error = BoxError; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _request: Request) -> Self::Future { + self.calls.fetch_add(1, Ordering::SeqCst); + std::future::pending().boxed() + } + } + #[derive(Clone, Debug)] struct RecordingInventoryResponder { transaction: Option, @@ -3001,6 +3302,11 @@ mod tests { > { let (pushed_tx, pushed_rx) = tokio::sync::mpsc::unbounded_channel(); let node = ZakuraTestNode::builder(seed) + // Multi-node gossip topologies dial several loopback peers from one + // node, so opt out of the production per-IP cap (1) that the default + // test node now enforces; these tests exercise gossip routing, not + // the per-IP admission gate. + .max_connections_per_ip(8) .service_from_supervisor(move |supervisor| { Arc::new(LegacyGossipSink::spawn( NormalNetworkResponder { @@ -3038,6 +3344,39 @@ mod tests { .ok_or_else(|| "pushed transaction recorder closed".into()) } + fn legacy_gossip_peer( + peer_id: ZakuraPeerId, + cancel_token: tokio_util::sync::CancellationToken, + ) -> (Peer, FramedSend) { + let (peer_send, service_recv) = framed_channel(8); + let (service_send, _peer_recv) = framed_channel(8); + let peer = Peer::new( + peer_id, + None, + ZAKURA_CAP_LEGACY_GOSSIP, + HashMap::from([(ZAKURA_STREAM_GOSSIP, (service_recv, service_send))]), + cancel_token, + ); + (peer, peer_send) + } + + async fn wait_for_legacy_gossip_panic_cleanup( + outbound: &LegacyGossipOutbound, + peer_id: &ZakuraPeerId, + cancel_token: &tokio_util::sync::CancellationToken, + ) -> Result<(), BoxError> { + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if cancel_token.is_cancelled() && !outbound.contains(peer_id) { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| -> BoxError { "timed out waiting for legacy gossip panic cleanup".into() }) + } + async fn wait_registered_count(node: &ZakuraTestNode, count: usize) -> Result<(), BoxError> { tokio::time::timeout(Duration::from_secs(5), async { loop { @@ -3051,6 +3390,64 @@ mod tests { .map_err(|_| -> BoxError { "timed out waiting for peer registration count".into() }) } + /// Regression for `claude-legacy-request-orphaned-handler-permits`. + /// + /// The request-stream side (`LegacyGossipSink::request`) waits only + /// `LEGACY_REQUEST_TIMEOUT` for the oneshot result and then drops the receiver; + /// the peer disconnecting drops it too. The spawned `handle_legacy_request` task + /// must not keep its in-flight permit (one of `LEGACY_REQUEST_IN_FLIGHT_LIMIT`) + /// or its backend service work alive after that. Before the fix it stayed blocked + /// in `service.call` for up to another full `LEGACY_REQUEST_TIMEOUT`, so an + /// attacker driving slow inbound work could occupy all 64 permits. + #[tokio::test(start_paused = true)] + async fn legacy_request_handler_releases_permit_when_request_stream_drops_receiver() { + let permits = Arc::new(Semaphore::new(LEGACY_REQUEST_IN_FLIGHT_LIMIT)); + let permit = permits + .clone() + .try_acquire_owned() + .expect("a permit is available"); + assert_eq!( + permits.available_permits(), + LEGACY_REQUEST_IN_FLIGHT_LIMIT - 1, + "one permit is held while the handler runs" + ); + + let (response_tx, response_rx) = oneshot::channel(); + let request = LegacyRequestInbound { + peer_id: ZakuraPeerId::new(vec![7u8; 32]).expect("valid peer id"), + request_id: 1, + // A non-Ping request so the handler drives the inbound service. + frame: LegacyRequestFrame::MempoolTransactionIds, + response_tx, + }; + + let handler = tokio::spawn(handle_legacy_request( + NeverCompletesService, + request, + permit, + ZakuraTrace::noop(), + )); + + // Let the handler enter the (never-completing) service call, then model the + // request-stream side giving up: it drops the oneshot receiver. + tokio::task::yield_now().await; + drop(response_rx); + + // The handler must observe the dropped receiver, abort the service work, and + // release the permit promptly. Without the fix this times out because the + // handler stays blocked in `service.call` until `LEGACY_REQUEST_TIMEOUT`. + tokio::time::timeout(Duration::from_secs(5), handler) + .await + .expect("handler aborts promptly after the request stream drops the receiver") + .expect("handler task does not panic"); + + assert_eq!( + permits.available_permits(), + LEGACY_REQUEST_IN_FLIGHT_LIMIT, + "the in-flight permit must be released once the handler aborts" + ); + } + #[test] fn block_gossip_round_trips_and_rejects_trailing_bytes() { let frame = LegacyGossipFrame::AdvertiseBlock(block_hash(1)); @@ -3506,6 +3903,57 @@ mod tests { Ok(()) } + /// Regression: while the legacy inbound service is slow/timing out, + /// `handle_legacy_gossip` returns without `mark_seen`, so an authenticated peer + /// could replay the same valid advertisement and make the serial worker re-pay + /// the full readiness/call timeout for every queued duplicate. After one + /// expensive failed attempt the cooldown must suppress identical duplicates. + /// + /// Paused time auto-advances the `LEGACY_GOSSIP_SERVICE_TIMEOUT` so the first + /// call's timeout fires without a real wall-clock wait. + #[tokio::test(start_paused = true)] + async fn duplicate_gossip_does_not_repeat_expensive_attempts_while_service_is_slow( + ) -> Result<(), BoxError> { + let calls = Arc::new(AtomicUsize::new(0)); + let (inbound_tx, inbound_rx) = mpsc::channel(16); + let supervisor = ZakuraSupervisorHandle::new(1); + let peer_id = ZakuraPeerId::new(vec![7; 32]).expect("test peer id is within bounds"); + + let worker = tokio::spawn(legacy_gossip_worker( + CountingPendingService { + calls: calls.clone(), + }, + inbound_rx, + LegacyGossipForwarder::new(supervisor), + ZakuraTrace::noop(), + )); + + // The call never completes, so the permanent first-seen cache is never + // updated; only the post-timeout cooldown can suppress these duplicates. + let frame = LegacyGossipFrame::AdvertiseBlock(block_hash(99)); + for _ in 0..4 { + inbound_tx + .send(LegacyInboundWork::Gossip(LegacyGossipInbound { + peer_id: peer_id.clone(), + frame: frame.clone(), + })) + .await?; + } + + drop(inbound_tx); + worker.await?; + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "duplicate gossip frames must not each re-pay the inbound readiness/call \ + timeout while the service is slow; expected one expensive attempt with \ + the rest suppressed by the cooldown" + ); + + Ok(()) + } + #[tokio::test] async fn service_request_routes_to_advertiser_then_fallback_after_missing( ) -> Result<(), BoxError> { @@ -3515,7 +3963,13 @@ mod tests { let (advertiser, mut advertiser_rx) = recording_inventory_node(65, None).await?; let (fallback, mut fallback_rx) = recording_inventory_node(66, Some(transaction.clone())).await?; - let requester = ZakuraTestNode::builder(67).spawn().await?; + // The requester dials two loopback peers (advertiser + fallback), so it + // opts out of the production per-IP cap (1) that the default test node + // now enforces; this test exercises service routing, not per-IP admission. + let requester = ZakuraTestNode::builder(67) + .max_connections_per_ip(8) + .spawn() + .await?; requester .connect_native(&advertiser, Duration::from_secs(5)) .await?; @@ -3723,6 +4177,56 @@ mod tests { Ok(()) } + #[tokio::test] + async fn legacy_gossip_replay_panic_cancels_peer_and_removes_outbound_session( + ) -> Result<(), BoxError> { + let (inbound_tx, _inbound_rx) = mpsc::channel(1); + let outbound = LegacyGossipOutbound::default(); + let sink = LegacyGossipSink { + inbound_tx, + outbound: outbound.clone(), + trace: ZakuraTrace::noop(), + }; + let peer_id = ZakuraPeerId::new(vec![92; 32]).expect("test peer id is within bounds"); + let cancel_token = tokio_util::sync::CancellationToken::new(); + let (peer, _peer_send) = legacy_gossip_peer(peer_id.clone(), cancel_token.clone()); + + let poisoned_latest_block = outbound.latest_block.clone(); + let _ = std::panic::catch_unwind(move || { + let _guard = poisoned_latest_block + .lock() + .expect("latest-block mutex starts unpoisoned"); + panic!("poison latest-block mutex before replay"); + }); + + sink.add_peer(peer); + wait_for_legacy_gossip_panic_cleanup(&outbound, &peer_id, &cancel_token).await + } + + #[tokio::test] + async fn legacy_gossip_recv_loop_panic_cancels_peer_and_removes_outbound_session( + ) -> Result<(), BoxError> { + let (inbound_tx, _inbound_rx) = mpsc::channel(1); + let outbound = LegacyGossipOutbound::default(); + let sink = LegacyGossipSink { + inbound_tx, + outbound: outbound.clone(), + trace: ZakuraTrace::noop(), + }; + let peer_id = ZakuraPeerId::new(vec![93; 32]).expect("test peer id is within bounds"); + let cancel_token = tokio_util::sync::CancellationToken::new(); + let (peer, peer_send) = legacy_gossip_peer(peer_id.clone(), cancel_token.clone()); + + arm_legacy_gossip_recv_loop_panic(peer_id.clone()); + sink.add_peer(peer); + peer_send + .send(LegacyGossipFrame::AdvertiseBlock(block_hash(93)).encode_frame()?) + .await + .map_err(|_| -> BoxError { "failed to send test gossip frame".into() })?; + + wait_for_legacy_gossip_panic_cleanup(&outbound, &peer_id, &cancel_token).await + } + #[tokio::test] async fn disconnected_peer_send_returns_error() -> Result<(), BoxError> { let peer_id = ZakuraPeerId::new(vec![9; 32]).expect("test peer id is within bounds"); @@ -4031,6 +4535,7 @@ mod tests { 7, Response::Blocks(vec![InventoryResponse::Available((block.clone(), None))]), max_frame_bytes, + max_frame_bytes, )?; assert!(frames.len() > 1, "large block response must be chunked"); @@ -4050,6 +4555,133 @@ mod tests { Ok(()) } + /// Regression test for `claude-outbound-write-ignores-message-cap` (legacy + /// response chunking facet). + /// + /// The handshake clamps `max_frame_bytes` and `max_message_bytes` + /// independently, so a peer can negotiate a message cap well below the frame + /// cap. `push_chunked_response` sized chunk frames against the frame cap + /// alone, so each chunk frame's payload could exceed the peer's accepted + /// `max_message_bytes`. The request-response writer (`write_response_frame`) + /// and the peer both reject such a frame as oversize, wasting the encode and + /// causing avoidable disconnects/interop loss. Chunking must size against the + /// effective cap `min(frame cap, message cap)` so every emitted chunk frame + /// fits the negotiated message cap while still round-tripping. + #[test] + fn encode_response_chunks_respect_message_cap() -> Result<(), BoxError> { + let block = Arc::new(Block::zcash_deserialize( + BLOCK_TESTNET_141042_BYTES.as_slice(), + )?); + + // Large frame cap, small message cap: the divergence the handshake + // permits. The message cap allows a 256-byte chunk payload plus the + // per-chunk response header. + let max_frame_bytes = u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?; + let max_message_bytes = u32::try_from(RESPONSE_CHUNK_HEADER_BYTES + 256)?; + + let frames = LegacyResponseCodec::encode_response( + 7, + Response::Blocks(vec![InventoryResponse::Available((block.clone(), None))]), + max_frame_bytes, + max_message_bytes, + )?; + + assert!( + frames.len() > 1, + "a block larger than the negotiated message cap must be chunked, not emitted as one \ + over-cap frame" + ); + for frame in &frames { + assert!( + frame.payload.len() <= max_message_bytes as usize, + "chunk frame payload {} exceeds the negotiated max_message_bytes {}; the peer \ + (and write_response_frame) would reject it as oversize", + frame.payload.len(), + max_message_bytes, + ); + // Each chunk must still fit the frame cap so the transport encodes it. + frame.encode(max_frame_bytes)?; + } + + // The smaller chunking must still round-trip back to the original block. + let response = LegacyResponseCodec::decode_response(7, LegacyRequestKind::Blocks, frames)?; + let Response::Blocks(blocks) = response else { + panic!("unexpected response: {response:?}"); + }; + assert!(matches!( + blocks.as_slice(), + [InventoryResponse::Available((received, None))] if received.hash() == block.hash() + )); + + Ok(()) + } + + /// Regression test for `claude-legacy-responder-response-aggregation-unbounded`. + /// + /// An authenticated peer can name up to `MAX_TX_INV_IN_SENT_MESSAGE` + /// block/transaction hashes on one request. Without a responder-side + /// aggregate budget, `encode_response` serializes and retains the entire + /// multi-frame `Vec` for every available item before the first byte + /// is written (worst case `MAX_TX_INV_IN_SENT_MESSAGE * + /// MAX_PROTOCOL_MESSAGE_LEN`, tens of GiB). The outbound reader already + /// enforces a symmetric `LegacyResponseBudget`; the responder must too, and + /// must abort encoding early rather than buffering an over-budget response. + #[test] + fn encode_response_aborts_when_aggregation_exceeds_budget() -> Result<(), BoxError> { + let block = Arc::new(Block::zcash_deserialize( + BLOCK_TESTNET_141042_BYTES.as_slice(), + )?); + let block_bytes = block.zcash_serialize_to_vec()?.len(); + let frame_cap = u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?; + + // A single available block is well within budget and still encodes. + LegacyResponseCodec::encode_response( + 1, + Response::Blocks(vec![InventoryResponse::Available((block.clone(), None))]), + frame_cap, + frame_cap, + )?; + + // Enough available blocks to overflow the cumulative byte budget must be + // rejected, not fully materialized as a `Vec`. + let block_copies = LEGACY_RESPONSE_MAX_AGGREGATE_BYTES / block_bytes + 2; + let many_blocks = Response::Blocks( + (0..block_copies) + .map(|_| InventoryResponse::Available((block.clone(), None))) + .collect(), + ); + assert!( + matches!( + LegacyResponseCodec::encode_response(2, many_blocks, frame_cap, frame_cap), + Err(LegacyGossipError::ResponseAggregateBudget(_)), + ), + "an over-budget BlocksByHash response must abort encoding early", + ); + + // The same cumulative budget guards the transaction responder path, + // which shares `push_chunked_response`. + let tx_copies = 2 * LEGACY_RESPONSE_MAX_AGGREGATE_BYTES / block_bytes + 2; + let many_transactions = Response::Transactions( + (0..tx_copies) + .flat_map(|_| { + block + .transactions + .iter() + .map(|tx| InventoryResponse::Available((UnminedTx::from(tx.clone()), None))) + }) + .collect(), + ); + assert!( + matches!( + LegacyResponseCodec::encode_response(3, many_transactions, frame_cap, frame_cap), + Err(LegacyGossipError::ResponseAggregateBudget(_)), + ), + "an over-budget TransactionsById response must abort encoding early", + ); + + Ok(()) + } + #[test] fn response_codec_round_trips_chain_sync_and_mempool_responses() -> Result<(), BoxError> { let block = Arc::new(Block::zcash_deserialize( @@ -4064,6 +4696,7 @@ mod tests { 8, block_hash_response.clone(), u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, + u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, )?; assert_eq!( LegacyResponseCodec::decode_response(8, LegacyRequestKind::FindBlocks, frames)?, @@ -4075,6 +4708,7 @@ mod tests { 9, header_response.clone(), u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, + u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, )?; assert_eq!( LegacyResponseCodec::decode_response(9, LegacyRequestKind::FindHeaders, frames)?, @@ -4086,6 +4720,7 @@ mod tests { 10, tx_ids_response.clone(), u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, + u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, )?; assert_eq!( LegacyResponseCodec::decode_response( @@ -4100,6 +4735,7 @@ mod tests { 11, Response::Pong(Duration::ZERO), u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, + u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, )?; assert!(matches!( LegacyResponseCodec::decode_response(11, LegacyRequestKind::Ping, frames)?, @@ -4110,6 +4746,7 @@ mod tests { 12, Response::Nil, u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, + u32::try_from(MAX_PROTOCOL_MESSAGE_LEN)?, )?; assert_eq!( LegacyResponseCodec::decode_response(12, LegacyRequestKind::PushTransaction, frames)?, diff --git a/zebra-network/src/zakura/testkit/cluster.rs b/zebra-network/src/zakura/testkit/cluster.rs index 88adb4db261..b9da2f539d0 100644 --- a/zebra-network/src/zakura/testkit/cluster.rs +++ b/zebra-network/src/zakura/testkit/cluster.rs @@ -136,15 +136,18 @@ mod tests { use super::super::{trace_reader::TraceValue, HostilePeer, WaitError}; use super::*; use crate::{ - zakura::trace::header_sync_trace as hs_trace, + zakura::trace::{block_sync_trace as bs_trace, header_sync_trace as hs_trace}, zakura::{ - spawn_header_sync_reactor, DiscoveryMessage, Frame, FramedRecv, FramedSend, - HeaderSyncAction, HeaderSyncCommitFailureKind, HeaderSyncEvent, HeaderSyncFrontiers, - HeaderSyncHandle, HeaderSyncMessage, HeaderSyncMisbehavior, HeaderSyncPeerSession, - HeaderSyncStartup, HeaderSyncStatus, Peer, Service, Stream, ZakuraHeaderSyncConfig, - ZakuraLocalLimits, ZakuraTrace, ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_HEADER_SYNC, - ZAKURA_CAP_LEGACY_GOSSIP, ZAKURA_STREAM_DISCOVERY, ZAKURA_STREAM_GOSSIP, - ZAKURA_STREAM_HEADER_SYNC, + block_sync::{MAX_BS_FRAME_BYTES, ZAKURA_CAP_BLOCK_SYNC, ZAKURA_STREAM_BLOCK_SYNC}, + spawn_header_sync_reactor, BlockApplyResult, BlockSizeEstimate, BlockSyncAction, + BlockSyncBlockMeta, BlockSyncEvent, BlockSyncFrontiers, BlockSyncMessage, + BlockSyncStatus, DiscoveryMessage, Frame, FramedRecv, FramedSend, HeaderSyncAction, + HeaderSyncCommitFailureKind, HeaderSyncEvent, HeaderSyncFrontiers, HeaderSyncHandle, + HeaderSyncMessage, HeaderSyncMisbehavior, HeaderSyncPeerSession, HeaderSyncStartup, + HeaderSyncStatus, Peer, Service, ServicePeerLimits, Stream, ZakuraBlockSyncConfig, + ZakuraHeaderSyncConfig, ZakuraLocalLimits, ZakuraTrace, MAX_BS_RESPONSE_BYTES, + ZAKURA_CAP_DISCOVERY, ZAKURA_CAP_HEADER_SYNC, ZAKURA_CAP_LEGACY_GOSSIP, + ZAKURA_STREAM_DISCOVERY, ZAKURA_STREAM_GOSSIP, ZAKURA_STREAM_HEADER_SYNC, }, Config, }; @@ -165,13 +168,21 @@ mod tests { }, Network, }, - serialization::ZcashDeserializeInto, + serialization::{ZcashDeserializeInto, ZcashSerialize}, }; use zebra_test::vectors::{ BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES, BLOCK_MAINNET_3_BYTES, BLOCK_MAINNET_4_BYTES, BLOCK_MAINNET_5_BYTES, BLOCK_MAINNET_GENESIS_BYTES, }; + fn headers_message(headers: Vec>) -> HeaderSyncMessage { + let body_sizes = vec![0; headers.len()]; + HeaderSyncMessage::Headers { + headers, + body_sizes, + } + } + #[derive(Debug, Default)] struct OrderedSourceProbeService { senders: Arc>>, @@ -387,9 +398,15 @@ mod tests { } fn frontiers(&self) -> HeaderSyncFrontiers { + let verified_block_hash = self + .headers + .get(&self.verified_block_tip) + .map(|(hash, _)| *hash) + .expect("verified test frontier has a known header"); HeaderSyncFrontiers { finalized_height: self.finalized_height, verified_block_tip: self.verified_block_tip, + verified_block_hash, } } @@ -827,7 +844,7 @@ mod tests { .handle .send(HeaderSyncEvent::WireMessage { peer: local.peer_id.clone(), - msg: HeaderSyncMessage::Headers(headers), + msg: headers_message(headers), }) .await; let _ = local @@ -847,6 +864,7 @@ mod tests { start_height, headers, finalized, + .. } => { let count = u32::try_from(headers.len()).unwrap_or(u32::MAX); let result = local @@ -917,6 +935,8 @@ mod tests { HeaderSyncAction::BodyGaps { from, to } => { local.observed_gaps.lock().await.push((from, to)); } + HeaderSyncAction::HeaderAdvanced { .. } => {} + HeaderSyncAction::HeaderReanchored { .. } => {} HeaderSyncAction::NewBlockReceived { peer, height, @@ -969,6 +989,98 @@ mod tests { Arc::new(bytes.zcash_deserialize_into().expect("block vector parses")) } + fn block_size(block: &block::Block) -> u32 { + u32::try_from( + block + .zcash_serialize_to_vec() + .expect("test block serializes") + .len(), + ) + .expect("test block size fits u32") + } + + async fn drive_native_block_sync_actions( + node: &ZakuraTestNode, + blocks: Vec>, + submitted: Arc>>, + ) -> JoinHandle<()> { + let endpoint = node.endpoint(); + let mut actions = node + .take_block_sync_actions() + .await + .expect("block-sync action receiver is enabled"); + let by_height: BTreeMap<_, _> = blocks + .into_iter() + .map(|block| { + ( + block.coinbase_height().expect("test block has height"), + block, + ) + }) + .collect(); + + tokio::spawn(async move { + while let Some(action) = actions.recv().await { + let Some(handle) = endpoint.block_sync() else { + continue; + }; + match action { + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + let metas = by_height + .range( + verified_block_tip.next().unwrap_or(verified_block_tip) + ..=best_header_tip, + ) + .map(|(height, block)| BlockSyncBlockMeta { + height: *height, + hash: block.hash(), + size: BlockSizeEstimate::Advertised(block_size(block)), + }) + .collect(); + let _ = handle.send(BlockSyncEvent::NeededBlocks(metas)).await; + } + BlockSyncAction::SubmitBlock { token, block } => { + let height = block.coinbase_height().expect("submitted block has height"); + submitted + .lock() + .expect("submitted list mutex is not poisoned") + .push(height); + let _ = handle + .send(BlockSyncEvent::BlockApplyFinished { + token, + height, + hash: block.hash(), + result: BlockApplyResult::Committed, + local_frontier: Some(BlockSyncFrontiers { + finalized_height: height, + verified_block_tip: height, + verified_block_hash: block.hash(), + }), + }) + .await; + } + BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => { + let _ = handle + .send(BlockSyncEvent::BlockRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + BlockSyncAction::Misbehavior { peer, .. } => { + let _ = endpoint.supervisor().disconnect_peer(&peer).await; + } + BlockSyncAction::SendMessage { .. } => {} + } + } + }) + } + fn block_bytes(height: u32) -> &'static [u8] { match height { 1 => &BLOCK_MAINNET_1_BYTES, @@ -1027,6 +1139,7 @@ mod tests { nu5: None, nu6: None, nu6_1: None, + nu6_2: None, nu7: None, #[cfg(zcash_unstable = "zfuture")] zfuture: None, @@ -1058,6 +1171,7 @@ mod tests { nu5: None, nu6: None, nu6_1: None, + nu6_2: None, nu7: None, #[cfg(zcash_unstable = "zfuture")] zfuture: None, @@ -1089,6 +1203,7 @@ mod tests { HeaderSyncFrontiers { finalized_height: block::Height(0), verified_block_tip: block::Height(0), + verified_block_hash: anchor.1, }, Some(anchor), ) @@ -1161,7 +1276,9 @@ mod tests { } HeaderSyncAction::QueryBestHeaderTip | HeaderSyncAction::QueryMissingBlockBodies { .. } - | HeaderSyncAction::BodyGaps { .. } => {} + | HeaderSyncAction::BodyGaps { .. } + | HeaderSyncAction::HeaderAdvanced { .. } + | HeaderSyncAction::HeaderReanchored { .. } => {} } } }) @@ -1233,6 +1350,325 @@ mod tests { Ok(()) } + #[tokio::test] + async fn native_stream6_oversize_frame_is_traceable_over_real_connection( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "native_stream6_oversize_frame_is_traceable_over_real_connection", + false, + )?; + let mut cluster = ZakuraTestCluster::new(); + let victim_idx = cluster.spawn_traced_node(1, &mut capture).await?; + let victim = cluster.node(victim_idx); + let hostile = + HostilePeer::connect_native_with_capabilities(victim, 2, ZAKURA_CAP_BLOCK_SYNC).await?; + let hostile_peer = hostile.id()?; + let peer_set = victim.supervisor().subscribe(); + + await_until("block-sync peer registered", Duration::from_secs(5), || { + peer_set.borrow().contains(&hostile_peer) + }) + .await?; + assert!( + MAX_BS_FRAME_BYTES < victim.limits().max_frame_bytes, + "test payload must fit the negotiated connection cap but exceed stream-6's cap" + ); + hostile + .send_frame_header_with_declared_payload_len( + ZAKURA_STREAM_BLOCK_SYNC, + MAX_BS_FRAME_BYTES, + ) + .await?; + + await_until("stream-6 oversize trace", Duration::from_secs(5), || { + capture.reader().is_ok_and(|reader| { + reader + .node("01") + .table("ratelimit") + .rows() + .iter() + .any(|row| { + row.get("event").and_then(serde_json::Value::as_str) + == Some("frame.oversize") + && row.get("stream_kind").and_then(serde_json::Value::as_str) + == Some("block_sync") + }) + }) + }) + .await?; + await_until( + "oversized stream-6 frame disconnects peer", + Duration::from_secs(5), + || !peer_set.borrow().contains(&hostile_peer), + ) + .await?; + + hostile.shutdown().await; + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test] + async fn native_stream6_declared_payload_above_old_cap_is_not_raw_oversize( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "native_stream6_declared_payload_above_old_cap_is_not_raw_oversize", + false, + )?; + let mut cluster = ZakuraTestCluster::new(); + let victim_idx = cluster.spawn_traced_node(1, &mut capture).await?; + let victim = cluster.node(victim_idx); + let hostile = + HostilePeer::connect_native_with_capabilities(victim, 2, ZAKURA_CAP_BLOCK_SYNC).await?; + let hostile_peer = hostile.id()?; + let peer_set = victim.supervisor().subscribe(); + + await_until("block-sync peer registered", Duration::from_secs(5), || { + peer_set.borrow().contains(&hostile_peer) + }) + .await?; + + let old_max_bs_message_bytes = + u32::try_from(block::MAX_BLOCK_BYTES).expect("max block bytes fits in u32") + 1; + let regression_payload_len = old_max_bs_message_bytes + 1; + assert!( + regression_payload_len < MAX_BS_FRAME_BYTES, + "test payload must exceed the old stream-6 cap but fit the new one" + ); + + hostile + .send_frame_header_with_declared_payload_len( + ZAKURA_STREAM_BLOCK_SYNC, + regression_payload_len, + ) + .await?; + + await_until( + "incomplete stream-6 frame disconnects peer", + Duration::from_secs(5), + || !peer_set.borrow().contains(&hostile_peer), + ) + .await?; + + let reader = capture.reader()?; + let raw_oversize = reader + .node("01") + .table("ratelimit") + .rows() + .iter() + .any(|row| { + row.get("event").and_then(serde_json::Value::as_str) == Some("frame.oversize") + && row.get("stream_kind").and_then(serde_json::Value::as_str) + == Some("block_sync") + }); + assert!( + !raw_oversize, + "payloads above the old stream-6 cap but below the new cap must reach \ + the stream payload reader instead of being dropped as raw frame.oversize" + ); + + hostile.shutdown().await; + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + + #[tokio::test] + async fn native_block_sync_getblocks_flushes_before_hostile_peer_sends_bodies( + ) -> Result<(), BoxError> { + let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "native_block_sync_getblocks_flushes_before_hostile_peer_sends_bodies", + false, + )?; + let blocks = vec![ + mainnet_block(&BLOCK_MAINNET_1_BYTES), + mainnet_block(&BLOCK_MAINNET_2_BYTES), + mainnet_block(&BLOCK_MAINNET_3_BYTES), + ]; + + let mut limits = ZakuraLocalLimits::from_config(&Config::default()); + limits.max_connections = 16; + limits.max_pending_handshakes = 8; + limits.max_open_streams = 16; + limits.max_inbound_queue_depth = 8; + limits.message_rate_per_second = 64; + limits.stream_open_rate_per_second = 64; + + let block_sync_config = ZakuraBlockSyncConfig { + near_tip_body_download_pause_blocks: 0, + max_blocks_per_response: 3, + max_inflight_block_bytes: u64::MAX, + request_timeout: Duration::from_secs(300), + peer_limits: ServicePeerLimits { + inbound_queue_depth: 1, + outbound_queue_depth: 1, + ..ServicePeerLimits::default() + }, + ..ZakuraBlockSyncConfig::default() + }; + + let anchor = (block::Height(0), mainnet_genesis_hash()); + let mut cluster = ZakuraTestCluster::new(); + let victim = ZakuraTestNode::builder(60) + .limits(limits) + .tracer(capture.tracer_for_node(60)) + .header_sync_driver( + e2e_network([3]), + anchor, + HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: anchor.1, + }, + Some((block::Height(3), blocks[2].hash())), + ) + .block_sync_config(block_sync_config) + .spawn() + .await?; + cluster.nodes.push(victim); + let victim = cluster.node(0); + assert!( + victim.block_sync().is_some(), + "block-sync handle should be enabled with the header-sync test driver" + ); + + let submitted = Arc::new(StdMutex::new(Vec::new())); + let driver = + drive_native_block_sync_actions(victim, blocks.clone(), submitted.clone()).await; + let hostile = + HostilePeer::connect_native_with_capabilities(victim, 61, ZAKURA_CAP_BLOCK_SYNC) + .await?; + let hostile_peer = hostile.id()?; + let peer_set = victim.supervisor().subscribe(); + await_until("block-sync peer registered", Duration::from_secs(5), || { + peer_set.borrow().contains(&hostile_peer) + }) + .await?; + + hostile + .send_raw_frame( + ZAKURA_STREAM_BLOCK_SYNC, + BlockSyncMessage::Status(BlockSyncStatus { + servable_low: block::Height(1), + servable_high: block::Height(3), + tip_hash: blocks[2].hash(), + max_blocks_per_response: 3, + max_inflight_requests: 1, + max_response_bytes: MAX_BS_RESPONSE_BYTES, + }) + .encode_frame()?, + ) + .await?; + + let (start_height, count) = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let frame = hostile.recv_ordered_frame(ZAKURA_STREAM_BLOCK_SYNC).await?; + match BlockSyncMessage::decode_frame(frame) + .map_err(|error| -> BoxError { Box::new(error) })? + { + BlockSyncMessage::GetBlocks { + start_height, + count, + } => return Ok::<_, BoxError>((start_height, count)), + BlockSyncMessage::Status(_) => {} + msg => { + return Err(format!("unexpected native block-sync message: {msg:?}").into()) + } + } + } + }) + .await + .map_err(|_| -> BoxError { + "timed out waiting for physical stream-6 GetBlocks frame".into() + })??; + + assert_eq!(start_height, block::Height(1)); + assert_eq!(count, 3); + assert!( + submitted + .lock() + .expect("submitted list mutex is not poisoned") + .is_empty(), + "the test-side responder must not send bodies or trigger submissions before it has \ + physically read GetBlocks from stream 6" + ); + + let end_height = start_height + .0 + .checked_add(count) + .expect("test request height range fits u32"); + for height in start_height.0..end_height { + let block = blocks + .iter() + .find(|block| block.coinbase_height() == Some(block::Height(height))) + .expect("requested test block exists") + .clone(); + hostile + .send_raw_frame( + ZAKURA_STREAM_BLOCK_SYNC, + BlockSyncMessage::Block(block).encode_frame()?, + ) + .await?; + } + hostile + .send_raw_frame( + ZAKURA_STREAM_BLOCK_SYNC, + BlockSyncMessage::BlocksDone { + start_height, + returned: count, + } + .encode_frame()?, + ) + .await?; + + let expected: Vec<_> = (start_height.0..end_height).map(block::Height).collect(); + await_until( + "native block-sync submitted requested bodies", + Duration::from_secs(5), + || { + let mut actual = submitted + .lock() + .expect("submitted list mutex is not poisoned") + .clone(); + actual.sort_unstable(); + actual.dedup(); + expected.iter().all(|height| actual.contains(height)) + }, + ) + .await?; + + await_until( + "native block-sync submitted trace rows", + Duration::from_secs(5), + || { + capture.reader().is_ok_and(|reader| { + let rows = reader.node("60").table("block_sync").rows(); + expected.iter().all(|height| { + rows.iter().any(|row| { + row.get(bs_trace::EVENT).and_then(serde_json::Value::as_str) + == Some(bs_trace::BLOCK_BODY_SUBMITTED) + && row + .get(bs_trace::HEIGHT) + .and_then(serde_json::Value::as_u64) + == Some(u64::from(height.0)) + }) + }) + }) + }, + ) + .await?; + + driver.abort(); + hostile.shutdown().await; + cluster.shutdown().await; + assert!(capture.finish().await?.is_none()); + Ok(()) + } + #[tokio::test] async fn unknown_stream_kind_is_reset_and_never_delivered() -> Result<(), BoxError> { // FLUP-015: a peer-controlled prelude naming an unknown kind must be @@ -1615,6 +2051,8 @@ mod tests { let flooding = HostilePeer::connect_native_with_capabilities(&victim, 30, ZAKURA_CAP_DISCOVERY) .await?; + // Exceeding the per-kind message rate is traced at transport ingress + // before the ordered stream is disconnected. flooding .flood_stream(ZAKURA_STREAM_DISCOVERY, 'd', 16) .await?; @@ -1671,9 +2109,15 @@ mod tests { #[tokio::test] async fn persistent_ordered_stream_uses_message_budget() -> Result<(), BoxError> { - // P2: a long-lived ordered stream still spends the transport-owned - // per-kind message-rate budget before frames reach the service. + // P2: a long-lived ordered stream spends the transport-owned per-kind + // message-rate budget before frames reach the service. A peer that + // floods past the budget is disconnected (we never drop a solicited + // frame), so no more than ~one budget of frames is ever delivered. let _guard = zebra_test::init(); + let mut capture = TraceCapture::for_test_with_keep_override( + "persistent_ordered_stream_uses_message_budget", + false, + )?; // Small, deterministic message budget so the aggregate cap is observable // without sending hundreds of frames. @@ -1687,7 +2131,11 @@ mod tests { limits.stream_open_rate_per_second = 64; let message_budget = limits.message_rate_per_second as usize; - let victim = ZakuraTestNode::builder(5).limits(limits).spawn().await?; + let victim = ZakuraTestNode::builder(5) + .limits(limits) + .tracer(capture.tracer_for_node(5)) + .spawn() + .await?; let recorder = victim.recorder(); let hostile = HostilePeer::connect_native_with_capabilities(&victim, 6, ZAKURA_CAP_LEGACY_GOSSIP) @@ -1695,15 +2143,33 @@ mod tests { let sent = message_budget * 8; for index in 0..sent { - hostile + // Once the budget is exceeded the victim disconnects, so later + // sends race the teardown and may error -- that is expected. + if hostile .send_frame(2, format!("a-{index}").into_bytes()) - .await?; + .await + .is_err() + { + break; + } } // Wait until rate limiting has clearly engaged (more frames sent than one // budget, so the bucket must have emptied at least once). await_until("rate limiting engaged", Duration::from_secs(5), || { - recorder.len() + recorder.dropped_count() >= message_budget + capture.reader().is_ok_and(|reader| { + reader + .node("05") + .table("ratelimit") + .rows() + .iter() + .any(|row| { + row.get("event").and_then(serde_json::Value::as_str) + == Some("message.throttled") + && row.get("stream_kind").and_then(serde_json::Value::as_str) + == Some("gossip") + }) + }) }) .await?; // Brief deterministic settle to let one refill window pass. A correct @@ -1723,6 +2189,7 @@ mod tests { hostile.shutdown().await; victim.shutdown().await; + assert!(capture.finish().await?.is_none()); Ok(()) } @@ -2008,7 +2475,7 @@ mod tests { HostilePeer::connect_native_with_capabilities(&victim, 13, ZAKURA_CAP_HEADER_SYNC) .await?; let unsolicited_headers = - HeaderSyncMessage::Headers(vec![mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone()]) + headers_message(vec![mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone()]) .encode_frame()?; unsolicited .send_raw_frame(ZAKURA_STREAM_HEADER_SYNC, unsolicited_headers) @@ -2068,6 +2535,25 @@ mod tests { .await?; truncated.shutdown().await; + // Each misbehavior disconnect cancels the peer's connection token, so the + // victim's connection handler exits via the cancellation arm and stamps + // the `closed.neutral` row with the bounded `cancelled` reason. + await_until( + "victim traces closed.neutral with a bounded reason", + Duration::from_secs(5), + || { + capture.reader().is_ok_and(|reader| { + reader.node("11").table("conn").rows().iter().any(|row| { + row.get("event").and_then(serde_json::Value::as_str) + == Some("closed.neutral") + && row.get("reason").and_then(serde_json::Value::as_str) + == Some("cancelled") + }) + }) + }, + ) + .await?; + capture.flush().await; let reader = capture.reader()?; let header_sync = reader.node("11").table("header_sync"); @@ -2081,6 +2567,10 @@ mod tests { "frame.oversize", &[("stream_kind", TraceValue::Str("header_sync"))], ); + reader.node("11").table("conn").assert_row( + "closed.neutral", + &[("reason", TraceValue::Str("cancelled"))], + ); victim.shutdown().await; driver.abort(); @@ -2431,9 +2921,7 @@ mod tests { .inject( victim, unsolicited, - HeaderSyncMessage::Headers(vec![mainnet_block(&BLOCK_MAINNET_1_BYTES) - .header - .clone()]), + headers_message(vec![mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone()]), ) .await; cluster @@ -2452,9 +2940,7 @@ mod tests { .inject( victim, out_of_range, - HeaderSyncMessage::Headers(vec![mainnet_block(&BLOCK_MAINNET_2_BYTES) - .header - .clone()]), + headers_message(vec![mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone()]), ) .await; cluster @@ -2496,7 +2982,7 @@ mod tests { .inject( victim, response_too_long, - HeaderSyncMessage::Headers(vec![ + headers_message(vec![ mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone(), ]), @@ -2534,7 +3020,7 @@ mod tests { .inject( bad_continuity_victim, bad_continuity, - HeaderSyncMessage::Headers(vec![ + headers_message(vec![ mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), Arc::new(non_contiguous), ]), @@ -2566,7 +3052,7 @@ mod tests { .inject( bad_pow_victim, bad_pow, - HeaderSyncMessage::Headers(vec![Arc::new(bad_pow_header)]), + headers_message(vec![Arc::new(bad_pow_header)]), ) .await; cluster @@ -2599,7 +3085,7 @@ mod tests { .inject( bad_daa_victim, bad_daa, - HeaderSyncMessage::Headers(vec![ + headers_message(vec![ mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone(), mainnet_block(&BLOCK_MAINNET_3_BYTES).header.clone(), @@ -2639,7 +3125,7 @@ mod tests { .inject( checkpointed, bad_checkpoint_backfill, - HeaderSyncMessage::Headers(vec![ + headers_message(vec![ mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone(), mainnet_block(&BLOCK_MAINNET_3_BYTES).header.clone(), diff --git a/zebra-network/src/zakura/testkit/hostile.rs b/zebra-network/src/zakura/testkit/hostile.rs index ff71516f4e4..5a5135cd44b 100644 --- a/zebra-network/src/zakura/testkit/hostile.rs +++ b/zebra-network/src/zakura/testkit/hostile.rs @@ -10,9 +10,11 @@ use super::{LocalEndpointFactory, ZakuraTestNode}; use crate::{ zakura::{ legacy_gossip::ZAKURA_STREAM_GOSSIP, run_native_initiator_handshake, Frame, StreamPrelude, - ZakuraHandshakeConfig, ZakuraLocalLimits, ZakuraPeerId, FRAME_HEADER_BYTES, P2P_V2_ALPN, - STREAM_PRELUDE_MAGIC, ZAKURA_CAP_HEADER_SYNC, ZAKURA_CAP_LEGACY_GOSSIP, - ZAKURA_STREAM_DISCOVERY, ZAKURA_STREAM_HEADER_SYNC, + ZakuraHandshakeConfig, ZakuraLocalLimits, ZakuraPeerId, FRAME_HEADER_BYTES, + LEGACY_GOSSIP_VERSION, P2P_V2_ALPN, STREAM_PRELUDE_MAGIC, ZAKURA_BLOCK_SYNC_STREAM_VERSION, + ZAKURA_CAP_HEADER_SYNC, ZAKURA_CAP_LEGACY_GOSSIP, ZAKURA_DISCOVERY_STREAM_VERSION, + ZAKURA_HEADER_SYNC_STREAM_VERSION, ZAKURA_STREAM_BLOCK_SYNC, ZAKURA_STREAM_DISCOVERY, + ZAKURA_STREAM_HEADER_SYNC, }, BoxError, Config, }; @@ -89,7 +91,10 @@ impl HostilePeer { pub async fn send_raw_frame(&self, stream_kind: u16, frame: Frame) -> Result<(), BoxError> { if matches!( stream_kind, - ZAKURA_STREAM_GOSSIP | ZAKURA_STREAM_DISCOVERY | ZAKURA_STREAM_HEADER_SYNC + ZAKURA_STREAM_GOSSIP + | ZAKURA_STREAM_DISCOVERY + | ZAKURA_STREAM_HEADER_SYNC + | ZAKURA_STREAM_BLOCK_SYNC ) { return self.send_ordered_raw_frame(stream_kind, frame).await; } @@ -268,15 +273,25 @@ impl HostilePeer { /// Send a frame header whose declared length exceeds the victim cap. pub async fn oversize_frame_declared_len(&self, stream_kind: u16) -> Result<(), BoxError> { + self.send_frame_header_with_declared_payload_len( + stream_kind, + self.limits.max_frame_bytes.saturating_add(1), + ) + .await + } + + /// Send a frame header with an explicit declared payload length and no payload. + pub async fn send_frame_header_with_declared_payload_len( + &self, + stream_kind: u16, + declared_payload_len: u32, + ) -> Result<(), BoxError> { let (mut send, _recv) = self.connection.open_bi().await?; self.write_prelude(&mut send, stream_kind).await?; let mut header = Vec::with_capacity(FRAME_HEADER_BYTES); WriteBytesExt::write_u16::(&mut header, 1)?; WriteBytesExt::write_u16::(&mut header, 0)?; - WriteBytesExt::write_u32::( - &mut header, - self.limits.max_frame_bytes.saturating_add(1), - )?; + WriteBytesExt::write_u32::(&mut header, declared_payload_len)?; send.write_all(&header).await?; let _ = send.finish(); Ok(()) @@ -311,7 +326,7 @@ impl HostilePeer { let prelude = StreamPrelude { magic: STREAM_PRELUDE_MAGIC, stream_kind, - stream_version: 1, + stream_version: Self::stream_version(stream_kind), request_id: None, max_frame_bytes: self.limits.max_frame_bytes, }; @@ -319,6 +334,16 @@ impl HostilePeer { Ok(()) } + fn stream_version(stream_kind: u16) -> u16 { + match stream_kind { + ZAKURA_STREAM_GOSSIP => LEGACY_GOSSIP_VERSION, + ZAKURA_STREAM_DISCOVERY => ZAKURA_DISCOVERY_STREAM_VERSION, + ZAKURA_STREAM_HEADER_SYNC => ZAKURA_HEADER_SYNC_STREAM_VERSION, + ZAKURA_STREAM_BLOCK_SYNC => ZAKURA_BLOCK_SYNC_STREAM_VERSION, + _ => 1, + } + } + async fn read_prelude(recv: &mut RecvStream) -> Result { let mut bytes = vec![0; 4 + 2 + 2 + 1]; recv.read_exact(&mut bytes).await?; @@ -343,10 +368,107 @@ impl HostilePeer { let mut reader = std::io::Cursor::new(&header); let _message_type = reader.read_u16::()?; let _flags = reader.read_u16::()?; - let payload_len = reader.read_u32::()?; - let mut payload = vec![0; usize::try_from(payload_len)?]; + let payload_len = usize::try_from(reader.read_u32::()?)?; + // Mirror production `read_frame`/`Frame::decode`: reject a declared frame + // larger than the cap BEFORE allocating the payload buffer. Without this + // a malicious or fuzzed responder can declare a huge payload_len and force + // an unbounded allocation (OOM/hang) in the harness response reader before + // the bounded `Frame::decode` rejection is ever reached. + let frame_len = FRAME_HEADER_BYTES.saturating_add(payload_len); + if frame_len > usize::try_from(max_frame_bytes)? { + return Err(format!( + "frame payload length {payload_len} exceeds max_frame_bytes {max_frame_bytes}" + ) + .into()); + } + let mut payload = vec![0; payload_len]; recv.read_exact(&mut payload).await?; header.extend_from_slice(&payload); Ok(Frame::decode(&header, max_frame_bytes)?) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + use iroh::protocol::{AcceptError, ProtocolHandler, Router}; + + const TEST_ALPN: &[u8] = b"/zakura/testkit/hostile-read-frame/0"; + const MAX_FRAME_BYTES: u32 = 4096; + /// Declared payload length far exceeding the cap. The unfixed reader + /// allocates this many bytes (then blocks reading a payload that never + /// arrives) before the `max_frame_bytes` check; a correct reader rejects on + /// the declared length first and never allocates it. + const DECLARED_PAYLOAD_LEN: u32 = 64 * 1024 * 1024; + + /// Malicious responder: opens a stream, writes only a frame header that + /// declares a payload far larger than the receiver's cap, then holds the + /// stream open without ever sending the payload. + #[derive(Clone, Debug)] + struct OversizeResponder; + + impl ProtocolHandler for OversizeResponder { + async fn accept(&self, connection: Connection) -> Result<(), AcceptError> { + if let Ok((mut send, _recv)) = connection.open_bi().await { + let mut header = Vec::with_capacity(FRAME_HEADER_BYTES); + header.extend_from_slice(&1u16.to_le_bytes()); // message_type + header.extend_from_slice(&0u16.to_le_bytes()); // flags + header.extend_from_slice(&DECLARED_PAYLOAD_LEN.to_le_bytes()); // payload_len + let _ = send.write_all(&header).await; + // Never send the declared payload; keep the stream open so a + // reader that allocates/reads the payload before the cap check + // blocks here. + tokio::time::sleep(Duration::from_secs(8)).await; + } + Ok(()) + } + } + + /// `read_frame` must reject a frame whose declared length exceeds + /// `max_frame_bytes` BEFORE allocating/reading the payload, so a malicious or + /// fuzzed responder cannot drive an unbounded allocation (or block) in the + /// harness response reader. + #[tokio::test] + async fn read_frame_rejects_oversize_declared_len_before_allocating_payload( + ) -> Result<(), BoxError> { + let server = LocalEndpointFactory::new().endpoint(4040).await?; + let router = Router::builder(server) + .accept(TEST_ALPN, OversizeResponder) + .spawn(); + let server_addr = LocalEndpointFactory::node_addr(router.endpoint()).await; + + let client = LocalEndpointFactory::new().endpoint(4041).await?; + client.add_node_addr(server_addr.clone())?; + let connection = client.connect(server_addr, TEST_ALPN).await?; + + let (_send, mut recv) = connection.accept_bi().await?; + + // A correct reader compares the declared length to `max_frame_bytes` and + // rejects before allocating/reading the payload, so this returns promptly. + // The unfixed reader allocates `DECLARED_PAYLOAD_LEN` bytes and then blocks + // reading a payload the responder never sends, so it times out here. + let outcome = tokio::time::timeout( + Duration::from_secs(2), + HostilePeer::read_frame(&mut recv, MAX_FRAME_BYTES), + ) + .await + .expect( + "read_frame must reject the oversize declared length before allocating/reading the \ + payload; a timeout here means payload_len was allocated/read before the \ + max_frame_bytes check", + ); + + let err = outcome.expect_err("oversize declared frame length must be rejected"); + let message = err.to_string(); + assert!( + message.contains("max_frame_bytes"), + "expected a max_frame_bytes rejection, got: {message}", + ); + + connection.close(0u32.into(), b"done"); + client.close().await; + Ok(()) + } +} diff --git a/zebra-network/src/zakura/testkit/mod.rs b/zebra-network/src/zakura/testkit/mod.rs index babdf8a46ed..594c5b59f7a 100644 --- a/zebra-network/src/zakura/testkit/mod.rs +++ b/zebra-network/src/zakura/testkit/mod.rs @@ -29,6 +29,6 @@ pub use pinned::{ }; pub use recorder::{InboundRecorder, RecordedInbound}; pub use trace_capture::TraceCapture; -pub use trace_reader::{TraceQuery, TraceReader}; +pub use trace_reader::{TraceQuery, TraceReader, TraceValue}; pub use wait::{await_until, WaitError}; pub use zebra_jsonl_trace::JsonlTracer; diff --git a/zebra-network/src/zakura/testkit/node.rs b/zebra-network/src/zakura/testkit/node.rs index 390b1813a13..2c40b26235c 100644 --- a/zebra-network/src/zakura/testkit/node.rs +++ b/zebra-network/src/zakura/testkit/node.rs @@ -13,11 +13,13 @@ use zebra_jsonl_trace::JsonlTracer; use super::{InboundRecorder, LocalEndpointFactory, WaitError}; use crate::{ zakura::{ - discovery::build_discovery_handle, service_registry, spawn_header_sync_reactor, - DiscoveryService, HeaderSyncAction, HeaderSyncFrontiers, HeaderSyncHandle, - HeaderSyncStartup, Service, ZakuraDiscoveryHandle, ZakuraEndpoint, ZakuraHandshakeConfig, - ZakuraHeaderSyncConfig, ZakuraLocalLimits, ZakuraPeerId, ZakuraProtocolHandler, - ZakuraServiceId, ZakuraSupervisorHandle, ZakuraTrace, P2P_V2_ALPN, + discovery::build_discovery_handle, service_registry, spawn_block_sync_reactor, + spawn_header_sync_reactor, BlockSyncAction, BlockSyncFrontiers, BlockSyncHandle, + BlockSyncStartup, DiscoveryService, HeaderSyncAction, HeaderSyncFrontiers, + HeaderSyncHandle, HeaderSyncStartup, Service, ZakuraBlockSyncConfig, ZakuraDiscoveryHandle, + ZakuraEndpoint, ZakuraHandshakeConfig, ZakuraHeaderSyncConfig, ZakuraLocalLimits, + ZakuraPeerId, ZakuraProtocolHandler, ZakuraServiceId, ZakuraSupervisorHandle, ZakuraTrace, + P2P_V2_ALPN, }, BoxError, Config, }; @@ -57,6 +59,7 @@ impl ZakuraTestNode { } /// Clone the underlying endpoint for test-only external drivers. + #[cfg(test)] pub(crate) fn endpoint(&self) -> ZakuraEndpoint { self.endpoint.clone() } @@ -67,12 +70,24 @@ impl ZakuraTestNode { self.endpoint.header_sync() } + /// Active block-sync handle, if this test node was spawned with stream-6 + /// block sync enabled. + pub fn block_sync(&self) -> Option { + self.endpoint.block_sync() + } + /// Take the stream-5 header-sync action receiver for an externally driven /// test node. pub async fn take_header_sync_actions(&self) -> Option> { self.endpoint.take_header_sync_actions().await } + /// Take the stream-6 block-sync action receiver for an externally driven + /// test node. + pub async fn take_block_sync_actions(&self) -> Option> { + self.endpoint.take_block_sync_actions().await + } + /// Local limits used by this node. pub fn limits(&self) -> &ZakuraLocalLimits { &self.limits @@ -188,6 +203,7 @@ async fn wait_for_peer_registration( pub struct ZakuraTestNodeBuilder { seed: u64, limits: ZakuraLocalLimits, + max_connections_per_ip: usize, transport_config: Option, legacy_upgrade: bool, tracer: JsonlTracer, @@ -196,6 +212,7 @@ pub struct ZakuraTestNodeBuilder { discovery_direct_addrs: Vec, extra_advertised_services: Vec, header_sync: Option, + block_sync_config: ZakuraBlockSyncConfig, } #[derive(Clone, Debug)] @@ -204,6 +221,7 @@ struct TestHeaderSyncStartup { anchor: (block::Height, block::Hash), frontiers: HeaderSyncFrontiers, best_header_tip: Option<(block::Height, block::Hash)>, + verified_block_tip_hash: block::Hash, } impl fmt::Debug for ZakuraTestNodeBuilder { @@ -211,6 +229,7 @@ impl fmt::Debug for ZakuraTestNodeBuilder { f.debug_struct("ZakuraTestNodeBuilder") .field("seed", &self.seed) .field("limits", &self.limits) + .field("max_connections_per_ip", &self.max_connections_per_ip) .field("transport_config", &self.transport_config.is_some()) .field("legacy_upgrade", &self.legacy_upgrade) .field("tracer", &self.tracer) @@ -234,6 +253,7 @@ impl ZakuraTestNodeBuilder { Self { seed, limits, + max_connections_per_ip: Config::default().max_connections_per_ip, transport_config: None, legacy_upgrade: false, tracer: JsonlTracer::noop(), @@ -242,6 +262,7 @@ impl ZakuraTestNodeBuilder { discovery_direct_addrs: Vec::new(), extra_advertised_services: Vec::new(), header_sync: None, + block_sync_config: ZakuraBlockSyncConfig::default(), } } @@ -263,6 +284,19 @@ impl ZakuraTestNodeBuilder { self } + /// Override the per-IP connection cap enforced by this node's supervisor. + /// + /// Defaults to the production [`Config::max_connections_per_ip`] (1) so that + /// security and integration tests built on the default node exercise the + /// real per-IP admission gate instead of silently admitting many same-IP + /// peers. Multi-peer loopback harnesses — where every node shares + /// `127.0.0.1`, so the per-IP cap collapses to a single bucket — raise this + /// to restore cluster ergonomics. + pub fn max_connections_per_ip(mut self, max_connections_per_ip: usize) -> Self { + self.max_connections_per_ip = max_connections_per_ip; + self + } + /// Mutate the transport configuration used by the endpoint factory. pub fn transport(mut self, configure: impl FnOnce(&mut TransportConfig)) -> Self { let mut transport = self.limits.transport_config(); @@ -312,10 +346,17 @@ impl ZakuraTestNodeBuilder { anchor, frontiers, best_header_tip, + verified_block_tip_hash: anchor.1, }); self } + /// Override the block-sync config used with [`Self::header_sync_driver`]. + pub fn block_sync_config(mut self, config: ZakuraBlockSyncConfig) -> Self { + self.block_sync_config = config; + self + } + /// Spawn the node. pub async fn spawn(self) -> Result { if self.legacy_upgrade { @@ -331,7 +372,7 @@ impl ZakuraTestNodeBuilder { let endpoint = LocalEndpointFactory::with_transport_config(transport) .endpoint(self.seed) .await?; - let supervisor = ZakuraSupervisorHandle::new(self.limits.max_connections); + let supervisor = ZakuraSupervisorHandle::new(self.max_connections_per_ip); let recorder = InboundRecorder::new(usize::from(self.limits.max_inbound_queue_depth)); let base_service = if let Some(factory) = self.service_factory { factory(supervisor.clone()) @@ -354,13 +395,22 @@ impl ZakuraTestNodeBuilder { let mut header_sync_handle = None; let mut header_sync_actions = None; + let mut block_sync_handle = None; + let mut block_sync_actions = None; let mut header_sync_tasks = Vec::new(); let header_sync = if let Some(header_sync) = self.header_sync { + let TestHeaderSyncStartup { + network, + anchor, + frontiers, + best_header_tip, + verified_block_tip_hash, + } = header_sync; let mut startup = HeaderSyncStartup::new( - header_sync.network, - header_sync.anchor, - header_sync.frontiers, - header_sync.best_header_tip, + network, + anchor, + frontiers, + best_header_tip, ZakuraHeaderSyncConfig::default(), self.limits.max_frame_bytes, ); @@ -375,6 +425,29 @@ impl ZakuraTestNodeBuilder { header_sync_tasks.push(task); header_sync_actions = Some((shutdown, actions)); header_sync_handle = Some(handle.clone()); + + let mut startup = BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: frontiers.finalized_height, + verified_block_tip: frontiers.verified_block_tip, + verified_block_hash: verified_block_tip_hash, + }, + best_header_tip.unwrap_or(anchor), + handle.subscribe_tip(), + self.block_sync_config.clone(), + ); + let shutdown = header_sync_actions + .as_ref() + .expect("header sync actions were just initialized") + .0 + .clone(); + startup.shutdown = shutdown; + startup.trace = ZakuraTrace::new(self.tracer.clone(), seed_label(self.seed)); + let (block_handle, actions, task) = spawn_block_sync_reactor(startup); + header_sync_tasks.push(task); + block_sync_actions = Some(actions); + block_sync_handle = Some(block_handle.clone()); + Some(handle) } else { // Recorder-only nodes use the stream-5 passthrough so tests can @@ -382,14 +455,22 @@ impl ZakuraTestNodeBuilder { None }; let discovery_service = if let Some(header_sync) = header_sync.as_ref() { - Arc::new(DiscoveryService::with_header_sync( + Arc::new(DiscoveryService::with_sync_services( discovery.clone(), header_sync.clone(), + block_sync_handle.clone(), )) as Arc } else { Arc::new(DiscoveryService::new(discovery.clone())) as Arc }; - let registry = service_registry(&supervisor, header_sync, base_service, discovery_service)?; + let registry = service_registry( + &supervisor, + header_sync, + block_sync_handle.clone(), + self.block_sync_config.clone(), + base_service, + discovery_service, + )?; let handler = ZakuraProtocolHandler::new_with_registry_and_trace( supervisor.clone(), network.clone(), @@ -401,17 +482,19 @@ impl ZakuraTestNodeBuilder { let router = Router::builder(endpoint) .accept(P2P_V2_ALPN, handler.clone()) .spawn(); - let endpoint = if let (Some(handle), Some((shutdown, actions))) = - (header_sync_handle, header_sync_actions) + let endpoint = if let (Some(header_handle), Some(block_handle), Some((shutdown, actions))) = + (header_sync_handle, block_sync_handle, header_sync_actions) { - ZakuraEndpoint::from_parts_with_header_sync( + ZakuraEndpoint::from_parts_with_sync_services( router, supervisor, handler, - handle, + header_handle, + block_handle, shutdown, header_sync_tasks, Some(actions), + block_sync_actions, ) } else { ZakuraEndpoint::from_parts(router, supervisor, handler) @@ -457,4 +540,74 @@ mod tests { assert!(error.to_string().contains("connect_via_upgrade")); } + + #[tokio::test] + async fn default_test_node_uses_production_per_ip_cap() { + // Every loopback test node binds 127.0.0.1, so the supervisor's per-IP + // cap collapses all peers into one IP bucket. A default test node must + // inherit the production per-IP cap (Config::max_connections_per_ip == 1) + // so security/integration tests built on it exercise the real per-IP + // admission gate. Before the fix the builder seeded the supervisor with + // max_connections (16), so a second same-IP peer was wrongly admitted and + // per-IP admission bugs could pass silently. + let peer1 = ZakuraTestNode::builder(9001) + .spawn() + .await + .expect("first loopback peer spawns"); + let peer2 = ZakuraTestNode::builder(9002) + .spawn() + .await + .expect("second loopback peer spawns"); + let node = ZakuraTestNode::builder(9003) + .spawn() + .await + .expect("default test node spawns"); + + node.connect_native(&peer1, Duration::from_secs(5)) + .await + .expect("first same-IP outbound peer registers under per-IP cap 1"); + let second = node.connect_native(&peer2, Duration::from_secs(5)).await; + assert!( + second.is_err(), + "second same-IP outbound dial must be rejected under the production \ + per-IP cap of 1, but it registered — the test node is not enforcing \ + production per-IP admission" + ); + + node.shutdown().await; + peer1.shutdown().await; + peer2.shutdown().await; + } + + #[tokio::test] + async fn per_ip_cap_opt_out_admits_multiple_same_ip_peers() { + // Multi-peer loopback harnesses (clusters, gossip meshes) intentionally + // admit several same-IP peers and do not exercise the per-IP gate. The + // explicit builder opt-out restores that ergonomics on top of the + // production-faithful default. + let peer1 = ZakuraTestNode::builder(9101) + .spawn() + .await + .expect("first loopback peer spawns"); + let peer2 = ZakuraTestNode::builder(9102) + .spawn() + .await + .expect("second loopback peer spawns"); + let node = ZakuraTestNode::builder(9103) + .max_connections_per_ip(8) + .spawn() + .await + .expect("opt-out test node spawns"); + + node.connect_native(&peer1, Duration::from_secs(5)) + .await + .expect("first same-IP peer registers with raised per-IP cap"); + node.connect_native(&peer2, Duration::from_secs(5)) + .await + .expect("second same-IP peer registers with raised per-IP cap"); + + node.shutdown().await; + peer1.shutdown().await; + peer2.shutdown().await; + } } diff --git a/zebra-network/src/zakura/testkit/trace_capture.rs b/zebra-network/src/zakura/testkit/trace_capture.rs index 60402ae9089..a49e79c2cbd 100644 --- a/zebra-network/src/zakura/testkit/trace_capture.rs +++ b/zebra-network/src/zakura/testkit/trace_capture.rs @@ -199,7 +199,14 @@ mod tests { sync::{Arc, Mutex}, }; - use crate::zakura::{ZakuraTrace, ZakuraTraceEvent, CONN_TABLE}; + use crate::zakura::{ + trace::{ + block_sync_trace as bs_trace, commit_state_trace as cs_trace, + header_sync_trace as hs_trace, BLOCK_SYNC_TABLE, COMMIT_STATE_TABLE, CONN_TABLE, + HEADER_SYNC_TABLE, + }, + ZakuraTrace, ZakuraTraceEvent, + }; use super::*; @@ -304,4 +311,65 @@ mod tests { let _ = fs::remove_dir_all(persisted); } + + #[tokio::test] + async fn zakura_sync_trace_tables_write_separate_files() { + let mut capture = TraceCapture::for_test_with_keep_override( + "zakura_sync_trace_tables_write_separate_files", + false, + ) + .unwrap(); + let trace = ZakuraTrace::new(capture.tracer(), "01"); + + trace.emit_with(BLOCK_SYNC_TABLE, |row| { + row.insert( + bs_trace::EVENT.to_string(), + serde_json::Value::String("block_test".to_string()), + ); + }); + trace.emit_with(HEADER_SYNC_TABLE, |row| { + row.insert( + hs_trace::EVENT.to_string(), + serde_json::Value::String("header_test".to_string()), + ); + }); + trace.emit_with(COMMIT_STATE_TABLE, |row| { + row.insert( + cs_trace::EVENT.to_string(), + serde_json::Value::String("commit_test".to_string()), + ); + }); + + capture.flush().await; + + assert!(capture.path().join(BLOCK_SYNC_TABLE.file_name()).exists()); + assert!(capture.path().join(HEADER_SYNC_TABLE.file_name()).exists()); + assert!(capture.path().join(COMMIT_STATE_TABLE.file_name()).exists()); + + let reader = capture.reader().unwrap(); + assert_eq!( + reader.table(BLOCK_SYNC_TABLE.table()).count("block_test"), + 1 + ); + assert_eq!( + reader.table(HEADER_SYNC_TABLE.table()).count("header_test"), + 1 + ); + assert_eq!( + reader + .table(COMMIT_STATE_TABLE.table()) + .count("commit_test"), + 1 + ); + assert_eq!( + reader.table(BLOCK_SYNC_TABLE.table()).count("commit_test"), + 0 + ); + assert_eq!( + reader.table(HEADER_SYNC_TABLE.table()).count("commit_test"), + 0 + ); + + let _ = capture.finish().await.unwrap(); + } } diff --git a/zebra-network/src/zakura/trace.rs b/zebra-network/src/zakura/trace.rs index 83933ecb85a..3e321041ceb 100644 --- a/zebra-network/src/zakura/trace.rs +++ b/zebra-network/src/zakura/trace.rs @@ -1,10 +1,12 @@ //! Structured JSONL trace helpers for Zakura P2P. //! //! `closed.punitive` is reserved in the schema until Zakura has a punitive close -//! path and matching metric. Connection admission rejects use -//! `rejected.admission` with a bounded `reason` label rather than one event name -//! per rejection metric, so readers should pivot by `event` plus `reason` for -//! those rows. +//! path and matching metric. `closed.neutral` carries a bounded `reason` label +//! describing the teardown cause (for example `idle_timeout`, `accept_failed`, +//! `outbound_closed`, `bad_response`, or `cancelled`). Connection admission +//! rejects use `rejected.admission` with a bounded `reason` label rather than +//! one event name per rejection metric, so readers should pivot by `event` plus +//! `reason` for those rows. use std::{ sync::Arc, @@ -72,22 +74,140 @@ pub const LEGACY_REQUEST_TABLE: ZakuraTraceTable = ZakuraTraceTable { file_name: "legacy_request.jsonl", }; +/// Block-sync (stream-6) scheduling, download, submit, and commit events. +pub const BLOCK_SYNC_TABLE: ZakuraTraceTable = ZakuraTraceTable { + table: "block_sync", + file_name: "block_sync.jsonl", +}; + +/// Zebrad adapter boundary events for commits, state reads, and frontier mirrors. +pub const COMMIT_STATE_TABLE: ZakuraTraceTable = ZakuraTraceTable { + table: "commit_state", + file_name: "commit_state.jsonl", +}; + +/// Shared block-sync trace event names and field keys. +/// +/// The block-sync body pipeline has no `tracing`-macro coverage in release +/// builds (the binary is compiled with `release_max_level_info`, which strips +/// the `debug!` sites), so these JSONL rows are the only runtime visibility into +/// scheduling, download, submit, and commit progress. The periodic +/// [`BLOCK_SYNC_STATE`] snapshot is the single most useful row for diagnosing a +/// stall: it reports where the body floor, verified tip, and header tip are, how +/// much is buffered/applying, and whether the byte budget or peer status is +/// blocking new downloads. +pub mod block_sync_trace { + /// Trace row event field. + pub const EVENT: &str = "event"; + /// Peer field. + pub const PEER: &str = "peer"; + /// Action/event/message kind field. + pub const KIND: &str = "kind"; + /// Height field. + pub const HEIGHT: &str = "height"; + /// Hash field. + pub const HASH: &str = "hash"; + /// Range start height field. + pub const RANGE_START: &str = "range_start"; + /// Range count field. + pub const RANGE_COUNT: &str = "range_count"; + /// Expected/requested count field. + pub const EXPECTED_COUNT: &str = "expected_count"; + /// Estimated byte reservation for a requested range. + pub const ESTIMATED_BYTES: &str = "estimated_bytes"; + /// Serialized byte size of a received body. + pub const SERIALIZED_BYTES: &str = "serialized_bytes"; + /// Commit result label (`committed`, `duplicate`, `rejected`, `timed_out`). + pub const RESULT: &str = "result"; + /// Reactor-local verifier submission token. + pub const APPLY_TOKEN: &str = "apply_token"; + /// Bounded reason field. + pub const REASON: &str = "reason"; + /// Highest contiguous body height already submitted for apply. + pub const BODY_DOWNLOAD_FLOOR: &str = "body_download_floor"; + /// Highest verified (committed) block-body height. + pub const VERIFIED_BLOCK_TIP: &str = "verified_block_tip"; + /// Best header tip driving the body-download target. + pub const BEST_HEADER_TIP: &str = "best_header_tip"; + /// Header tip minus verified body tip. + pub const BODY_LAG: &str = "body_lag"; + /// Count of blocks submitted-but-not-yet-committed (held against budget). + pub const APPLYING: &str = "applying"; + /// Count of out-of-order bodies buffered awaiting a contiguous prefix. + pub const REORDER: &str = "reorder"; + /// Count of outstanding (in-flight) range requests across peers. + pub const OUTSTANDING: &str = "outstanding"; + /// Remaining in-flight body byte budget. + pub const BUDGET_AVAILABLE: &str = "budget_available"; + /// Reserved in-flight body byte budget. + pub const BUDGET_RESERVED: &str = "budget_reserved"; + /// Connected block-sync peers. + pub const PEERS: &str = "peers"; + /// Connected block-sync peers whose status we have received (schedulable). + pub const PEERS_WITH_STATUS: &str = "peers_with_status"; + /// Lowest height still in the body-sync `needed` set (the gap to fetch next). + pub const NEEDED_MIN: &str = "needed_min"; + /// Number of heights in the body-sync `needed` set after buffer filtering. + pub const NEEDED_COUNT: &str = "needed_count"; + /// Number of ranges queued in the scheduler. + pub const QUEUE_LEN: &str = "queue_len"; + /// Lowest start height across queued scheduler ranges. + pub const QUEUE_MIN_START: &str = "queue_min_start"; + /// Number of distinct assigned range keys in the scheduler. + pub const ASSIGNED_LEN: &str = "assigned_len"; + /// Highest end height across the scheduler's covered intervals. + pub const COVERED_MAX_END: &str = "covered_max_end"; + + /// Peer status received (servable body range advertised by the peer). + pub const BLOCK_STATUS_RECEIVED: &str = "block_status_received"; + /// Body range request sent to a peer. + pub const BLOCK_GET_BLOCKS_SENT: &str = "block_get_blocks_sent"; + /// Reactor accepted an inbound event. + pub const BLOCK_EVENT_RECEIVED: &str = "block_event_received"; + /// Reactor queued an outbound driver action. + pub const BLOCK_ACTION_DISPATCHED: &str = "block_action_dispatched"; + /// Body received from a peer. + pub const BLOCK_BODY_RECEIVED: &str = "block_body_received"; + /// Body submitted to the verifier for commit. + pub const BLOCK_BODY_SUBMITTED: &str = "block_body_submitted"; + /// Verifier finished applying a submitted body. + pub const BLOCK_APPLY_FINISHED: &str = "block_apply_finished"; + /// Peer reported a requested range as unavailable. + pub const BLOCK_RANGE_UNAVAILABLE: &str = "block_range_unavailable"; + /// New body downloads were paused (lag, near-tip, or budget). + pub const BLOCK_DOWNLOADS_PAUSED: &str = "block_downloads_paused"; + /// Verified body frontier advanced from state. + pub const BLOCK_FRONTIERS_CHANGED: &str = "block_frontiers_changed"; + /// Chain tip reset rolled the body frontier back. + pub const BLOCK_CHAIN_TIP_RESET: &str = "block_chain_tip_reset"; + /// Periodic reactor state snapshot (the key stall-diagnosis row). + pub const BLOCK_SYNC_STATE: &str = "block_sync_state"; +} + /// Shared header-sync trace event names and field keys. pub mod header_sync_trace { /// Trace row event field. pub const EVENT: &str = "event"; /// Peer field. pub const PEER: &str = "peer"; + /// Action/event/message kind field. + pub const KIND: &str = "kind"; /// Source peer field for forwarded full-block floods. pub const SOURCE_PEER: &str = "source_peer"; /// Height field. pub const HEIGHT: &str = "height"; /// Hash field. pub const HASH: &str = "hash"; + /// Header anchor hash field. + pub const ANCHOR_HASH: &str = "anchor_hash"; /// Range start height field. pub const RANGE_START: &str = "range_start"; /// Range count field. pub const RANGE_COUNT: &str = "range_count"; + /// Header validation stage field. + pub const VALIDATION_STAGE: &str = "validation_stage"; + /// Concrete validation error kind field. + pub const ERROR_KIND: &str = "error_kind"; /// Advertised peer range cap field. pub const ADVERTISED_CAP: &str = "advertised_cap"; /// Expected header count field. @@ -99,6 +219,10 @@ pub mod header_sync_trace { /// Bounded reason field. pub const REASON: &str = "reason"; + /// Reactor accepted an inbound event. + pub const HEADER_EVENT_RECEIVED: &str = "header_event_received"; + /// Reactor queued an outbound driver action. + pub const HEADER_ACTION_DISPATCHED: &str = "header_action_dispatched"; /// Local status sent to a peer. pub const HEADER_STATUS_SENT: &str = "header_status_sent"; /// Peer status received. @@ -125,10 +249,121 @@ pub mod header_sync_trace { pub const HEADER_PEER_DISCONNECT_REQUESTED: &str = "header_peer_disconnect_requested"; /// Header frontier advanced. pub const HEADER_FRONTIER_ADVANCED: &str = "header_frontier_advanced"; + /// Header frontier re-anchored down to the verified block frontier. + pub const HEADER_FRONTIER_REANCHORED: &str = "header_frontier_reanchored"; /// Missing block bodies reported. pub const HEADER_MISSING_BODIES_REPORTED: &str = "header_missing_bodies_reported"; } +/// Shared commit/frontier adapter trace event names and field keys. +pub mod commit_state_trace { + /// Trace row event field. + pub const EVENT: &str = "event"; + /// Source driver/subsystem field. + pub const SOURCE: &str = "source"; + /// Height field. + pub const HEIGHT: &str = "height"; + /// Hash field. + pub const HASH: &str = "hash"; + /// Range start height field. + pub const RANGE_START: &str = "range_start"; + /// Range count field. + pub const RANGE_COUNT: &str = "range_count"; + /// Result label field. + pub const RESULT: &str = "result"; + /// Bounded reason field. + pub const REASON: &str = "reason"; + /// Reactor-local block apply token field. + pub const APPLY_TOKEN: &str = "apply_token"; + /// Apply class field (`checkpoint` or `full`). + pub const APPLY_CLASS: &str = "apply_class"; + /// Finalized height observed from state. + pub const FINALIZED_HEIGHT: &str = "finalized_height"; + /// Verified full-block/body tip height. + pub const VERIFIED_BLOCK_TIP: &str = "verified_block_tip"; + /// Verified full-block/body tip hash. + pub const VERIFIED_BLOCK_HASH: &str = "verified_block_hash"; + /// Best header tip height. + pub const BEST_HEADER_TIP: &str = "best_header_tip"; + /// Elapsed milliseconds field. + pub const ELAPSED_MS: &str = "elapsed_ms"; + /// Peer field. + pub const PEER: &str = "peer"; + /// Queue length field. + pub const QUEUE_LEN: &str = "queue_len"; + /// In-flight count field. + pub const IN_FLIGHT_COUNT: &str = "in_flight_count"; + /// Action kind field. + pub const ACTION: &str = "action"; + /// Whether an optional frontier was present. + pub const LOCAL_FRONTIER: &str = "local_frontier"; + + /// Driver received a reactor action. + pub const ACTION_RECEIVED: &str = "action_received"; + /// State read started. + pub const STATE_READ_START: &str = "state_read_start"; + /// State read completed successfully. + pub const STATE_READ_SUCCESS: &str = "state_read_success"; + /// State read failed or returned an unexpected response. + pub const STATE_READ_ERROR: &str = "state_read_error"; + /// State read timed out. + pub const STATE_READ_TIMEOUT: &str = "state_read_timeout"; + /// Block submit was queued in the driver. + pub const BLOCK_SUBMIT_QUEUED: &str = "block_submit_queued"; + /// Verifier commit started. + pub const COMMIT_START: &str = "commit_start"; + /// Verifier commit exceeded the driver timeout but is still being awaited. + pub const COMMIT_STALLED: &str = "commit_stalled"; + /// Verifier commit finished. + pub const COMMIT_FINISH: &str = "commit_finish"; + /// Post-commit frontier query started. + pub const FRONTIER_QUERY_START: &str = "frontier_query_start"; + /// Post-commit frontier query finished. + pub const FRONTIER_QUERY_FINISH: &str = "frontier_query_finish"; + /// Driver sent an event back to a reactor. + pub const REACTOR_EVENT_SENT: &str = "reactor_event_sent"; + /// Delayed checkpoint frontier refresh attempted. + pub const CHECKPOINT_REFRESH_ATTEMPT: &str = "checkpoint_refresh_attempt"; + /// Delayed checkpoint frontier refresh sent a frontier event. + pub const CHECKPOINT_REFRESH_SENT: &str = "checkpoint_refresh_sent"; + /// Header-sync driver notified block sync about a header tip. + pub const BLOCK_SYNC_NOTIFY_SENT: &str = "block_sync_notify_sent"; + /// Chain-tip mirror observed a watch action. + pub const CHAIN_TIP_ACTION: &str = "chain_tip_action"; + /// Chain-tip mirror derived local frontiers. + pub const FRONTIER_DERIVED: &str = "frontier_derived"; + /// Shared sync exchange accepted or ignored a frontier update. + pub const SYNC_FRONTIER_TRANSITION: &str = "sync_frontier_transition"; + /// Monotonic shared sync exchange transition sequence. + pub const SEQUENCE: &str = "sequence"; + /// Shared sync exchange transition cause. + pub const CAUSE: &str = "cause"; + /// Previous finalized frontier height. + pub const OLD_FINALIZED_HEIGHT: &str = "old_finalized_height"; + /// Previous finalized frontier hash. + pub const OLD_FINALIZED_HASH: &str = "old_finalized_hash"; + /// Previous verified body frontier height. + pub const OLD_VERIFIED_BODY_HEIGHT: &str = "old_verified_body_height"; + /// Previous verified body frontier hash. + pub const OLD_VERIFIED_BODY_HASH: &str = "old_verified_body_hash"; + /// Previous best header frontier height. + pub const OLD_BEST_HEADER_HEIGHT: &str = "old_best_header_height"; + /// Previous best header frontier hash. + pub const OLD_BEST_HEADER_HASH: &str = "old_best_header_hash"; + /// New finalized frontier height. + pub const NEW_FINALIZED_HEIGHT: &str = "new_finalized_height"; + /// New finalized frontier hash. + pub const NEW_FINALIZED_HASH: &str = "new_finalized_hash"; + /// New verified body frontier height. + pub const NEW_VERIFIED_BODY_HEIGHT: &str = "new_verified_body_height"; + /// New verified body frontier hash. + pub const NEW_VERIFIED_BODY_HASH: &str = "new_verified_body_hash"; + /// New best header frontier height. + pub const NEW_BEST_HEADER_HEIGHT: &str = "new_best_header_height"; + /// New best header frontier hash. + pub const NEW_BEST_HEADER_HASH: &str = "new_best_header_hash"; +} + /// Cloneable Zakura trace emitter. #[derive(Clone, Debug)] pub struct ZakuraTrace { @@ -204,6 +439,9 @@ pub struct ZakuraTraceEvent<'a> { event: &'static str, conn: Option, stream: Option, + payload_len: Option, + frame_len: Option, + max_frame_bytes: Option, peer: Option<&'a str>, role: Option<&'static str>, phase: Option<&'static str>, @@ -221,6 +459,9 @@ impl<'a> ZakuraTraceEvent<'a> { event, conn: None, stream: None, + payload_len: None, + frame_len: None, + max_frame_bytes: None, peer: None, role: None, phase: None, @@ -244,6 +485,24 @@ impl<'a> ZakuraTraceEvent<'a> { self } + /// Attach a declared frame payload length. + pub fn payload_len(mut self, payload_len: u64) -> Self { + self.payload_len = Some(payload_len); + self + } + + /// Attach an encoded frame length. + pub fn frame_len(mut self, frame_len: u64) -> Self { + self.frame_len = Some(frame_len); + self + } + + /// Attach the effective frame byte cap used by the receiver. + pub fn max_frame_bytes(mut self, max_frame_bytes: u64) -> Self { + self.max_frame_bytes = Some(max_frame_bytes); + self + } + /// Attach a bounded peer label. pub fn peer(mut self, peer: &'a str) -> Self { self.peer = Some(peer); @@ -302,6 +561,9 @@ impl<'a> ZakuraTraceEvent<'a> { row.insert("event".to_string(), Value::String(self.event.to_string())); insert_optional_u64(row, "conn", self.conn); insert_optional_u64(row, "stream", self.stream); + insert_optional_u64(row, "payload_len", self.payload_len); + insert_optional_u64(row, "frame_len", self.frame_len); + insert_optional_u64(row, "max_frame_bytes", self.max_frame_bytes); insert_optional_str(row, "peer", self.peer); insert_optional_str(row, "role", self.role); insert_optional_str(row, "phase", self.phase); diff --git a/zebra-network/src/zakura/transport/guard.rs b/zebra-network/src/zakura/transport/guard.rs new file mode 100644 index 00000000000..e94051265bf --- /dev/null +++ b/zebra-network/src/zakura/transport/guard.rs @@ -0,0 +1,283 @@ +//! Shared per-session protections for Zakura services. +//! +//! [`SessionGuard`] is the single home for the protections that wrap a peer +//! session: the allowed inbound message-type filter, the optional byte budget, +//! and the per-peer semantic meters. It reuses the existing limiter primitives +//! rather than inventing new limiter math. +//! +//! Boundary note (do not double-count). The transport's per-connection, +//! per-stream-kind message-rate `TokenBucket` and the oversize check already +//! run in the transport stream worker *before* a frame reaches the service. +//! `SessionGuard` therefore owns only the **service-specific** protections +//! (allowed-types, byte budget, per-peer semantic meters); the transport keeps +//! its connection-global count bucket exactly as-is. Document this split at the +//! [`SessionGuard::new`] call site. + +use super::Frame; + +/// Byte-rate reservation budget for inflight stream payloads. +/// +/// Promoted here from `block_sync/state.rs` so byte-rate protection is reusable +/// across services; only block_sync currently passes `Some(..)` to a guard. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) struct ByteBudget { + max_bytes: u64, + reserved_bytes: u64, +} + +impl ByteBudget { + pub(crate) fn new(max_bytes: u64) -> Self { + Self { + max_bytes, + reserved_bytes: 0, + } + } + + pub(crate) fn available(self) -> u64 { + self.max_bytes.saturating_sub(self.reserved_bytes) + } + + pub(crate) fn reserved(self) -> u64 { + self.reserved_bytes + } + + pub(crate) fn try_reserve(&mut self, bytes: u64) -> bool { + if bytes == 0 || bytes > self.available() { + return false; + } + self.reserved_bytes = self.reserved_bytes.saturating_add(bytes); + true + } + + pub(crate) fn release(&mut self, bytes: u64) { + self.reserved_bytes = self.reserved_bytes.saturating_sub(bytes); + } +} + +/// Outcome of admitting one inbound frame through a [`SessionGuard`]. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum Admit { + /// The frame is admitted for processing. + Pass, + /// The frame is dropped but the peer is kept (rate/budget back-pressure). + Throttle, + /// The peer violated a protocol-level protection and should be disconnected. + Reject(&'static str), +} + +/// Per-peer semantic meters (status spam, new-block spam, ...). +/// +/// Minimal in Phase 0: a pass-through that admits everything. The real +/// per-service semantic meters (the `RateMeter`s currently in the per-peer +/// state) are moved here in a later migration phase. +#[derive(Debug)] +pub(crate) struct PeerMeters; + +impl PeerMeters { + pub(crate) fn new() -> Self { + Self + } + + pub(crate) fn try_take(&mut self, _message_type: u8) -> bool { + true + } +} + +/// The single home for the protections that wrap one peer session. +#[derive(Debug)] +pub(crate) struct SessionGuard { + /// Allowed inbound message types for this stream kind. + allowed: &'static [u8], + /// Maximum reassembled message size in bytes. + max_bytes: u32, + /// Optional byte budget; `None` for services without byte-rate protection. + byte_budget: Option, + /// Per-peer semantic meters. + meters: PeerMeters, +} + +impl SessionGuard { + /// Build a session guard for one peer stream. + /// + /// Boundary note (do not double-count): the transport already applies the + /// per-connection, per-stream-kind message-rate bucket and the oversize cap + /// before frames reach this guard. Pass `Some(..)` for `byte_budget` only + /// when this service owns byte-rate protection (block_sync); header_sync and + /// others pass `None`. + /// + /// Phase 1 header_sync uses [`SessionGuard::oversize_only`] (the allowed-type + /// filter stays off so the decode stage remains the sole validity arbiter); + /// the explicit-`allowed` constructor is consumed when block_sync/discovery + /// move their allowed-type lists onto the guard in later phases. + #[allow(dead_code)] // consumed when block_sync/discovery move their type filters onto the guard + pub(crate) fn new( + allowed: &'static [u8], + max_bytes: u32, + byte_budget: Option, + ) -> Self { + Self { + allowed, + max_bytes, + byte_budget, + meters: PeerMeters::new(), + } + } + + /// Build a guard that applies only the oversize cap and admits every type. + /// + /// This is the behavior-preserving configuration for a service that has not + /// yet moved its allowed-type filter and per-peer semantic meters behind the + /// guard: the decode stage remains the sole arbiter of message validity, so + /// the exact same wire events fire as before the lift. The allowed-type + /// filter (`ALL_TYPES`) and the meters are wired per-service in later phases; + /// `byte_budget` stays `None` because only block_sync owns byte-rate + /// protection. + pub(crate) fn oversize_only(max_bytes: u32) -> Self { + // An empty `allowed` slot would reject every type; `ALL_TYPES` admits + // every `u8` discriminator so type validity is left to the decode stage. + const ALL_TYPES: &[u8] = &{ + let mut all = [0u8; 256]; + let mut ty = 0usize; + while ty < all.len() { + // `ty` ranges 0..=255 so the `as u8` truncation is exact. + all[ty] = ty as u8; + ty += 1; + } + all + }; + Self { + allowed: ALL_TYPES, + max_bytes, + byte_budget: None, + meters: PeerMeters::new(), + } + } + + /// Admit one inbound frame through the service-specific protections. + pub(crate) fn admit(&mut self, frame: &Frame) -> Admit { + // `message_type` is a wire `u16`; allowed message types are single + // bytes, so a value that does not fit in a `u8` is by definition not an + // allowed type and is treated as a protocol violation. + let Ok(ty) = u8::try_from(frame.message_type) else { + return Admit::Reject("bad type"); + }; + if !self.allowed.contains(&ty) { + return Admit::Reject("disallowed type"); + } + // `max_bytes` is a `u32`; widen to `usize` for the payload-length + // comparison (`usize` is at least 32 bits on supported targets). + if frame.payload.len() > self.max_bytes as usize { + return Admit::Reject("oversize"); + } + if let Some(budget) = &mut self.byte_budget { + // Payload length fits in `u64` on all supported targets. + if !budget.try_reserve(frame.payload.len() as u64) { + return Admit::Throttle; + } + } + if !self.meters.try_take(ty) { + return Admit::Throttle; + } + Admit::Pass + } + + /// Return reserved bytes to the byte budget once a message is processed. + /// + /// Unused until a service with a byte budget (block_sync) moves onto the + /// guard; header_sync passes `byte_budget: None`, so it never calls this. + #[allow(dead_code)] // consumed when block_sync moves its byte budget onto the guard + pub(crate) fn release(&mut self, bytes: u64) { + if let Some(budget) = &mut self.byte_budget { + budget.release(bytes); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ALLOWED: &[u8] = &[1, 2]; + + fn frame(message_type: u16, payload_len: usize) -> Frame { + Frame { + message_type, + flags: 0, + payload: vec![0u8; payload_len], + } + } + + #[test] + fn byte_budget_reserves_and_releases() { + let mut budget = ByteBudget::new(1_000); + assert_eq!(budget.available(), 1_000); + assert!(budget.try_reserve(400)); + assert_eq!(budget.reserved(), 400); + assert_eq!(budget.available(), 600); + // Zero-byte and over-budget reservations are rejected without mutation. + assert!(!budget.try_reserve(0)); + assert!(!budget.try_reserve(601)); + assert_eq!(budget.reserved(), 400); + budget.release(400); + assert_eq!(budget.reserved(), 0); + } + + #[test] + fn admit_rejects_disallowed_type() { + let mut guard = SessionGuard::new(ALLOWED, 1_024, None); + assert_eq!(guard.admit(&frame(99, 0)), Admit::Reject("disallowed type")); + } + + #[test] + fn admit_rejects_non_u8_type() { + let mut guard = SessionGuard::new(ALLOWED, 1_024, None); + assert_eq!(guard.admit(&frame(0x0100, 0)), Admit::Reject("bad type")); + } + + #[test] + fn admit_rejects_oversize() { + let mut guard = SessionGuard::new(ALLOWED, 4, None); + assert_eq!(guard.admit(&frame(1, 5)), Admit::Reject("oversize")); + } + + #[test] + fn admit_throttles_when_budget_exhausted() { + let mut guard = SessionGuard::new(ALLOWED, 1_024, Some(ByteBudget::new(8))); + // First frame reserves 8 bytes and passes. + assert_eq!(guard.admit(&frame(1, 8)), Admit::Pass); + // Second frame cannot reserve and is throttled (peer kept). + assert_eq!(guard.admit(&frame(1, 8)), Admit::Throttle); + // Releasing the budget admits the next frame again. + guard.release(8); + assert_eq!(guard.admit(&frame(1, 8)), Admit::Pass); + } + + #[test] + fn admit_passes_allowed_under_caps() { + let mut guard = SessionGuard::new(ALLOWED, 1_024, None); + assert_eq!(guard.admit(&frame(1, 16)), Admit::Pass); + assert_eq!(guard.admit(&frame(2, 16)), Admit::Pass); + } + + #[test] + fn oversize_only_admits_all_u8_types_under_cap() { + let mut guard = SessionGuard::oversize_only(1_024); + // Every single-byte discriminator is admitted, including ones no + // service knows about, so decode stays the arbiter of type validity. + assert_eq!(guard.admit(&frame(0, 0)), Admit::Pass); + assert_eq!(guard.admit(&frame(99, 16)), Admit::Pass); + assert_eq!(guard.admit(&frame(255, 16)), Admit::Pass); + } + + #[test] + fn oversize_only_rejects_non_u8_type() { + let mut guard = SessionGuard::oversize_only(1_024); + assert_eq!(guard.admit(&frame(0x0100, 0)), Admit::Reject("bad type")); + } + + #[test] + fn oversize_only_rejects_oversize() { + let mut guard = SessionGuard::oversize_only(4); + assert_eq!(guard.admit(&frame(1, 5)), Admit::Reject("oversize")); + } +} diff --git a/zebra-network/src/zakura/transport/mod.rs b/zebra-network/src/zakura/transport/mod.rs index 2837332ba1d..c6f16840593 100644 --- a/zebra-network/src/zakura/transport/mod.rs +++ b/zebra-network/src/zakura/transport/mod.rs @@ -5,14 +5,25 @@ mod clock; mod frame; +mod guard; mod io; +mod pipe; mod registry; mod service; mod session; pub use clock::{Clock, RealClock}; pub use frame::{Frame, StreamPrelude, ZakuraTrace}; +// `SessionGuard` and `ByteBudget` are imported through this re-export; `Admit` +// and `PeerMeters` are reached directly via the `guard` module path, so they are +// unused *here* until a later service imports them through `crate::zakura`. +#[allow(unused_imports)] +pub(crate) use guard::{Admit, ByteBudget, PeerMeters, SessionGuard}; pub use io::{framed_channel, FramedRecv, FramedSend}; +pub(crate) use pipe::{ + handle_pipe_exit, spawn_supervised_peer_task, spawn_supervised_pipe, Edge, Flow, Node, + NodeKind, Pipe, PipeCx, PipeShape, +}; pub use registry::{RegistryError, ServiceRegistry}; pub(crate) use service::ServiceStream; pub use service::{ diff --git a/zebra-network/src/zakura/transport/pipe.rs b/zebra-network/src/zakura/transport/pipe.rs new file mode 100644 index 00000000000..6e51d9e8b42 --- /dev/null +++ b/zebra-network/src/zakura/transport/pipe.rs @@ -0,0 +1,666 @@ +//! The core per-peer pipe vocabulary shared by every Zakura service. +//! +//! A *pipe* is the single async function that drives one peer's ordered stream +//! from connect to disconnect. This module owns the shared abstraction — the +//! per-stage control flow ([`Flow`]), the per-peer context ([`PipeCx`]), the +//! checked-documentation DAG ([`PipeShape`]), the generic inbound runner +//! ([`Pipe`]/[`PipeSink`]), and the panic-containment launcher +//! ([`spawn_supervised_pipe`]). Services depend on this vocabulary but never +//! re-implement it. +//! +//! Anti-framework rule: [`PipeShape`] is documentation that is *checked*, never +//! an interpreter that is *executed*. Stages are plain `fn`s; the hot path is a +//! `match`, not a walk over `PipeShape.edges`. + +use std::future::Future; + +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use super::{Frame, FramedRecv, SinkReject}; +use crate::zakura::transport::guard::{Admit, SessionGuard}; +use crate::zakura::ZakuraPeerId; + +/// Per-stage control flow. +/// +/// Continue with a value, finish this frame cleanly (throttled/deduped — *not* +/// an error), or reject the peer. +pub(crate) enum Flow { + /// Continue the traversal carrying `T`. + Continue(T), + /// Finish this frame cleanly without rejecting the peer. + Done, + /// Reject the peer; the connection should close. + Reject(SinkReject), +} + +/// Per-peer context handed to every stage by reference. +/// +/// Local state is owned here (`S`); shared state is reached through `env`. +pub(crate) struct PipeCx<'a, S, Env> { + /// Authenticated identity of the peer this pipe drives. + pub(crate) peer_id: &'a ZakuraPeerId, + /// The per-peer state machine — no locking. + pub(crate) local: &'a mut S, + /// Arc-cloneable shared environment handle. + pub(crate) env: &'a Env, +} + +/// One inbound frame's whole traversal — the only fn-ptr indirection per frame. +pub(crate) type PipeEntry = fn(&mut PipeCx<'_, S, Env>, Frame) -> Flow<()>; + +/// A node kind in a [`PipeShape`] DAG. +/// +/// `Validate`/`Outcome`-style effect nodes and the `Stage`/`branch!`/`Outcome` +/// scaffolding for them are deliberately absent: today every service forwards a +/// decoded `WireMessage` to its compatibility reactor, so the live shapes only +/// use these variants. The effect-returning `Core` migration (see the pipelines +/// plan) reintroduces exactly what it needs when it lands. +#[derive(Copy, Clone, Debug)] +pub(crate) enum NodeKind { + /// The shared protection stage. + Guard, + /// `Frame` to typed message. + Decode, + /// The single `match` on the typed message. + Branch, + /// Local/shared state mutation producing an effect. + Mutate, + /// Effect execution (the async sends/actions/fanout). + Emit, +} + +/// One node in a [`PipeShape`] DAG. +#[derive(Copy, Clone, Debug)] +// `PIPE_SHAPE` constructs these nodes in non-test lib code, but the fields are +// only *read* by the per-service `pipe_shape_matches_runtime` drift test, so the +// lib build still sees them as unread. +#[allow(dead_code)] +pub(crate) struct Node { + /// Stable node identifier, referenced by [`Edge`]s. + pub(crate) id: &'static str, + /// The kind of work this node performs. + pub(crate) kind: NodeKind, +} + +/// One directed edge in a [`PipeShape`] DAG. +#[derive(Copy, Clone, Debug)] +// Read only by the per-service drift test (see [`Node`]). +#[allow(dead_code)] +pub(crate) struct Edge { + /// Source [`Node::id`]. + pub(crate) from: &'static str, + /// Destination [`Node::id`]. + pub(crate) to: &'static str, + /// The condition (branch arm or stage result) this edge is taken on. + pub(crate) on: &'static str, +} + +/// The inspectable DAG for one service's per-peer pipe. +/// +/// This is checked documentation: a per-service drift test asserts the runtime +/// `match` matches this shape. It is never walked to decide control flow. +#[derive(Copy, Clone, Debug)] +// Read only by the per-service drift test (see [`Node`]). +#[allow(dead_code)] +pub(crate) struct PipeShape { + /// Stable service name for diagnostics. + pub(crate) service: &'static str, + /// Every node in the DAG. + pub(crate) nodes: &'static [Node], + /// Every directed edge in the DAG. + pub(crate) edges: &'static [Edge], +} + +impl PipeShape { + /// Validate that every edge endpoint names a real node. + /// + /// This is the reusable half of the per-service drift test; the + /// service-specific half (matching the runtime `match` arms) lands with each + /// service migration. Called only from those `#[cfg(test)]` drift tests. + #[allow(dead_code)] // called only from per-service drift tests + pub(crate) fn validate(&self) -> Result<(), String> { + for edge in self.edges { + if !self.nodes.iter().any(|node| node.id == edge.from) { + return Err(format!( + "{}: edge from unknown node {:?}", + self.service, edge.from + )); + } + if !self.nodes.iter().any(|node| node.id == edge.to) { + return Err(format!( + "{}: edge to unknown node {:?}", + self.service, edge.to + )); + } + } + Ok(()) + } +} + +/// A per-peer pipe, built once per connected peer and consumed by the runner. +pub(crate) struct Pipe { + peer_id: ZakuraPeerId, + local: S, + env: Env, + guard: SessionGuard, + entry: PipeEntry, + // The declared shape is retained for per-service drift diagnostics; it is + // checked documentation, never walked at runtime, so the runner never reads + // it. The per-service drift test reads the same `&'static PipeShape`. + #[allow(dead_code)] // retained for diagnostics; the runner never reads it + shape: &'static PipeShape, +} + +impl Pipe { + /// Build a per-peer pipe from its local state, environment, guard, entry, + /// and declared shape. + pub(crate) fn new( + peer_id: ZakuraPeerId, + local: S, + env: Env, + guard: SessionGuard, + entry: PipeEntry, + shape: &'static PipeShape, + ) -> Self { + Self { + peer_id, + local, + env, + guard, + entry, + shape, + } + } + + /// Run one inbound frame: admit through the guard, then the entry traversal. + /// + /// This is the single inbound traversal used by both the production sink and + /// the test/recorder path, so the two can never disagree. + pub(crate) fn run_one(&mut self, frame: Frame) -> Flow<()> { + match self.guard.admit(&frame) { + Admit::Throttle => Flow::Done, + Admit::Reject(reason) => Flow::Reject(SinkReject::protocol(reason)), + Admit::Pass => { + let mut cx = PipeCx { + peer_id: &self.peer_id, + local: &mut self.local, + env: &self.env, + }; + (self.entry)(&mut cx, frame) + } + } + } + + /// Borrow the peer-local state for service-specific side channels. + pub(crate) fn local_mut(&mut self) -> &mut S { + &mut self.local + } +} + +/// The generic inbound runner for a per-peer pipe. +/// +/// `PipeSink::run` is the single inbound loop: `recv → guard.admit → entry → +/// map Flow`. block_sync drives its stream-6 inbound path through it directly; +/// header_sync and discovery fork their own loops (command draining / async +/// handler handoff) that are genuinely different shapes. +#[allow(dead_code)] +pub(crate) struct PipeSink { + pipe: Pipe, + recv: FramedRecv, + cancel: CancellationToken, +} + +#[allow(dead_code)] +impl PipeSink +where + S: Send + 'static, + Env: Send + Sync + 'static, +{ + /// Build the inbound runner around a pipe and its receive half. + pub(crate) fn new(pipe: Pipe, recv: FramedRecv, cancel: CancellationToken) -> Self { + Self { pipe, recv, cancel } + } + + /// Run the inbound loop until the stream closes or the peer is cancelled. + /// + /// `Throttle` continues the loop, `Reject` returns `Err(SinkReject)`, and + /// stream end or cancellation returns `Ok(())`. + pub(crate) async fn run(mut self) -> Result<(), SinkReject> { + loop { + let frame = tokio::select! { + () = self.cancel.cancelled() => return Ok(()), + frame = self.recv.recv() => frame, + }; + let Some(frame) = frame else { + return Ok(()); + }; + match self.pipe.run_one(frame) { + Flow::Continue(()) | Flow::Done => continue, + Flow::Reject(reject) => return Err(reject), + } + } + } +} + +/// Cleanup that runs when a supervised pipe task ends, on every exit path. +/// +/// Living in `Drop`, the teardown runs whether the pipe future returns normally, +/// returns after a reject, or unwinds on a panic — all from within the single +/// spawned pipe task, with no second `tokio::spawn`. This depends on the build +/// unwinding rather than aborting: under `panic = "unwind"` the `Drop` runs as +/// the panic unwinds the task and tokio catches the panic at the task boundary, +/// so the blast radius is exactly one peer (security_requirements.md SR-1). The +/// `#[cfg(panic = "abort")] compile_error!` above [`spawn_supervised_pipe`] +/// refuses to build the node with abort, where this `Drop` could not run and a +/// single peer panic would kill the whole node. +struct PipeTeardown { + peer_id: ZakuraPeerId, + cancel: CancellationToken, + on_teardown: Option, + on_panic: Option

, +} + +impl Drop for PipeTeardown { + fn drop(&mut self) { + if std::thread::panicking() { + metrics::counter!("zakura.pipe.panic").increment(1); + tracing::error!( + peer_id = ?self.peer_id, + "Zakura peer pipe panicked; disconnecting peer only" + ); + if let Some(on_panic) = self.on_panic.take() { + on_panic(); + } + } + self.cancel.cancel(); + if let Some(on_teardown) = self.on_teardown.take() { + on_teardown(); + } + } +} + +// Peer-pipe panic containment (security_requirements.md SR-1) depends on +// unwinding: `PipeTeardown`'s `Drop` runs during the unwind to disconnect and +// clean up the panicked peer, and tokio catches the task panic so the blast +// radius is one peer. Under `panic = "abort"` none of that can happen — a single +// peer panic aborts the whole node — so refuse to build that way. The workspace +// [profile.*] tables set `panic = "unwind"`; this guard catches a silent +// regression back to abort. +#[cfg(panic = "abort")] +compile_error!( + "Zakura peer-pipe panic containment requires `panic = \"unwind\"` \ + (security_requirements.md SR-1); this build sets `panic = \"abort\"`. \ + Set panic = \"unwind\" in the workspace [profile.dev] and [profile.release]." +); + +/// Launch a per-peer pipe in its own supervised task. +/// +/// This is the single way pipes are launched. The pipe runs inside one spawned +/// task guarded by a [`PipeTeardown`], which cancels the caller-supplied `cancel` +/// token and runs `on_teardown` (which must be idempotent) on every exit path — +/// normal return, reject, or panic. On panic only, it also runs `on_panic`. +/// Callers pass the token whose cancellation is safe on *every* exit (e.g. a +/// per-service token, not a shared connection token that other services ride +/// on); connection-level teardown that must only happen on a fatal reject +/// belongs inside the `pipe` future (see [`handle_pipe_exit`]), and +/// connection-level teardown for panic belongs in `on_panic`. A panicking peer is +/// contained to its own task without a nested `tokio::spawn`. +/// +/// Returns the task's [`JoinHandle`]: services let it drop to detach the task (it +/// self-reaps; the `PipeTeardown` still runs on every exit), while the +/// panic-containment test awaits it to observe the contained panic. +pub(crate) fn spawn_supervised_pipe( + peer_id: ZakuraPeerId, + cancel: CancellationToken, + on_teardown: impl FnOnce() + Send + 'static, + on_panic: impl FnOnce() + Send + 'static, + pipe: impl Future + Send + 'static, +) -> JoinHandle<()> { + tokio::spawn(async move { + let _teardown = PipeTeardown { + peer_id, + cancel, + on_teardown: Some(on_teardown), + on_panic: Some(on_panic), + }; + pipe.await; + }) +} + +/// Cleanup that runs when a supervised *non-pipe* peer task ends, on every exit +/// path. +/// +/// This is the task-level sibling of [`PipeTeardown`] for the peer-influenced +/// service tasks that are not the generic inbound [`Pipe`] runner — e.g. the +/// discovery source/admission helpers and (once adopted) the legacy gossip +/// replay/receive loops. Unlike [`PipeTeardown`] it owns no [`CancellationToken`] +/// of its own: those tasks decide which token(s) a panic must cancel from inside +/// their `on_panic` hook, because some of them ride directly on the shared +/// connection token, which must *not* be cancelled on a normal exit (a clean +/// stream-end of one service must not tear the whole connection down). On panic +/// it runs `on_panic` during the unwind; on every exit it runs `on_teardown`. +struct PeerTaskTeardown { + peer_id: ZakuraPeerId, + on_teardown: Option, + on_panic: Option

, +} + +impl Drop for PeerTaskTeardown { + fn drop(&mut self) { + if std::thread::panicking() { + metrics::counter!("zakura.pipe.panic").increment(1); + tracing::error!( + peer_id = ?self.peer_id, + "Zakura peer task panicked; disconnecting peer only" + ); + if let Some(on_panic) = self.on_panic.take() { + on_panic(); + } + } + if let Some(on_teardown) = self.on_teardown.take() { + on_teardown(); + } + } +} + +/// Launch a peer-influenced, non-pipe service task in its own supervised task. +/// +/// This is the task-level counterpart to [`spawn_supervised_pipe`] for the +/// peer-driven helper tasks that are *not* the generic inbound pipe runner +/// (discovery source/admission, legacy gossip replay/receive). The task runs +/// inside one spawned task guarded by a [`PeerTaskTeardown`], which runs +/// `on_teardown` (which must be idempotent) on every exit path — normal return +/// or panic — and `on_panic` on panic only. Callers put whatever connection / +/// service cancellation a panic requires inside `on_panic`, so a buggy or hostile +/// peer that panics one of these tasks still disconnects *that one peer* and runs +/// its cleanup instead of leaving stale service state behind a half-live +/// connection (security_requirements.md SR-1). Like [`spawn_supervised_pipe`] +/// this depends on the build unwinding rather than aborting; the +/// `#[cfg(panic = "abort")] compile_error!` above guards that for both wrappers. +/// +/// Returns the task's [`JoinHandle`]: services let it drop to detach the task (it +/// self-reaps; the `PeerTaskTeardown` still runs on every exit), while the +/// panic-containment test awaits it to observe the contained panic. +pub(crate) fn spawn_supervised_peer_task( + peer_id: ZakuraPeerId, + on_teardown: impl FnOnce() + Send + 'static, + on_panic: impl FnOnce() + Send + 'static, + task: impl Future + Send + 'static, +) -> JoinHandle<()> { + tokio::spawn(async move { + let _teardown = PeerTaskTeardown { + peer_id, + on_teardown: Some(on_teardown), + on_panic: Some(on_panic), + }; + task.await; + }) +} + +/// Map a finished pipe run to its connection-teardown effect — the single place +/// the "is this exit fatal to the whole connection?" decision lives. +/// +/// A protocol reject is fatal: it cancels the shared `connection_cancel` token so +/// the whole connection tears down. A local reject (e.g. a closed service queue) +/// tears down only this stream — the per-service token is already cancelled by +/// the [`PipeTeardown`] — so it is logged and the connection is left for other +/// services. `Ok` is a normal/parked exit and does nothing here. Panic-path +/// connection teardown is separate (`on_panic`), because a panic never returns a +/// `Result` to inspect. +pub(crate) fn handle_pipe_exit( + service: &'static str, + connection_cancel: &CancellationToken, + result: Result<(), SinkReject>, +) { + match result { + Ok(()) => {} + Err(SinkReject::Protocol(error)) => { + tracing::debug!( + ?error, + service, + "Zakura stream rejected protocol-invalid frame" + ); + connection_cancel.cancel(); + } + Err(SinkReject::Local(error)) => { + tracing::debug!(?error, service, "Zakura stream stopped on local error"); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + use super::*; + use crate::zakura::transport::framed_channel; + use crate::zakura::transport::guard::ByteBudget; + + const SHAPE: PipeShape = PipeShape { + service: "test", + nodes: &[ + Node { + id: "guard", + kind: NodeKind::Guard, + }, + Node { + id: "decode", + kind: NodeKind::Decode, + }, + ], + edges: &[Edge { + from: "guard", + to: "decode", + on: "Pass", + }], + }; + + const DANGLING_SHAPE: PipeShape = PipeShape { + service: "test-dangling", + nodes: &[Node { + id: "guard", + kind: NodeKind::Guard, + }], + edges: &[Edge { + from: "guard", + to: "missing", + on: "Pass", + }], + }; + + fn peer_id() -> ZakuraPeerId { + ZakuraPeerId::new(vec![1, 2, 3]).expect("3-byte id is within the node-id bound") + } + + fn noop_entry(_cx: &mut PipeCx<'_, (), ()>, _frame: Frame) -> Flow<()> { + Flow::Continue(()) + } + + fn pass_guard() -> SessionGuard { + // Allow message type 1, generous size cap, no byte budget. + SessionGuard::new(&[1], 1_024, None) + } + + fn frame(message_type: u16) -> Frame { + Frame { + message_type, + flags: 0, + payload: Vec::new(), + } + } + + #[test] + fn pipe_shape_validate_accepts_consistent_graph() { + assert!(SHAPE.validate().is_ok()); + } + + #[test] + fn pipe_shape_validate_rejects_dangling_edge() { + let err = DANGLING_SHAPE + .validate() + .expect_err("dangling edge target must fail validation"); + assert!(err.contains("missing"), "error names the bad node: {err}"); + } + + #[test] + fn run_one_passes_admitted_frame_to_entry() { + let mut pipe = Pipe::new(peer_id(), (), (), pass_guard(), noop_entry, &SHAPE); + assert!(matches!(pipe.run_one(frame(1)), Flow::Continue(()))); + } + + #[test] + fn run_one_throttles_when_budget_exhausted() { + // A budget that admits one 4-byte payload then throttles the next. + let guard = SessionGuard::new(&[1], 1_024, Some(ByteBudget::new(4))); + let mut pipe = Pipe::new(peer_id(), (), (), guard, noop_entry, &SHAPE); + let mut framed = frame(1); + framed.payload = vec![0u8; 4]; + assert!(matches!(pipe.run_one(framed.clone()), Flow::Continue(()))); + assert!(matches!(pipe.run_one(framed), Flow::Done)); + } + + #[test] + fn run_one_rejects_disallowed_type() { + let mut pipe = Pipe::new(peer_id(), (), (), pass_guard(), noop_entry, &SHAPE); + assert!(matches!(pipe.run_one(frame(2)), Flow::Reject(_))); + } + + #[tokio::test] + async fn pipe_sink_run_returns_ok_on_stream_end() { + let (send, recv) = framed_channel(4); + let cancel = CancellationToken::new(); + let pipe = Pipe::new(peer_id(), (), (), pass_guard(), noop_entry, &SHAPE); + let sink = PipeSink::new(pipe, recv, cancel); + // One admitted frame, then close the stream by dropping the sender. + send.send(frame(1)).await.expect("channel has capacity"); + drop(send); + assert!(sink.run().await.is_ok()); + } + + #[tokio::test] + async fn pipe_sink_run_returns_err_on_reject() { + let (send, recv) = framed_channel(4); + let cancel = CancellationToken::new(); + let pipe = Pipe::new(peer_id(), (), (), pass_guard(), noop_entry, &SHAPE); + let sink = PipeSink::new(pipe, recv, cancel); + // A disallowed type rejects the peer. + send.send(frame(2)).await.expect("channel has capacity"); + assert!(sink.run().await.is_err()); + } + + #[tokio::test] + async fn supervised_pipe_runs_teardown_on_panic() { + let cancel = CancellationToken::new(); + let torn_down = Arc::new(AtomicBool::new(false)); + let flag = torn_down.clone(); + let panic_disconnected = Arc::new(AtomicBool::new(false)); + let panic_flag = panic_disconnected.clone(); + + // Production drops this handle to detach the task; here we await it to + // observe the contained panic. + let handle = spawn_supervised_pipe( + peer_id(), + cancel.clone(), + move || flag.store(true, Ordering::SeqCst), + move || panic_flag.store(true, Ordering::SeqCst), + async { + panic!("peer pipe panics"); + }, + ); + + // The pipe runs in a single task with no nested unwind-isolation spawn, + // so the task itself surfaces the panic to its `JoinHandle`. The + // `Drop`-based teardown still runs during the unwind, so cleanup is + // guaranteed and the panic is contained to this one task. + let join_error = handle + .await + .expect_err("a panicking pipe surfaces a join error"); + assert!( + join_error.is_panic(), + "the pipe panic is reported as a panic, not a cancellation" + ); + + assert!( + torn_down.load(Ordering::SeqCst), + "teardown runs even when the pipe panics" + ); + assert!( + panic_disconnected.load(Ordering::SeqCst), + "panic-only disconnect hook runs when the pipe panics" + ); + assert!(cancel.is_cancelled(), "the peer connection is cancelled"); + } + + #[tokio::test] + async fn supervised_peer_task_runs_teardown_and_disconnect_on_panic() { + // The surprising input: a peer-influenced helper task (e.g. discovery + // source/admission, legacy gossip recv loop) panics *after* its per-peer + // service state is registered, before its normal-path cleanup runs. + let torn_down = Arc::new(AtomicBool::new(false)); + let teardown_flag = torn_down.clone(); + let disconnected = Arc::new(AtomicBool::new(false)); + let disconnect_flag = disconnected.clone(); + + // Production drops this handle to detach the task; here we await it to + // observe the contained panic. + let handle = spawn_supervised_peer_task( + peer_id(), + move || teardown_flag.store(true, Ordering::SeqCst), + move || disconnect_flag.store(true, Ordering::SeqCst), + async { + panic!("peer task panics after state registration"); + }, + ); + + let join_error = handle + .await + .expect_err("a panicking peer task surfaces a join error"); + assert!( + join_error.is_panic(), + "the task panic is reported as a panic, not a cancellation" + ); + // The safe expectation (SR-1): the panic still ran cleanup and the + // peer-disconnect hook, so no stale service state survives behind a + // half-live connection. + assert!( + torn_down.load(Ordering::SeqCst), + "teardown runs even when the peer task panics" + ); + assert!( + disconnected.load(Ordering::SeqCst), + "panic-only disconnect hook runs when the peer task panics" + ); + } + + #[tokio::test] + async fn supervised_peer_task_skips_disconnect_on_normal_exit() { + let torn_down = Arc::new(AtomicBool::new(false)); + let teardown_flag = torn_down.clone(); + let disconnected = Arc::new(AtomicBool::new(false)); + let disconnect_flag = disconnected.clone(); + + let handle = spawn_supervised_peer_task( + peer_id(), + move || teardown_flag.store(true, Ordering::SeqCst), + move || disconnect_flag.store(true, Ordering::SeqCst), + async {}, + ); + + handle + .await + .expect("a normal peer task exit does not panic"); + assert!( + torn_down.load(Ordering::SeqCst), + "teardown runs on a normal exit" + ); + // A clean exit (e.g. one service's stream ends) must NOT trip the + // panic-only disconnect — that would tear down peers that rode on the + // same connection. + assert!( + !disconnected.load(Ordering::SeqCst), + "the panic-only disconnect hook must not fire on a normal exit" + ); + } +} diff --git a/zebra-network/src/zakura/transport/registry.rs b/zebra-network/src/zakura/transport/registry.rs index 94d048a88cd..ebb4ff8f777 100644 --- a/zebra-network/src/zakura/transport/registry.rs +++ b/zebra-network/src/zakura/transport/registry.rs @@ -276,6 +276,7 @@ impl ServiceRegistry { stream_kind: u16, request_id: u64, max_frame_bytes: u32, + max_message_bytes: u32, frame: Frame, ) -> Result, SinkReject> { let Some(service) = self.service_for_kind(stream_kind) else { @@ -290,7 +291,14 @@ impl ServiceRegistry { }; handler - .request_frame(peer_id, stream_kind, request_id, max_frame_bytes, frame) + .request_frame( + peer_id, + stream_kind, + request_id, + max_frame_bytes, + max_message_bytes, + frame, + ) .await } diff --git a/zebra-network/src/zakura/transport/service.rs b/zebra-network/src/zakura/transport/service.rs index 5346bce551f..be3186c3503 100644 --- a/zebra-network/src/zakura/transport/service.rs +++ b/zebra-network/src/zakura/transport/service.rs @@ -257,12 +257,18 @@ pub trait Service: fmt::Debug + Send + Sync + 'static { /// A Zakura service that accepts one-shot request/response streams. pub trait RequestResponseService: Service { /// Deliver one request-response request frame to this service. + /// + /// `max_frame_bytes` bounds each encoded outbound frame; `max_message_bytes` + /// is the peer's negotiated full-message cap. Responders must size outbound + /// frame payloads against the smaller of the two so the peer never receives + /// a frame larger than its accepted message cap. fn request_frame<'a>( &'a self, peer_id: ZakuraPeerId, stream_kind: u16, request_id: u64, max_frame_bytes: u32, + max_message_bytes: u32, frame: Frame, ) -> BoxRunFuture<'a, Result, SinkReject>>; } diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 299d4f4ba95..b9b81e635d3 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -2377,8 +2377,15 @@ where // when the tip changes. let precompute_coinbase = |network, height, params| { tokio::task::spawn_blocking(move || { - TransactionTemplate::new_coinbase(&network, height, ¶ms, Amount::zero()) - .expect("valid coinbase tx") + TransactionTemplate::new_coinbase( + &network, + height, + ¶ms, + Amount::zero(), + #[cfg(all(zcash_unstable = "nu7", feature = "tx_v6"))] + None, + ) + .expect("valid coinbase tx") }) }; @@ -2470,6 +2477,8 @@ where server_long_poll_id, vec![], submit_old, + #[cfg(all(zcash_unstable = "nu7", feature = "tx_v6"))] + None, ) .into()) } diff --git a/zebra-rpc/src/methods/types/get_block_template.rs b/zebra-rpc/src/methods/types/get_block_template.rs index b1fd16dc4ec..cca85270607 100644 --- a/zebra-rpc/src/methods/types/get_block_template.rs +++ b/zebra-rpc/src/methods/types/get_block_template.rs @@ -27,8 +27,6 @@ use zcash_script::{ opcode::{Evaluable, PushValue}, pv::push_value, }; -#[cfg(all(zcash_unstable = "nu7", feature = "tx_v6"))] -use zebra_chain::amount::{Amount, NonNegative}; use zebra_chain::{ amount::{self, Amount, NonNegative}, block::{ diff --git a/zebra-rpc/src/methods/types/get_block_template/zip317.rs b/zebra-rpc/src/methods/types/get_block_template/zip317.rs index e5085ad6c9a..6139ac83287 100644 --- a/zebra-rpc/src/methods/types/get_block_template/zip317.rs +++ b/zebra-rpc/src/methods/types/get_block_template/zip317.rs @@ -24,9 +24,6 @@ use zebra_node_services::mempool::TransactionDependencies; use crate::methods::types::transaction::TransactionTemplate; -#[cfg(all(zcash_unstable = "nu7", feature = "tx_v6"))] -use crate::methods::Amount; - #[cfg(all(zcash_unstable = "nu7", feature = "tx_v6"))] use zebra_chain::amount::NonNegative; diff --git a/zebra-rpc/src/methods/types/transaction.rs b/zebra-rpc/src/methods/types/transaction.rs index d25e7bf3c1b..3122686df03 100644 --- a/zebra-rpc/src/methods/types/transaction.rs +++ b/zebra-rpc/src/methods/types/transaction.rs @@ -152,7 +152,9 @@ impl TransactionTemplate { #[cfg(all(zcash_unstable = "nu7", feature = "tx_v6"))] { let zip233_amount = if cfg!(zcash_unstable = "zip235") { - zip233_amount.unwrap_or_else(|| ((miner_fee * 6).unwrap() / 10).unwrap()) + // ZIP235 defaults to 60% of the positive fees paid by selected transactions. + // The coinbase template's `fee` field is the negative of this total. + zip233_amount.unwrap_or_else(|| ((txs_fee * 6).unwrap() / 10).unwrap()) } else { zip233_amount.unwrap_or(Amount::zero()) }; diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index 0b4269ca1dd..d3b33a9d5af 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -102,6 +102,14 @@ pub struct Config { /// Zebra's last non-finalized state before it shut down. pub should_backup_non_finalized_state: bool, + /// Whether committed full blocks should seed the Zakura header-only store. + /// + /// This is enabled by zebrad only for the Zakura v2 path. It is skipped in + /// serde so the generic Zebra state config does not expose a Zakura-specific + /// user setting. + #[serde(skip)] + pub enable_zakura_header_seed_from_committed_blocks: bool, + /// Whether to delete the old database directories when present. /// /// Set to `true` by default. If this is set to `false`, @@ -393,6 +401,7 @@ impl Default for Config { cache_dir: default_cache_dir(), ephemeral: false, should_backup_non_finalized_state: true, + enable_zakura_header_seed_from_committed_blocks: false, delete_old_database: true, storage_mode: StorageMode::default(), debug_stop_at_height: None, diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs index dc69023999e..01136925997 100644 --- a/zebra-state/src/constants.rs +++ b/zebra-state/src/constants.rs @@ -91,7 +91,7 @@ const DATABASE_FORMAT_VERSION: u64 = 27; /// - adding new column families, /// - changing the format of a column family in a compatible way, or /// - breaking changes with compatibility code in all supported Zebra versions. -const DATABASE_FORMAT_MINOR_VERSION: u64 = 1; +const DATABASE_FORMAT_MINOR_VERSION: u64 = 2; /// The database format patch version, incremented each time the on-disk database format has a /// significant format compatibility fix. diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 15da13f4c5d..04b2250d9df 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -78,6 +78,14 @@ impl CommitBlockError { matches!(self, CommitBlockError::Duplicate { .. }) } + /// Returns the state location for duplicate commit requests. + pub fn duplicate_location(&self) -> Option<&KnownBlock> { + match self { + CommitBlockError::Duplicate { location, .. } => Some(location), + _ => None, + } + } + /// Returns a suggested misbehaviour score increment for a certain error. pub fn misbehavior_score(&self) -> u32 { 0 @@ -95,6 +103,13 @@ impl From for CommitBlockError { #[error("could not commit semantically-verified block")] pub struct CommitSemanticallyVerifiedError(#[from] CommitBlockError); +impl CommitSemanticallyVerifiedError { + /// Returns the state location for duplicate commit requests. + pub fn duplicate_location(&self) -> Option<&KnownBlock> { + self.0.duplicate_location() + } +} + impl From for CommitSemanticallyVerifiedError { fn from(value: ValidateContextError) -> Self { Self(CommitBlockError::ValidateContextError(Box::new(value))) @@ -129,6 +144,13 @@ impl From for LayeredStateError { #[error("could not commit checkpoint-verified block")] pub struct CommitCheckpointVerifiedError(#[from] CommitBlockError); +impl CommitCheckpointVerifiedError { + /// Returns the state location for duplicate commit requests. + pub fn duplicate_location(&self) -> Option<&KnownBlock> { + self.0.duplicate_location() + } +} + impl From for CommitCheckpointVerifiedError { fn from(value: ValidateContextError) -> Self { Self(CommitBlockError::ValidateContextError(Box::new(value))) @@ -158,6 +180,15 @@ pub enum CommitHeaderRangeError { actual: usize, }, + /// The request supplied a different number of body-size hints than headers. + #[error("header range body-size count {body_sizes} does not match header count {headers}")] + BodySizeCountMismatch { + /// Header count. + headers: usize, + /// Body-size hint count. + body_sizes: usize, + }, + /// The supplied anchor is not known to state. #[error("header range anchor {anchor} is not known")] UnknownAnchor { @@ -165,6 +196,14 @@ pub enum CommitHeaderRangeError { anchor: block::Hash, }, + /// The supplied anchor is the network genesis hash, but the genesis block has not been + /// committed to state yet. + #[error("header range genesis anchor {anchor} is not committed to state yet")] + MissingGenesisAnchor { + /// The supplied genesis anchor hash. + anchor: block::Hash, + }, + /// The inferred header height overflowed the valid block height range. #[error("header height overflow")] HeightOverflow, diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index d6cddbf8a49..59bc7fbd460 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -42,7 +42,7 @@ pub use config::{ }; pub use constants::{state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT}; pub use error::{ - BoxError, CloneError, CommitBlockError, CommitHeaderRangeError, + BoxError, CloneError, CommitBlockError, CommitCheckpointVerifiedError, CommitHeaderRangeError, CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError, }; pub use request::{ diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 4df145237d8..992d91812d7 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -846,6 +846,10 @@ pub enum Request { anchor: block::Hash, /// Contiguous headers in ascending height order. headers: Vec>, + /// Advisory serialized body sizes, parallel to `headers`. + /// + /// A `0` value means unknown. These hints are not consensus data. + body_sizes: Vec, }, /// Computes the depth in the current best chain of the block identified by the given hash. @@ -1314,6 +1318,27 @@ pub enum ReadRequest { limit: u32, }, + /// Returns scheduling-only body-size hints for a contiguous height range. + /// + /// Confirmed committed block sizes win over untrusted advertised header + /// hints. Unknown advertised sizes are returned as `None`. + BlockSizeHints { + /// First height to read. + from: block::Height, + /// Maximum number of heights to return. + count: u32, + }, + + /// Returns contiguous committed blocks by height, in ascending order. + /// + /// The response stops before the first height without a committed body. + BlocksByHeightRange { + /// First height to read. + start: block::Height, + /// Maximum number of blocks to return. + count: u32, + }, + /// Looks up a Sapling note commitment tree either by a hash or height. /// /// Returns @@ -1482,6 +1507,8 @@ impl ReadRequest { ReadRequest::HeadersByHeightRange { .. } => "headers_by_height_range", ReadRequest::BestHeaderTip => "best_header_tip", ReadRequest::MissingBlockBodies { .. } => "missing_block_bodies", + ReadRequest::BlockSizeHints { .. } => "block_size_hints", + ReadRequest::BlocksByHeightRange { .. } => "blocks_by_height_range", ReadRequest::SaplingTree { .. } => "sapling_tree", ReadRequest::OrchardTree { .. } => "orchard_tree", ReadRequest::SaplingSubtrees { .. } => "sapling_subtrees", diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 762c44f7663..e60135eb92a 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -394,6 +394,12 @@ pub enum ReadResponse { /// Response to [`ReadRequest::MissingBlockBodies`]. MissingBlockBodies(Vec), + /// Response to [`ReadRequest::BlockSizeHints`]. + BlockSizeHints(Vec<(block::Height, Option)>), + + /// Response to [`ReadRequest::BlocksByHeightRange`]. + Blocks(Vec<(block::Height, Arc, usize)>), + /// The response to a `UnspentBestChainUtxo` request, from verified blocks in the /// _best_ non-finalized chain, or the finalized chain. UnspentBestChainUtxo(Option), @@ -570,6 +576,8 @@ impl TryFrom for Response { | ReadResponse::Headers(_) | ReadResponse::BestHeaderTip(_) | ReadResponse::MissingBlockBodies(_) + | ReadResponse::BlockSizeHints(_) + | ReadResponse::Blocks(_) | ReadResponse::NonFinalizedBlocksListener(_) | ReadResponse::IsTransparentOutputSpent(_) => { Err("there is no corresponding Response for this ReadResponse") diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 4a0af44d65e..07323488994 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -984,6 +984,7 @@ impl StateService { &self, anchor: block::Hash, headers: Vec>, + body_sizes: Vec, ) -> oneshot::Receiver> { let (rsp_tx, rsp_rx) = oneshot::channel(); @@ -996,6 +997,7 @@ impl StateService { sender.send(NonFinalizedWriteMessage::CommitHeaderRange { anchor, headers, + body_sizes, rsp_tx, }) { @@ -1230,9 +1232,13 @@ impl Service for StateService { .boxed() } - Request::CommitHeaderRange { anchor, headers } => { + Request::CommitHeaderRange { + anchor, + headers, + body_sizes, + } => { let rsp_rx = tokio::task::block_in_place(move || { - span.in_scope(|| self.send_header_range(anchor, headers)) + span.in_scope(|| self.send_header_range(anchor, headers, body_sizes)) }); let span = Span::current(); @@ -1730,6 +1736,28 @@ impl Service for ReadStateService { )) } + ReadRequest::BlockSizeHints { from, count } => Ok(ReadResponse::BlockSizeHints( + read::block_size_hints(state.latest_best_chain(), &state.db, from, count), + )), + + ReadRequest::BlocksByHeightRange { start, count } => { + let best_chain = state.latest_best_chain(); + let blocks = (0..count) + .map_while(|offset| { + start + .0 + .checked_add(offset) + .map(block::Height) + .and_then(|height| { + read::block_and_size(best_chain.clone(), &state.db, height.into()) + .map(|(block, size)| (height, block, size)) + }) + }) + .collect(); + + Ok(ReadResponse::Blocks(blocks)) + } + ReadRequest::SaplingTree(hash_or_height) => Ok(ReadResponse::SaplingTree( read::sapling_tree(state.latest_best_chain(), &state.db, hash_or_height), )), diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index f610229cb09..9f4ccf44f8b 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -21,6 +21,7 @@ use std::{ use zebra_chain::{block, parallel::tree::NoteCommitmentTrees, parameters::Network}; use zebra_db::{ + block::ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, chain::BLOCK_INFO, transparent::{BALANCE_BY_TRANSPARENT_ADDR, TX_LOC_BY_SPENT_OUT_LOC}, }; @@ -81,6 +82,7 @@ pub const STATE_COLUMN_FAMILIES_IN_CODE: &[&str] = &[ "zakura_header_hash_by_height", "zakura_header_height_by_hash", "zakura_header_by_height", + ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, // Transactions "tx_by_loc", "hash_by_tx_loc", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap index b5926f73e64..d548705b0b8 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap @@ -30,6 +30,7 @@ expression: cf_names "tx_loc_by_transparent_addr_loc", "utxo_by_out_loc", "utxo_loc_by_transparent_addr_loc", + "zakura_header_body_size_by_height", "zakura_header_by_height", "zakura_header_hash_by_height", "zakura_header_height_by_hash", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap index 3f48bc4c88a..6d858dc42ea 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap @@ -15,6 +15,7 @@ expression: empty_column_families "tx_loc_by_transparent_addr_loc: no entries", "utxo_by_out_loc: no entries", "utxo_loc_by_transparent_addr_loc: no entries", + "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap index 446df58d472..4fb77151719 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap @@ -11,6 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap index 446df58d472..4fb77151719 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap @@ -11,6 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap index 03a0f42d1ff..618b220b649 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap @@ -29,6 +29,7 @@ expression: empty_column_families "tx_loc_by_transparent_addr_loc: no entries", "utxo_by_out_loc: no entries", "utxo_loc_by_transparent_addr_loc: no entries", + "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap index 3f48bc4c88a..6d858dc42ea 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap @@ -15,6 +15,7 @@ expression: empty_column_families "tx_loc_by_transparent_addr_loc: no entries", "utxo_by_out_loc: no entries", "utxo_loc_by_transparent_addr_loc: no entries", + "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap index 446df58d472..4fb77151719 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap @@ -11,6 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap index 446df58d472..4fb77151719 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap @@ -11,6 +11,7 @@ expression: empty_column_families "sapling_nullifiers: no entries", "sprout_nullifiers: no entries", "tx_loc_by_spent_out_loc: no entries", + "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/upgrade.rs b/zebra-state/src/service/finalized_state/disk_format/upgrade.rs index 9861bc371ba..d4f5872006c 100644 --- a/zebra-state/src/service/finalized_state/disk_format/upgrade.rs +++ b/zebra-state/src/service/finalized_state/disk_format/upgrade.rs @@ -106,7 +106,11 @@ fn format_upgrades( "add pruning metadata column family", Version::new(27, 1, 0), )), - ] as [Box; 6]) + Box::new(no_migration::NoMigration::new( + "add Zakura header body size hints", + Version::new(27, 2, 0), + )), + ] as [Box; 7]) .into_iter() .filter(move |upgrade| upgrade.version() > min_version()) } @@ -868,3 +872,12 @@ fn format_upgrades_are_in_version_order() { last_version = upgrade.version(); } } + +#[test] +fn zakura_header_body_size_cf_upgrade_is_no_migration() { + let upgrades: Vec<_> = format_upgrades(Some(Version::new(27, 1, 0))).collect(); + + assert_eq!(upgrades.len(), 1); + assert_eq!(upgrades[0].version(), Version::new(27, 2, 0)); + assert!(!upgrades[0].needs_migration()); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 3f5b6a1cab8..be95a13a2a8 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -45,7 +45,7 @@ use crate::{ transparent::{AddressBalanceLocationUpdates, OutputLocation}, }, zebra_db::{metrics::block_precommit_metrics, ZebraDb}, - FromDisk, RawBytes, PRUNING_METADATA, + FromDisk, IntoDisk, RawBytes, PRUNING_METADATA, }, HashOrHeight, }; @@ -59,6 +59,38 @@ mod tests; const ZAKURA_HEADER_HASH_BY_HEIGHT: &str = "zakura_header_hash_by_height"; const ZAKURA_HEADER_HEIGHT_BY_HASH: &str = "zakura_header_height_by_hash"; const ZAKURA_HEADER_BY_HEIGHT: &str = "zakura_header_by_height"; +pub const ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT: &str = "zakura_header_body_size_by_height"; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +struct AdvertisedBodySize(u32); + +impl AdvertisedBodySize { + fn new(size: u32) -> Option { + (size != 0).then_some(Self(size)) + } + + fn get(self) -> u32 { + self.0 + } +} + +impl IntoDisk for AdvertisedBodySize { + type Bytes = [u8; 4]; + + fn as_bytes(&self) -> Self::Bytes { + self.0.to_be_bytes() + } +} + +impl FromDisk for AdvertisedBodySize { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + let bytes = bytes + .as_ref() + .try_into() + .expect("advertised body sizes are stored as u32"); + Self(u32::from_be_bytes(bytes)) + } +} impl ZebraDb { // Read block methods @@ -105,6 +137,22 @@ impl ZebraDb { self.db.zs_contains(&tx_by_loc, &first_tx) } + /// Returns the advisory body-size hint for a header-only height, if known. + /// + /// `None` means the peer supplied the `0` unknown sentinel or no hint has been + /// stored. This value is not consensus data. + #[allow(clippy::unwrap_in_result)] + pub fn advertised_body_size(&self, height: block::Height) -> Option { + let body_size_by_height = self + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + + self.db + .zs_get(&body_size_by_height, &height) + .map(AdvertisedBodySize::get) + } + /// Returns the finalized hash for a given `block::Height` if it is present. #[allow(clippy::unwrap_in_result)] pub fn hash(&self, height: block::Height) -> Option { @@ -886,6 +934,21 @@ impl ZebraDb { self.db.zs_compact_range(&tx_by_loc, range_start, range_end); } + + /// Seed or reconcile the Zakura header store from a committed full block. + pub(crate) fn seed_zakura_header_from_committed_block( + &self, + height: block::Height, + block: &Arc, + ) -> Result<(), CommitHeaderRangeError> { + let mut batch = DiskWriteBatch::new(); + batch.prepare_zakura_header_from_committed_block(&self.db, height, block)?; + self.db + .write(batch) + .map_err(|error| CommitHeaderRangeError::StorageWriteError { + error: error.to_string(), + }) + } } /// Lookup the output location for an outpoint. @@ -982,10 +1045,8 @@ impl DiskWriteBatch { value_pool: ValueBalance, prev_note_commitment_trees: Option, ) -> Result<(), CommitCheckpointVerifiedError> { - let db = &zebra_db.db; - // Commit block, transaction, and note commitment tree data. - self.prepare_block_header_and_transaction_data_batch(db, finalized)?; + self.prepare_block_header_and_transaction_data_batch(zebra_db, finalized)?; // The consensus rules are silent on shielded transactions in the genesis block, // because there aren't any in the mainnet or testnet genesis blocks. @@ -1100,16 +1161,15 @@ impl DiskWriteBatch { #[allow(clippy::unwrap_in_result)] pub fn prepare_block_header_and_transaction_data_batch( &mut self, - db: &DiskDb, + zebra_db: &ZebraDb, finalized: &FinalizedBlock, ) -> Result<(), CommitCheckpointVerifiedError> { + let db = &zebra_db.db; + // Blocks let block_header_by_height = db.cf_handle("block_header_by_height").unwrap(); let hash_by_height = db.cf_handle("hash_by_height").unwrap(); let height_by_hash = db.cf_handle("height_by_hash").unwrap(); - let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); - let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); - let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); // Transactions let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); @@ -1135,8 +1195,67 @@ impl DiskWriteBatch { ); } + if zebra_db + .config() + .enable_zakura_header_seed_from_committed_blocks + { + self.prepare_zakura_header_from_committed_block(db, *height, block)?; + } + + // Index the block header, hash, and height. This also restores the + // verified full block row after any provisional cleanup above. + self.zs_insert(&block_header_by_height, height, &block.header); + self.zs_insert(&hash_by_height, height, hash); + self.zs_insert(&height_by_hash, hash, height); + + for (transaction_index, (transaction, transaction_hash)) in block + .transactions + .iter() + .zip(transaction_hashes.iter()) + .enumerate() + { + let transaction_location = TransactionLocation::from_usize(*height, transaction_index); + + // Commit each transaction's data + self.zs_insert(&tx_by_loc, transaction_location, transaction); + + // Index each transaction hash and location + self.zs_insert(&hash_by_tx_loc, transaction_location, transaction_hash); + self.zs_insert(&tx_loc_by_hash, transaction_hash, transaction_location); + } + + Ok(()) + } + + /// Prepare a database batch that seeds the Zakura header store from a + /// committed full block. + /// + /// Full block verification is authoritative for the stored body. If a + /// provisional Zakura header at this height differs, replace it with the + /// block-derived header and drop stale provisional descendants. + #[allow(clippy::unwrap_in_result)] + pub fn prepare_zakura_header_from_committed_block( + &mut self, + db: &DiskDb, + height: block::Height, + block: &Arc, + ) -> Result<(), CommitHeaderRangeError> { + let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); + let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); + + let hash = block.hash(); let existing_zakura_header: Option> = - db.zs_get(&zakura_header_by_height, height); + db.zs_get(&zakura_header_by_height, &height); + + if existing_zakura_header.as_ref() == Some(&block.header) + && db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) == Some(hash) + { + return Ok(()); + } + if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { let best_header_tip: Option<(block::Height, block::Hash)> = db.zs_last_key_value(&zakura_hash_by_height); @@ -1145,7 +1264,7 @@ impl DiskWriteBatch { for old_height in height.0..=best_header_tip.0 { let old_height = block::Height(old_height); - if old_height != *height + if old_height != height && db.zs_contains( &tx_by_loc, &TransactionLocation::min_for_height(old_height), @@ -1153,8 +1272,7 @@ impl DiskWriteBatch { { return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height: old_height, - } - .into()); + }); } if let Some(old_hash) = @@ -1165,32 +1283,21 @@ impl DiskWriteBatch { self.zs_delete(&zakura_hash_by_height, old_height); self.zs_delete(&zakura_header_by_height, old_height); + self.zs_delete(&zakura_body_size_by_height, old_height); } } - } - - // Index the block header, hash, and height. This also restores the - // verified full block row after any provisional cleanup above. - self.zs_insert(&block_header_by_height, height, &block.header); - self.zs_insert(&hash_by_height, height, hash); - self.zs_insert(&height_by_hash, hash, height); - - for (transaction_index, (transaction, transaction_hash)) in block - .transactions - .iter() - .zip(transaction_hashes.iter()) - .enumerate() + } else if let Some(old_hash) = + db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) { - let transaction_location = TransactionLocation::from_usize(*height, transaction_index); - - // Commit each transaction's data - self.zs_insert(&tx_by_loc, transaction_location, transaction); - - // Index each transaction hash and location - self.zs_insert(&hash_by_tx_loc, transaction_location, transaction_hash); - self.zs_insert(&tx_loc_by_hash, transaction_hash, transaction_location); + if old_hash != hash { + self.zs_delete(&zakura_height_by_hash, old_hash); + } } + self.zs_insert(&zakura_header_by_height, height, &block.header); + self.zs_insert(&zakura_hash_by_height, height, hash); + self.zs_insert(&zakura_height_by_hash, hash, height); + Ok(()) } @@ -1200,11 +1307,19 @@ impl DiskWriteBatch { zebra_db: &ZebraDb, anchor: block::Hash, headers: &[Arc], + body_sizes: &[u32], ) -> Result { if headers.is_empty() { return Err(CommitHeaderRangeError::EmptyRange); } + if headers.len() != body_sizes.len() { + return Err(CommitHeaderRangeError::BodySizeCountMismatch { + headers: headers.len(), + body_sizes: body_sizes.len(), + }); + } + if headers.len() > MAX_HEADER_SYNC_HEIGHT_RANGE as usize { return Err(CommitHeaderRangeError::RangeTooLong { actual: headers.len(), @@ -1214,6 +1329,10 @@ impl DiskWriteBatch { let header_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); let hash_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); let height_by_hash = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let body_size_by_height = zebra_db + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); let anchor_height = zebra_db .header_height(anchor) @@ -1232,6 +1351,9 @@ impl DiskWriteBatch { let mut recent_headers = zebra_db.recent_header_context(anchor_height); if recent_headers.is_empty() { + if anchor == zebra_db.network().genesis_hash() && anchor_height == block::Height(0) { + return Err(CommitHeaderRangeError::MissingGenesisAnchor { anchor }); + } return Err(CommitHeaderRangeError::UnknownAnchor { anchor }); } @@ -1244,6 +1366,7 @@ impl DiskWriteBatch { let height = (anchor_height + i64::from(offset)) .ok_or(CommitHeaderRangeError::HeightOverflow)?; let hash = block::Hash::from(&**header); + let body_size = body_sizes[index]; if let Some(expected) = checkpoints.hash(height) { if expected != hash { @@ -1290,7 +1413,7 @@ impl DiskWriteBatch { recent_headers.insert(0, (header.difficulty_threshold, header.time)); recent_headers.truncate(check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); - validated_headers.push((height, hash, header)); + validated_headers.push((height, hash, header, body_size)); } if let (Some(first_conflicting_height), Some(best_header_tip)) = @@ -1309,13 +1432,19 @@ impl DiskWriteBatch { self.zs_delete(&hash_by_height, height); self.zs_delete(&header_by_height, height); + self.zs_delete(&body_size_by_height, height); } } - for (height, hash, header) in validated_headers { + for (height, hash, header, body_size) in validated_headers { self.zs_insert(&header_by_height, height, header); self.zs_insert(&hash_by_height, height, hash); self.zs_insert(&height_by_hash, hash, height); + if let Some(body_size) = AdvertisedBodySize::new(body_size) { + self.zs_insert(&body_size_by_height, height, body_size); + } else { + self.zs_delete(&body_size_by_height, height); + } } Ok(block::Hash::from( diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 546c3b520dc..87296ff9aec 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -21,6 +21,7 @@ use zebra_chain::{ }, Block, Height, }, + block_info::BlockInfo, parameters::{ testnet, Network::{self, *}, @@ -42,7 +43,7 @@ use crate::{ service::{ check::difficulty::AdjustedDifficulty, finalized_state::{ - disk_db::{DiskWriteBatch, WriteDisk}, + disk_db::{DiskWriteBatch, ReadDisk, WriteDisk}, ZebraDb, PRUNING_METADATA, STATE_COLUMN_FAMILIES_IN_CODE, }, read, @@ -98,7 +99,12 @@ fn header_range_commit_keeps_body_availability_separate() { let mut batch = DiskWriteBatch::new(); let committed_hash = batch - .prepare_header_range_batch(&state, genesis.hash(), std::slice::from_ref(&block1.header)) + .prepare_header_range_batch( + &state, + genesis.hash(), + std::slice::from_ref(&block1.header), + &[0], + ) .expect("block 1 header links to genesis and has valid context"); state .write_batch(batch) @@ -126,6 +132,68 @@ fn header_range_commit_keeps_body_availability_separate() { .is_none()); } +#[test] +fn header_range_commit_stores_advertised_body_sizes_with_zero_as_unknown() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + + let headers = vec![block1.header.clone(), block2.header.clone()]; + let mut batch = DiskWriteBatch::new(); + batch + .prepare_header_range_batch(&state, genesis.hash(), &headers, &[123_456, 0]) + .expect("block headers link to genesis and have valid context"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + + assert_eq!(state.advertised_body_size(Height(1)), Some(123_456)); + assert_eq!(state.advertised_body_size(Height(2)), None); +} + +#[test] +fn block_size_hints_prefer_confirmed_block_info_over_advertised_hint() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + + let mut batch = DiskWriteBatch::new(); + batch + .prepare_header_range_batch( + &state, + genesis.hash(), + std::slice::from_ref(&block1.header), + &[999_999], + ) + .expect("block 1 header links to genesis and has valid context"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + assert_eq!(state.advertised_body_size(Height(1)), Some(999_999)); + + write_full_block_header_and_transactions(&state, block1.clone()); + let block1_size = u32::try_from(block1.zcash_serialize_to_vec().unwrap().len()) + .expect("serialized block size fits in u32"); + let mut block_info_batch = DiskWriteBatch::new(); + let _ = state + .block_info_cf() + .with_batch_for_writing(&mut block_info_batch) + .zs_insert(&Height(1), &BlockInfo::new(Default::default(), block1_size)); + state + .db + .write(block_info_batch) + .expect("block info batch writes successfully"); + + assert_eq!( + crate::service::read::block_size_hints( + None::>, + &state, + Height(1), + 1, + ), + vec![(Height(1), Some(block1_size))], + ); +} + #[test] fn header_range_read_is_contiguous_capped_and_stops_at_first_gap() { let _init_guard = zebra_test::init(); @@ -133,7 +201,12 @@ fn header_range_read_is_contiguous_capped_and_stops_at_first_gap() { let mut batch = DiskWriteBatch::new(); batch - .prepare_header_range_batch(&state, genesis.hash(), std::slice::from_ref(&block1.header)) + .prepare_header_range_batch( + &state, + genesis.hash(), + std::slice::from_ref(&block1.header), + &[0], + ) .expect("block 1 header is valid"); state.write_batch(batch).expect("header batch writes"); @@ -196,6 +269,98 @@ fn missing_block_bodies_respects_from_limit_and_empty_body_gap() { ); } +#[test] +fn committed_block_seeds_missing_zakura_header() { + let _init_guard = zebra_test::init(); + let (state, _genesis, block1) = mainnet_state_with_genesis_and_zakura_seed(); + + assert!(state.headers_by_height_range(Height(1), 1).is_empty()); + + write_full_block_header_and_transactions(&state, block1.clone()); + + assert_eq!(state.best_header_tip(), Some((Height(1), block1.hash()))); + assert_eq!( + state.headers_by_height_range(Height(1), 1), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); +} + +#[test] +fn committed_block_does_not_seed_zakura_header_by_default() { + let _init_guard = zebra_test::init(); + let (state, _genesis, block1) = mainnet_state_with_genesis(); + let zakura_header_by_height = state.db.cf_handle("zakura_header_by_height").unwrap(); + + assert!(state.headers_by_height_range(Height(1), 1).is_empty()); + assert!(state + .db + .zs_get::<_, _, Arc>(&zakura_header_by_height, &Height(1)) + .is_none()); + + write_full_block_header_and_transactions(&state, block1); + + assert!(state + .db + .zs_get::<_, _, Arc>(&zakura_header_by_height, &Height(1)) + .is_none()); +} + +#[test] +fn committed_block_replaces_mismatched_zakura_header() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis_and_zakura_seed(); + let block2 = mainnet_block(2); + let old_header1 = alternate_header(genesis.hash(), &block1.header, 1); + let old_hash1 = block::Hash::from(&*old_header1); + let old_header2 = alternate_header(old_hash1, &block2.header, 2); + let old_hash2 = block::Hash::from(&*old_header2); + + let header_by_height = state.db.cf_handle("zakura_header_by_height").unwrap(); + let hash_by_height = state.db.cf_handle("zakura_header_hash_by_height").unwrap(); + let height_by_hash = state.db.cf_handle("zakura_header_height_by_hash").unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_by_height, Height(1), &old_header1); + batch.zs_insert(&hash_by_height, Height(1), old_hash1); + batch.zs_insert(&height_by_hash, old_hash1, Height(1)); + batch.zs_insert(&header_by_height, Height(2), &old_header2); + batch.zs_insert(&hash_by_height, Height(2), old_hash2); + batch.zs_insert(&height_by_hash, old_hash2, Height(2)); + state.db.write(batch).expect("mismatched headers write"); + + assert_eq!( + state.headers_by_height_range(Height(1), 2), + vec![ + (Height(1), old_hash1, old_header1), + (Height(2), old_hash2, old_header2), + ], + ); + + write_full_block_header_and_transactions(&state, block1.clone()); + + assert_eq!(state.best_header_tip(), Some((Height(1), block1.hash()))); + assert_eq!( + state.headers_by_height_range(Height(1), 2), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); +} + +#[test] +fn committed_block_with_matching_zakura_header_is_noop() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis_and_zakura_seed(); + + commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header)); + + assert_eq!(state.best_header_tip(), Some((Height(1), block1.hash()))); + write_full_block_header_and_transactions(&state, block1.clone()); + + assert_eq!(state.best_header_tip(), Some((Height(1), block1.hash()))); + assert_eq!( + state.headers_by_height_range(Height(1), 1), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); +} + #[test] fn header_range_commit_rejects_finalized_or_body_conflicts() { let _init_guard = zebra_test::init(); @@ -207,7 +372,7 @@ fn header_range_commit_rejects_finalized_or_body_conflicts() { let mut batch = DiskWriteBatch::new(); assert!(matches!( - batch.prepare_header_range_batch(&state, genesis.hash(), &[Arc::new(conflicting)]), + batch.prepare_header_range_batch(&state, genesis.hash(), &[Arc::new(conflicting)], &[0]), Err(CommitHeaderRangeError::ImmutableConflict { height: Height(1) }) | Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height: Height(1) }) )); @@ -226,7 +391,7 @@ fn header_range_commit_rejects_checkpoint_conflicts() { let mut batch = DiskWriteBatch::new(); assert!(matches!( - batch.prepare_header_range_batch(&state, genesis.hash(), &[Arc::new(forged)]), + batch.prepare_header_range_batch(&state, genesis.hash(), &[Arc::new(forged)], &[0]), Err(CommitHeaderRangeError::CheckpointConflict { height: Height(1), expected, @@ -365,7 +530,7 @@ fn header_range_reorg_rejects_too_deep_overwrite() { let mut batch = DiskWriteBatch::new(); assert!(matches!( - batch.prepare_header_range_batch(&state, conflict_anchor, &[conflicting_header]), + batch.prepare_header_range_batch(&state, conflict_anchor, &[conflicting_header], &[0]), Err(CommitHeaderRangeError::ReorgTooDeep { height, best_header_tip, @@ -440,7 +605,7 @@ fn header_range_commit_rejects_non_current_anchor_hash() { let mut batch = DiskWriteBatch::new(); assert!(matches!( - batch.prepare_header_range_batch(&state, stale_anchor, &[alternate_block2]), + batch.prepare_header_range_batch(&state, stale_anchor, &[alternate_block2], &[0]), Err(CommitHeaderRangeError::UnknownAnchor { anchor }) if anchor == stale_anchor )); @@ -451,6 +616,37 @@ fn header_range_commit_rejects_non_current_anchor_hash() { ); } +#[test] +fn header_range_commit_reports_missing_genesis_before_scratch_bootstrap() { + let _init_guard = zebra_test::init(); + let genesis = mainnet_block(0); + let block1 = mainnet_block(1); + let state = ZebraDb::new( + &Config::ephemeral(), + STATE_DATABASE_KIND, + &state_database_format_version_in_code(), + &Mainnet, + true, + STATE_COLUMN_FAMILIES_IN_CODE + .iter() + .map(ToString::to_string), + false, + ); + + let mut batch = DiskWriteBatch::new(); + assert!(matches!( + batch.prepare_header_range_batch( + &state, + genesis.hash(), + std::slice::from_ref(&block1.header), + &[0], + ), + Err(CommitHeaderRangeError::MissingGenesisAnchor { anchor }) if anchor == genesis.hash() + )); + + assert_eq!(state.best_header_tip(), None); +} + #[test] fn header_range_rows_and_tip_survive_reopen_without_body_availability() { let _init_guard = zebra_test::init(); @@ -543,7 +739,7 @@ fn full_block_commit_over_identical_header_only_row_is_noop_for_header_indexes() #[test] fn full_block_commit_overwrites_conflicting_header_only_rows() { let _init_guard = zebra_test::init(); - let (state, genesis, block1) = mainnet_state_with_genesis(); + let (state, genesis, block1) = mainnet_state_with_genesis_and_zakura_seed(); let block2 = mainnet_block(2); let block3 = mainnet_block(3); @@ -578,9 +774,32 @@ fn mainnet_state_with_genesis() -> (ZebraDb, Arc, Arc) { (state, genesis, block1) } +fn mainnet_state_with_genesis_and_zakura_seed() -> (ZebraDb, Arc, Arc) { + let genesis = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into::>() + .expect("genesis block deserializes"); + let block1 = zebra_test::vectors::BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into::>() + .expect("block 1 deserializes"); + let state = state_with_genesis_and_zakura_seed(&Mainnet, genesis.clone()); + + (state, genesis, block1) +} + fn state_with_genesis(network: &Network, genesis: Arc) -> ZebraDb { + state_with_genesis_config(network, genesis, Config::ephemeral()) +} + +fn state_with_genesis_and_zakura_seed(network: &Network, genesis: Arc) -> ZebraDb { + let mut config = Config::ephemeral(); + config.enable_zakura_header_seed_from_committed_blocks = true; + + state_with_genesis_config(network, genesis, config) +} + +fn state_with_genesis_config(network: &Network, genesis: Arc, config: Config) -> ZebraDb { let state = ZebraDb::new( - &Config::ephemeral(), + &config, STATE_DATABASE_KIND, &state_database_format_version_in_code(), network, @@ -738,8 +957,9 @@ fn commit_header_range( headers: &[Arc], ) -> block::Hash { let mut batch = DiskWriteBatch::new(); + let body_sizes = vec![0; headers.len()]; let committed_hash = batch - .prepare_header_range_batch(state, anchor, headers) + .prepare_header_range_batch(state, anchor, headers, &body_sizes) .expect("header range is valid"); state .write_batch(batch) @@ -754,7 +974,7 @@ fn write_full_block_header_and_transactions(state: &ZebraDb, block: Arc) let mut batch = DiskWriteBatch::new(); batch - .prepare_block_header_and_transaction_data_batch(&state.db, &finalized) + .prepare_block_header_and_transaction_data_batch(state, &finalized) .expect("full block header and transaction batch is valid"); state.db.write(batch).expect("full block batch writes"); } @@ -832,7 +1052,7 @@ fn test_block_db_round_trip_with( // Skip validation by writing the block directly to the database let mut batch = DiskWriteBatch::new(); batch - .prepare_block_header_and_transaction_data_batch(&state.db, &finalized) + .prepare_block_header_and_transaction_data_batch(&state, &finalized) .expect("test block header and transaction batch is valid"); state.db.write(batch).expect("block is valid for writing"); diff --git a/zebra-state/src/service/read.rs b/zebra-state/src/service/read.rs index c27aee1ee82..94780ae912d 100644 --- a/zebra-state/src/service/read.rs +++ b/zebra-state/src/service/read.rs @@ -30,8 +30,8 @@ pub use address::{ }; pub use block::{ any_block, any_transaction, any_utxo, block, block_and_size, block_header, block_info, - mined_transaction, transaction_hashes_for_any_block, transaction_hashes_for_block, - unspent_utxo, + block_size_hints, mined_transaction, transaction_hashes_for_any_block, + transaction_hashes_for_block, unspent_utxo, }; #[cfg(feature = "indexer")] diff --git a/zebra-state/src/service/read/block.rs b/zebra-state/src/service/read/block.rs index 2c6200dcbb1..11b4fe5defe 100644 --- a/zebra-state/src/service/read/block.rs +++ b/zebra-state/src/service/read/block.rs @@ -25,6 +25,7 @@ use zebra_chain::{ }; use crate::{ + constants::MAX_HEADER_SYNC_HEIGHT_RANGE, response::{AnyTx, MinedTx}, service::{ finalized_state::ZebraDb, @@ -362,3 +363,39 @@ where .and_then(|chain| chain.as_ref().block_info(hash_or_height)) .or_else(|| db.block_info(hash_or_height)) } + +/// Returns block-size hints for a contiguous height range. +/// +/// Confirmed committed sizes from [`BlockInfo`] win over untrusted advertised +/// header-sync hints. Advertised hints are scheduling-only data and are never +/// consulted by verification. +pub fn block_size_hints( + chain: Option, + db: &ZebraDb, + from: Height, + count: u32, +) -> Vec<(Height, Option)> +where + C: AsRef, +{ + let count = count.min(MAX_HEADER_SYNC_HEIGHT_RANGE); + let mut hints = Vec::with_capacity( + usize::try_from(count).expect("capped block size hint count fits in usize"), + ); + + for offset in 0..count { + let Some(height) = from.0.checked_add(offset).map(Height) else { + break; + }; + let confirmed_size = chain + .as_ref() + .and_then(|chain| chain.as_ref().block_info(height.into())) + .or_else(|| db.block_info(height.into())) + .map(|info| info.size()); + let size = confirmed_size.or_else(|| db.advertised_body_size(height)); + + hints.push((height, size)); + } + + hints +} diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index fb176793304..06cc4e36237 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -14,7 +14,7 @@ use zebra_chain::{ chain_tip::ChainTip, fmt::SummaryDebug, parameters::{Network, NetworkUpgrade}, - serialization::{ZcashDeserialize, ZcashDeserializeInto}, + serialization::{ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize}, transaction, transparent, value_balance::ValueBalance, }; @@ -482,17 +482,57 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R ReadResponse::FinalizedTip(Some((Height(0), genesis.hash()))), ); + let genesis_size = u32::try_from(genesis.zcash_serialize_to_vec()?.len()) + .expect("serialized block size fits in u32"); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::BlocksByHeightRange { + start: Height(0), + count: 3, + }) + .await?, + ReadResponse::Blocks(vec![( + Height(0), + genesis.clone(), + genesis.zcash_serialize_to_vec()?.len(), + )]), + ); + + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::BlockSizeHints { + from: Height(0), + count: 1, + }) + .await?, + ReadResponse::BlockSizeHints(vec![(Height(0), Some(genesis_size))]), + ); + assert_eq!( state .clone() .oneshot(Request::CommitHeaderRange { anchor: genesis.hash(), headers: vec![block1.header.clone(), block2.header.clone()], + body_sizes: vec![999_999, 0], }) .await?, Response::Committed(block2_hash), ); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::BlockSizeHints { + from: Height(1), + count: 2, + }) + .await?, + ReadResponse::BlockSizeHints(vec![(Height(1), Some(999_999)), (Height(2), None)]), + ); + assert_eq!( state.clone().oneshot(Request::Depth(block1_hash)).await?, Response::Depth(None), @@ -534,6 +574,7 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R ); assert_eq!( state + .clone() .oneshot(Request::AnyChainBlock(block1_hash.into())) .await?, Response::Block(None), @@ -566,6 +607,17 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R (Height(2), block2_hash, block2.header.clone()), ]), ); + assert_eq!( + read_state + .clone() + .oneshot(ReadRequest::BlocksByHeightRange { + start: Height(1), + count: 2, + }) + .await?, + ReadResponse::Blocks(Vec::new()), + ); + assert_eq!( read_state .clone() @@ -583,6 +635,7 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R .await?, ReadResponse::MissingBlockBodies(vec![Height(1), Height(2)]), ); + assert_eq!( read_state.oneshot(ReadRequest::FinalizedTip).await?, ReadResponse::FinalizedTip(Some((Height(0), genesis.hash()))), @@ -591,6 +644,90 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R Ok(()) } +/// A node still in the finalized (checkpoint) write phase must be able to commit +/// a Zakura header range. +/// +/// This reproduces the Zakura catch-up deadlock. A freshly started node has an +/// empty non-finalized chain set, so it keeps its finalized block-write sender +/// and the block write task drains the finalized channel before handling any +/// non-finalized message. The finalized->non-finalized transition only fires +/// when a non-finalized block is queued as a child of the finalized tip (the +/// legacy commit path). A node catching up to a peer that sits at a *static* +/// tip over Zakura commits header ranges via `CommitHeaderRange` (a +/// non-finalized message) but never queues such a block, so before the fix the +/// request never completes: the header tip stays at genesis, block sync stays +/// gated off (`best_header_tip <= verified_block_tip`), and the node stalls. +/// +/// Unlike `header_only_service_requests_preserve_body_boundary`, this test does +/// NOT drop `block_write_sender.finalized`, so it exercises the real catch-up +/// state. Without the fix the `CommitHeaderRange` future never resolves and the +/// bounded wait below fails the test. +#[tokio::test(flavor = "multi_thread")] +async fn commit_header_range_completes_while_in_finalized_write_phase( +) -> std::result::Result<(), BoxError> { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let (mut state_service, read_state, _, _) = + StateService::new(Config::ephemeral(), &network, Height::MAX, 0).await; + let genesis = + zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES.zcash_deserialize_into::>()?; + let block1 = + zebra_test::vectors::BLOCK_MAINNET_1_BYTES.zcash_deserialize_into::>()?; + let block2 = + zebra_test::vectors::BLOCK_MAINNET_2_BYTES.zcash_deserialize_into::>()?; + let block2_hash = block2.hash(); + + assert_eq!( + state_service + .ready() + .await? + .call(Request::CommitCheckpointVerifiedBlock( + CheckpointVerifiedBlock::from(genesis.clone()), + )) + .await?, + Response::Committed(genesis.hash()), + ); + + // The node is still in the finalized write phase: committing a checkpoint + // block does not trigger the finalized->non-finalized transition, which is + // exactly the state a Zakura node catching up to a static tip is stuck in. + assert!( + state_service.block_write_sender.finalized.is_some(), + "a fresh node stays in the finalized write phase after a checkpoint commit", + ); + let state = Buffer::new(BoxService::new(state_service), 1); + + let committed = tokio::time::timeout( + Duration::from_secs(20), + state.clone().oneshot(Request::CommitHeaderRange { + anchor: genesis.hash(), + headers: vec![block1.header.clone(), block2.header.clone()], + body_sizes: vec![999_999, 0], + }), + ) + .await + .expect("CommitHeaderRange must not deadlock while in the finalized write phase")?; + + assert_eq!(committed, Response::Committed(block2_hash)); + + assert_eq!( + state + .clone() + .oneshot(Request::CommitCheckpointVerifiedBlock( + CheckpointVerifiedBlock::from(block1.clone()), + )) + .await?, + Response::Committed(block1.hash()), + ); + + assert_eq!( + read_state.oneshot(ReadRequest::FinalizedTip).await?, + ReadResponse::FinalizedTip(Some((Height(1), block1.hash()))), + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn header_range_reads_include_non_finalized_best_chain_blocks() -> Result<()> { let _init_guard = zebra_test::init(); @@ -882,9 +1019,12 @@ proptest! { for block in finalized_blocks { let expected_block = block.clone(); - let expected_action = if expected_block.height <= block::Height(1) { - // 0: reset by both initialization and the Genesis network upgrade - // 1: reset by the BeforeOverwinter network upgrade + let expected_action = if expected_block.height == block::Height(0) { + // Height 0 is reset by initialization. The BeforeOverwinter upgrade + // (activation height 1) also resets at height 0 rather than at height 1, + // because `ChainTipChange` resets one block *before* an activation height + // (it checks `height.next()`, matching the height the mempool verifies + // against). See `ChainTipChange::action`. TipAction::reset_with(expected_block.clone().into()) } else { TipAction::grow_with(expected_block.clone().into()) @@ -906,12 +1046,10 @@ proptest! { for block in non_finalized_blocks { let expected_block = block.clone(); - let expected_action = if expected_block.height == block::Height(1) { - // 1: reset by the BeforeOverwinter network upgrade - TipAction::reset_with(expected_block.clone().into()) - } else { - TipAction::grow_with(expected_block.clone().into()) - }; + // The genesis block (height 0) is always finalized, and the BeforeOverwinter + // reset fires at height 0 (one block before its activation height of 1), so + // every non-finalized block (height >= 1) grows the chain. + let expected_action = TipAction::grow_with(expected_block.clone().into()); let result_receiver = state_service.queue_and_commit_to_non_finalized_state(block); let result = result_receiver.blocking_recv(); diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index d9145d82300..96070107433 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -1,13 +1,15 @@ //! Writing blocks to the finalized and non-finalized states. use std::{ + collections::VecDeque, path::{Path, PathBuf}, sync::Arc, + time::Duration, }; use indexmap::IndexMap; use tokio::sync::{ - mpsc::{UnboundedReceiver, UnboundedSender}, + mpsc::{error::TryRecvError, UnboundedReceiver, UnboundedSender}, oneshot, watch, }; @@ -119,6 +121,33 @@ fn update_latest_chain_channels( tip_block_height } +fn commit_header_range( + finalized_state: &FinalizedState, + anchor: block::Hash, + headers: Vec>, + body_sizes: Vec, + rsp_tx: oneshot::Sender>, +) { + let mut batch = crate::service::finalized_state::DiskWriteBatch::new(); + let result = batch + .prepare_header_range_batch(&finalized_state.db, anchor, &headers, &body_sizes) + .and_then(|hash| { + finalized_state + .db + .write_batch(batch) + .map(|()| hash) + .map_err(|error| { + tracing::error!(?error, "failed to write validated header range"); + + CommitHeaderRangeError::StorageWriteError { + error: error.to_string(), + } + }) + }); + + let _ = rsp_tx.send(result); +} + /// A worker task that reads, validates, and writes blocks to the /// `finalized_state` or `non_finalized_state`. struct WriteBlockWorkerTask { @@ -126,6 +155,7 @@ struct WriteBlockWorkerTask { non_finalized_block_write_receiver: UnboundedReceiver, finalized_state: FinalizedState, non_finalized_state: NonFinalizedState, + seed_zakura_header_from_best_chain_commits: bool, invalid_block_reset_sender: UnboundedSender, /// Signals the [`crate::service::StateService`] that a non-finalized block was rejected by /// the write task, so its hash should be removed from @@ -152,6 +182,7 @@ pub enum NonFinalizedWriteMessage { CommitHeaderRange { anchor: block::Hash, headers: Vec>, + body_sizes: Vec, rsp_tx: oneshot::Sender>, }, /// The hash of a block that should be invalidated and removed from @@ -224,6 +255,11 @@ impl BlockWriteSender { let (non_finalized_rejected_sender, non_finalized_rejected_receiver) = tokio::sync::mpsc::unbounded_channel(); + let seed_zakura_header_from_best_chain_commits = finalized_state + .db + .config() + .enable_zakura_header_seed_from_committed_blocks; + let span = Span::current(); let task = std::thread::spawn(move || { span.in_scope(|| { @@ -232,6 +268,7 @@ impl BlockWriteSender { non_finalized_block_write_receiver, finalized_state, non_finalized_state, + seed_zakura_header_from_best_chain_commits, invalid_block_reset_sender, non_finalized_rejected_sender, chain_tip_sender, @@ -276,14 +313,40 @@ impl WriteBlockWorkerTask { non_finalized_rejected_sender, chain_tip_sender, non_finalized_state_sender, + seed_zakura_header_from_best_chain_commits, backup_dir_path, } = &mut self; let mut prev_finalized_note_commitment_trees = None; + let mut deferred_non_finalized_messages = VecDeque::new(); // Write all the finalized blocks sent by the state, // until the state closes the finalized block channel's sender. - while let Some(ordered_block) = finalized_block_write_receiver.blocking_recv() { + loop { + match non_finalized_block_write_receiver.try_recv() { + Ok(NonFinalizedWriteMessage::CommitHeaderRange { + anchor, + headers, + body_sizes, + rsp_tx, + }) => { + commit_header_range(finalized_state, anchor, headers, body_sizes, rsp_tx); + continue; + } + Ok(msg) => deferred_non_finalized_messages.push_back(msg), + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => {} + } + + let ordered_block = match finalized_block_write_receiver.try_recv() { + Ok(block) => block, + Err(TryRecvError::Empty) => { + std::thread::park_timeout(Duration::from_millis(10)); + continue; + } + Err(TryRecvError::Disconnected) => break, + }; + // TODO: split these checks into separate functions if invalid_block_reset_sender.is_closed() { @@ -362,35 +425,19 @@ impl WriteBlockWorkerTask { // Save any errors to propagate down to queued child blocks let mut parent_error_map: IndexMap = IndexMap::new(); - while let Some(msg) = non_finalized_block_write_receiver.blocking_recv() { + while let Some(msg) = deferred_non_finalized_messages + .pop_front() + .or_else(|| non_finalized_block_write_receiver.blocking_recv()) + { let queued_child_and_rsp_tx = match msg { NonFinalizedWriteMessage::Commit(queued_child) => Some(queued_child), NonFinalizedWriteMessage::CommitHeaderRange { anchor, headers, + body_sizes, rsp_tx, } => { - let mut batch = crate::service::finalized_state::DiskWriteBatch::new(); - let result = batch - .prepare_header_range_batch(&finalized_state.db, anchor, &headers) - .and_then(|hash| { - finalized_state - .db - .write_batch(batch) - .map(|()| hash) - .map_err(|error| { - tracing::error!( - ?error, - "failed to write validated header range" - ); - - CommitHeaderRangeError::StorageWriteError { - error: error.to_string(), - } - }) - }); - - let _ = rsp_tx.send(result); + commit_header_range(finalized_state, anchor, headers, body_sizes, rsp_tx); continue; } NonFinalizedWriteMessage::Invalidate { hash, rsp_tx } => { @@ -418,6 +465,8 @@ impl WriteBlockWorkerTask { let child_hash = queued_child.hash; let parent_hash = queued_child.block.header.previous_block_hash; + let child_height = queued_child.height; + let child_block = queued_child.block.clone(); let parent_error = parent_error_map.get(&parent_hash); // If the parent block was marked as rejected, also reject all its children. @@ -468,6 +517,19 @@ impl WriteBlockWorkerTask { continue; } + if should_seed_zakura_header_from_non_finalized_commit( + *seed_zakura_header_from_best_chain_commits, + non_finalized_state, + child_height, + child_hash, + ) { + seed_zakura_header_from_committed_block( + &finalized_state.db, + child_height, + &child_block, + ); + } + // Committing blocks to the finalized state keeps the same chain, // so we can update the chain seen by the rest of the application now. // @@ -521,3 +583,112 @@ impl WriteBlockWorkerTask { std::mem::drop(self.finalized_state); } } + +fn seed_zakura_header_from_committed_block( + finalized_state: &ZebraDb, + height: block::Height, + block: &Arc, +) { + match finalized_state.seed_zakura_header_from_committed_block(height, block) { + Ok(()) => { + tracing::trace!(?height, hash = ?block.hash(), "seeded Zakura header from committed block"); + } + Err(error) => { + tracing::warn!( + ?height, + hash = ?block.hash(), + ?error, + "failed to seed Zakura header from committed block" + ); + } + } +} + +fn should_seed_zakura_header_from_non_finalized_commit( + enabled: bool, + non_finalized_state: &NonFinalizedState, + height: block::Height, + hash: block::Hash, +) -> bool { + enabled && non_finalized_state.best_tip() == Some((height, hash)) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use zebra_chain::{ + parameters::Network, serialization::ZcashDeserializeInto, value_balance::ValueBalance, + }; + + use crate::{ + arbitrary::Prepare, + service::{ + finalized_state::FinalizedState, + non_finalized_state::NonFinalizedState, + write::{ + seed_zakura_header_from_committed_block, + should_seed_zakura_header_from_non_finalized_commit, + }, + }, + tests::FakeChainHelper, + Config, + }; + + #[test] + fn side_chain_commit_does_not_seed_zakura_headers() { + let _init_guard = zebra_test::init(); + + let network = Network::Mainnet; + let mut config = Config::ephemeral(); + config.enable_zakura_header_seed_from_committed_blocks = true; + let finalized_state = FinalizedState::new( + &config, + &network, + #[cfg(feature = "elasticsearch")] + false, + ); + finalized_state.set_finalized_value_pool(ValueBalance::fake_populated_pool()); + + let parent = zebra_test::vectors::BLOCK_MAINNET_434873_BYTES + .zcash_deserialize_into::>() + .expect("block deserializes"); + let best_block = parent.make_fake_child().set_work(10); + let side_block = parent.make_fake_child().set_work(1); + let best_height = best_block + .coinbase_height() + .expect("fake child block has a coinbase height"); + + let mut non_finalized_state = NonFinalizedState::new(&network); + + non_finalized_state + .commit_new_chain(best_block.clone().prepare(), &finalized_state) + .expect("best block commits to a new chain"); + assert!(should_seed_zakura_header_from_non_finalized_commit( + true, + &non_finalized_state, + best_height, + best_block.hash(), + )); + seed_zakura_header_from_committed_block(&finalized_state.db, best_height, &best_block); + + non_finalized_state + .commit_new_chain(side_block.clone().prepare(), &finalized_state) + .expect("side block commits to a losing fork"); + assert!(!should_seed_zakura_header_from_non_finalized_commit( + true, + &non_finalized_state, + best_height, + side_block.hash(), + )); + + assert_eq!( + finalized_state.db.best_header_tip(), + Some((best_height, best_block.hash())) + ); + assert_eq!( + finalized_state.db.headers_by_height_range(best_height, 1), + vec![(best_height, best_block.hash(), best_block.header.clone())], + ); + } +} diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index ceb7d89befc..9cd1568d3a9 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -302,7 +302,7 @@ color-eyre = { workspace = true } zebra-chain = { path = "../zebra-chain", version = "9.0.0", features = ["proptest-impl"] } zebra-consensus = { path = "../zebra-consensus", version = "8.0.0", features = ["proptest-impl"] } -zebra-network = { path = "../zebra-network", version = "8.0.0", features = ["proptest-impl"] } +zebra-network = { path = "../zebra-network", version = "8.0.0", features = ["proptest-impl", "zakura-testkit"] } zebra-state = { path = "../zebra-state", version = "8.0.0", features = ["proptest-impl"] } zebra-test = { path = "../zebra-test", version = "3.0.0" } diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 42f69720925..190b341d324 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -73,27 +73,30 @@ //! //! Some of the diagnostic features are optional, and need to be enabled at compile-time. -use std::{future::Future, net::SocketAddr, path::Path, sync::Arc}; +mod zakura; + +use std::{net::SocketAddr, path::Path, sync::Arc}; use abscissa_core::{config, Command, FrameworkError}; use color_eyre::eyre::{eyre, Report}; use futures::FutureExt; use tokio::{ pin, select, - sync::{mpsc, oneshot, watch}, + sync::{oneshot, watch}, }; -use tower::{builder::ServiceBuilder, util::BoxService, Service, ServiceExt}; +use tower::{builder::ServiceBuilder, util::BoxService, ServiceExt}; use tracing_futures::Instrument; -use zebra_chain::block::{self, genesis::regtest_genesis_block}; +use zebra_chain::block::genesis::regtest_genesis_block; use zebra_consensus::router::BackgroundTaskHandles; use zebra_network::types::PeerServices; -use zebra_network::zakura::{ - HeaderSyncAction, HeaderSyncCommitFailureKind, HeaderSyncEvent, HeaderSyncFrontiers, - ZakuraEndpoint, ZakuraHeaderSyncDriverStartup, DEFAULT_HS_RANGE, -}; use zebra_rpc::{methods::RpcImpl, server::RpcServer, SubmitBlockChannel}; +use zakura::{ + drive_block_sync_actions, drive_zakura_header_sync_actions, mirror_zakura_full_block_commits, + zakura_header_sync_driver_startup, ZakuraHeaderSyncDriverHandles, +}; + use crate::{ application::{build_version, user_agent, LAST_WARN_ERROR_LOG_SENDER}, components::{ @@ -161,465 +164,8 @@ fn check_tcp_slow_start_after_idle() { ); } -async fn zakura_header_sync_driver_startup( - read_state: zebra_state::ReadStateService, - network: &zebra_chain::parameters::Network, -) -> Result { - let best_header_tip = match read_state - .clone() - .oneshot(zebra_state::ReadRequest::BestHeaderTip) - .await - .map_err(|error| eyre!("{error}"))? - { - zebra_state::ReadResponse::BestHeaderTip(tip) => tip, - response => Err(eyre!("unexpected BestHeaderTip response: {response:?}"))?, - }; - - let finalized_tip = match read_state - .clone() - .oneshot(zebra_state::ReadRequest::FinalizedTip) - .await - .map_err(|error| eyre!("{error}"))? - { - zebra_state::ReadResponse::FinalizedTip(tip) => tip, - response => Err(eyre!("unexpected FinalizedTip response: {response:?}"))?, - }; - - let verified_block_tip = match read_state - .oneshot(zebra_state::ReadRequest::Tip) - .await - .map_err(|error| eyre!("{error}"))? - { - zebra_state::ReadResponse::Tip(tip) => tip, - response => Err(eyre!("unexpected Tip response: {response:?}"))?, - }; - - let empty_state_tip = (block::Height(0), network.genesis_hash()); - Ok(ZakuraHeaderSyncDriverStartup { - frontiers: HeaderSyncFrontiers { - finalized_height: finalized_tip.map_or(block::Height(0), |(height, _)| height), - verified_block_tip: verified_block_tip.map_or(block::Height(0), |(height, _)| height), - }, - best_header_tip: Some(best_header_tip.unwrap_or(empty_state_tip)), - }) -} - -async fn drive_zakura_header_sync_actions( - mut actions: mpsc::Receiver, - endpoint: ZakuraEndpoint, - header_sync: zebra_network::zakura::HeaderSyncHandle, - state: State, - read_state: ReadState, - block_verifier: BlockVerifier, - shutdown: impl Future + Send + 'static, -) where - State: Service< - zebra_state::Request, - Response = zebra_state::Response, - Error = zebra_state::BoxError, - > + Clone - + Send - + 'static, - State::Future: Send + 'static, - ReadState: Service< - zebra_state::ReadRequest, - Response = zebra_state::ReadResponse, - Error = zebra_state::BoxError, - > + Clone - + Send - + 'static, - ReadState::Future: Send + 'static, - BlockVerifier: - Service + Clone + Send + 'static, - BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, - BlockVerifier::Future: Send + 'static, -{ - pin!(shutdown); - loop { - let action = select! { - _ = &mut shutdown => return, - action = actions.recv() => { - let Some(action) = action else { - return; - }; - action - } - }; - - match action { - HeaderSyncAction::Misbehavior { peer, reason } => { - debug!( - ?peer, - ?reason, - "disconnecting peer for Zakura header-sync violation" - ); - let _ = endpoint.supervisor().disconnect_peer(&peer).await; - } - HeaderSyncAction::NewBlockReceived { - peer, - height, - hash, - block, - } => { - match block_verifier - .clone() - .oneshot(zebra_consensus::Request::Commit(block.clone())) - .await - { - Ok(committed_hash) if committed_hash == hash => { - let _ = header_sync - .send(HeaderSyncEvent::NewBlockAccepted { - peer, - height, - hash, - block, - }) - .await; - } - Ok(committed_hash) => { - warn!( - ?peer, - ?hash, - ?committed_hash, - "Zakura NewBlock verifier returned an unexpected hash" - ); - let _ = header_sync - .send(HeaderSyncEvent::NewBlockRejected { peer, hash }) - .await; - } - Err(error) => { - if block_verify_error_is_duplicate(&error) { - debug!( - ?peer, - ?height, - ?hash, - ?error, - "Zakura NewBlock was already known by the block verifier" - ); - let _ = header_sync - .send(HeaderSyncEvent::NewBlockDuplicate { peer, height, hash }) - .await; - continue; - } - - debug!( - ?peer, - ?hash, - ?error, - "Zakura NewBlock rejected by block verifier" - ); - let _ = header_sync - .send(HeaderSyncEvent::NewBlockRejected { peer, hash }) - .await; - } - } - } - HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { - match read_state - .clone() - .oneshot(zebra_state::ReadRequest::HeadersByHeightRange { start, count }) - .await - { - Ok(zebra_state::ReadResponse::Headers(headers)) => { - let headers = headers - .into_iter() - .map(|(_height, _hash, header)| header) - .collect(); - let _ = header_sync - .send(HeaderSyncEvent::HeaderRangeResponseReady { - peer, - start_height: start, - requested_count: count, - headers, - }) - .await; - } - Ok(response) => { - warn!(?peer, ?response, "unexpected HeadersByHeightRange response"); - let _ = header_sync - .send(HeaderSyncEvent::HeaderRangeResponseFinished { - peer, - start_height: start, - requested_count: count, - returned_count: 0, - }) - .await; - } - Err(error) => { - warn!( - ?peer, - ?error, - "failed to read Zakura Headers response from state" - ); - let _ = header_sync - .send(HeaderSyncEvent::HeaderRangeResponseFinished { - peer, - start_height: start, - requested_count: count, - returned_count: 0, - }) - .await; - } - } - } - HeaderSyncAction::CommitHeaderRange { - peer, - anchor, - start_height, - headers, - finalized: _finalized, - } => { - let count = u32::try_from(headers.len()).unwrap_or(u32::MAX); - match state - .clone() - .oneshot(zebra_state::Request::CommitHeaderRange { anchor, headers }) - .await - { - Ok(zebra_state::Response::Committed(tip_hash)) => { - let tip_height = - block::Height(start_height.0.saturating_add(count.saturating_sub(1))); - let _ = header_sync - .send(HeaderSyncEvent::HeaderRangeCommitted { - start_height, - tip_height, - tip_hash, - }) - .await; - } - Ok(response) => { - warn!(?peer, ?response, "unexpected CommitHeaderRange response"); - let _ = header_sync - .send(HeaderSyncEvent::HeaderRangeCommitFailed { - peer, - start_height, - count, - kind: HeaderSyncCommitFailureKind::Local, - }) - .await; - } - Err(error) => { - let kind = header_range_commit_failure_kind(error.as_ref()); - debug!( - ?peer, - ?start_height, - ?count, - ?kind, - ?error, - "Zakura header range commit failed" - ); - let _ = header_sync - .send(HeaderSyncEvent::HeaderRangeCommitFailed { - peer, - start_height, - count, - kind, - }) - .await; - } - } - } - HeaderSyncAction::QueryBestHeaderTip => { - match read_state - .clone() - .oneshot(zebra_state::ReadRequest::BestHeaderTip) - .await - { - Ok(zebra_state::ReadResponse::BestHeaderTip(Some((tip_height, tip_hash)))) => { - // Reuse the range-commit event as a startup/tip-refresh fact: a - // single-height covered mark is harmless and refreshes the reactor tip. - let _ = header_sync - .send(HeaderSyncEvent::HeaderRangeCommitted { - start_height: tip_height, - tip_height, - tip_hash, - }) - .await; - } - Ok(zebra_state::ReadResponse::BestHeaderTip(None)) => {} - Ok(response) => warn!(?response, "unexpected BestHeaderTip response"), - Err(error) => warn!(?error, "failed to query Zakura best header tip"), - } - } - HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { - log_missing_block_bodies(read_state.clone(), from, limit).await; - } - HeaderSyncAction::BodyGaps { from, to } => { - let limit = - to.0.saturating_sub(from.0) - .saturating_add(1) - .min(DEFAULT_HS_RANGE); - log_missing_block_bodies(read_state.clone(), from, limit).await; - } - } - } -} - -fn block_verify_error_is_duplicate(error: &Error) -> bool -where - Error: std::fmt::Debug + Send + Sync + 'static, -{ - let error = error as &dyn std::any::Any; - - error - .downcast_ref::() - .is_some_and(zebra_consensus::RouterError::is_duplicate_request) - || error - .downcast_ref::() - .is_some_and(zebra_consensus::VerifyBlockError::is_duplicate_request) - || error - .downcast_ref::() - .is_some_and(|error| { - error - .downcast_ref::() - .is_some_and(zebra_consensus::RouterError::is_duplicate_request) - || error - .downcast_ref::() - .is_some_and(zebra_consensus::VerifyBlockError::is_duplicate_request) - }) -} - -async fn log_missing_block_bodies(read_state: ReadState, from: block::Height, limit: u32) -where - ReadState: Service< - zebra_state::ReadRequest, - Response = zebra_state::ReadResponse, - Error = zebra_state::BoxError, - > + Send - + 'static, - ReadState::Future: Send + 'static, -{ - // This driver boundary is observational only: header sync exposes bounded body gaps - // from state, but block download must consume them through its own policy/watches. - match read_state - .oneshot(zebra_state::ReadRequest::MissingBlockBodies { from, limit }) - .await - { - Ok(zebra_state::ReadResponse::MissingBlockBodies(heights)) => { - let first = heights.first().copied(); - let last = heights.last().copied(); - let count = heights.len(); - debug!( - ?from, - ?limit, - ?count, - ?first, - ?last, - "Zakura header-known body gaps from state" - ); - } - Ok(response) => warn!(?response, "unexpected MissingBlockBodies response"), - Err(error) => warn!(?error, "failed to query Zakura missing block bodies"), - } -} - -fn header_range_commit_failure_kind( - error: &(dyn std::error::Error + Send + Sync + 'static), -) -> HeaderSyncCommitFailureKind { - let Some(error) = error.downcast_ref::() else { - return HeaderSyncCommitFailureKind::Local; - }; - - match error { - zebra_state::CommitHeaderRangeError::StorageWriteError { .. } - | zebra_state::CommitHeaderRangeError::SendCommitRequestFailed - | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { - HeaderSyncCommitFailureKind::Local - } - zebra_state::CommitHeaderRangeError::EmptyRange - | zebra_state::CommitHeaderRangeError::RangeTooLong { .. } - | zebra_state::CommitHeaderRangeError::UnknownAnchor { .. } - | zebra_state::CommitHeaderRangeError::HeightOverflow - | zebra_state::CommitHeaderRangeError::ImmutableConflict { .. } - | zebra_state::CommitHeaderRangeError::ReorgTooDeep { .. } - | zebra_state::CommitHeaderRangeError::CheckpointConflict { .. } - | zebra_state::CommitHeaderRangeError::ConflictingFullBlockHeader { .. } - | zebra_state::CommitHeaderRangeError::ValidateContextError(_) => { - HeaderSyncCommitFailureKind::InvalidPeerRange - } - _ => HeaderSyncCommitFailureKind::Local, - } -} - -async fn mirror_zakura_full_block_commits( - mut chain_tip_change: zebra_state::ChainTipChange, - read_state: ReadState, - header_sync: zebra_network::zakura::HeaderSyncHandle, - shutdown: impl Future + Send + 'static, -) where - ReadState: Service< - zebra_state::ReadRequest, - Response = zebra_state::ReadResponse, - Error = zebra_state::BoxError, - > + Clone - + Send - + 'static, - ReadState::Future: Send + 'static, -{ - pin!(shutdown); - loop { - let action = select! { - _ = &mut shutdown => return, - action = chain_tip_change.wait_for_tip_change() => { - let Ok(action) = action else { - return; - }; - action - } - }; - let height = action.best_tip_height(); - let hash = action.best_tip_hash(); - - let finalized_height = match read_state - .clone() - .oneshot(zebra_state::ReadRequest::FinalizedTip) - .await - { - Ok(zebra_state::ReadResponse::FinalizedTip(Some((height, _hash)))) => height, - Ok(zebra_state::ReadResponse::FinalizedTip(None)) => block::Height(0), - Ok(response) => { - warn!(?response, "unexpected FinalizedTip response"); - block::Height(0) - } - Err(error) => { - warn!(?error, "failed to query Zakura finalized frontier"); - block::Height(0) - } - }; - - let _ = header_sync - .send(HeaderSyncEvent::StateFrontiersChanged( - HeaderSyncFrontiers { - finalized_height, - verified_block_tip: height, - }, - )) - .await; - - match read_state - .clone() - .oneshot(zebra_state::ReadRequest::Block(hash.into())) - .await - { - Ok(zebra_state::ReadResponse::Block(Some(block))) => { - let _ = header_sync - .send(HeaderSyncEvent::FullBlockCommitted { - height, - hash, - header: block.header.clone(), - }) - .await; - } - Ok(zebra_state::ReadResponse::Block(None)) => { - debug!( - ?height, - ?hash, - "Zakura full-block mirror could not find committed tip block" - ); - } - Ok(response) => warn!(?response, "unexpected block lookup response"), - Err(error) => warn!(?error, "failed to mirror Zakura full-block commit"), - } - } +fn use_zakura_block_sync(config: &zebra_network::Config) -> bool { + config.v2_p2p } #[cfg(not(target_os = "linux"))] @@ -953,9 +499,12 @@ impl StartCmd { info!("opening database, this may take a few minutes"); + let mut state_config = config.state.clone(); + state_config.enable_zakura_header_seed_from_committed_blocks = config.network.v2_p2p; + let (state_service, read_only_state_service, latest_chain_tip, chain_tip_change) = zebra_state::init( - config.state.clone(), + state_config, &config.network.network, max_checkpoint_height, config.sync.checkpoint_verify_concurrency_limit @@ -1036,6 +585,7 @@ impl StartCmd { .await; if let Some(endpoint) = zakura_endpoint.clone() { + let trace = endpoint.trace(); if let (Some(header_sync), Some(shutdown), Some(actions)) = ( endpoint.header_sync(), endpoint.header_sync_shutdown(), @@ -1044,22 +594,52 @@ impl StartCmd { let driver_task = tokio::spawn( drive_zakura_header_sync_actions( actions, - endpoint.clone(), - header_sync.clone(), + ZakuraHeaderSyncDriverHandles { + endpoint: endpoint.clone(), + header_sync: header_sync.clone(), + }, state.clone(), read_only_state_service.clone(), block_verifier_router.clone(), + trace.clone(), shutdown.clone().cancelled_owned(), ) .in_current_span(), ); endpoint.push_header_sync_task(driver_task).await; + if let (Some(block_sync), Some(block_actions)) = ( + endpoint.block_sync(), + endpoint.take_block_sync_actions().await, + ) { + let block_driver_task = tokio::spawn( + drive_block_sync_actions( + block_actions, + endpoint.supervisor(), + Some(endpoint.clone()), + block_sync.clone(), + latest_chain_tip.clone(), + read_only_state_service.clone(), + block_verifier_router.clone(), + max_checkpoint_height, + config.sync.checkpoint_verify_concurrency_limit, + config.sync.full_verify_concurrency_limit, + trace.clone(), + shutdown.clone().cancelled_owned(), + ) + .in_current_span(), + ); + endpoint.push_block_sync_task(block_driver_task).await; + } + let full_block_task = tokio::spawn( mirror_zakura_full_block_commits( chain_tip_change.clone(), + latest_chain_tip.clone(), read_only_state_service.clone(), header_sync, + endpoint.clone(), + trace, shutdown.cancelled_owned(), ) .in_current_span(), @@ -1294,7 +874,12 @@ impl StartCmd { // that multi-hop block propagation works: gossiped blocks that arrive out of // order (e.g. only the latest tip hash was gossiped) will be recovered by the // syncer using block locators within REGTEST_SYNC_RESTART_DELAY (2 seconds). + // + // `debug_skip_regtest_genesis_self_seed` opts a node out of this shortcut so it + // downloads genesis from a peer instead, exercising the production + // genesis-bootstrap path (e.g. a Zakura-only node fetching genesis over Zakura). if is_regtest + && !config.sync.debug_skip_regtest_genesis_self_seed && !syncer .state_contains(config.network.network.genesis_hash()) .await? @@ -1311,7 +896,12 @@ impl StartCmd { "validated block hash should match network genesis hash" ) } - let syncer_task_handle = tokio::spawn(syncer.sync().in_current_span()); + let syncer_task_handle = if use_zakura_block_sync(&config.network) { + info!("Zakura block sync is replacing the legacy ChainSync body downloader"); + tokio::spawn(syncer.bootstrap_genesis_then_pause().in_current_span()) + } else { + tokio::spawn(syncer.sync().in_current_span()) + }; // And finally, spawn the internal Zcash miner, if it is enabled. // @@ -2333,6 +1923,127 @@ mod tests { #[cfg(test)] mod zakura_header_sync_driver_tests { use super::*; + use std::{ + collections::VecDeque, + future, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, Mutex, + }, + time::Duration, + }; + + use futures::stream::{FuturesUnordered, StreamExt}; + use tokio::sync::mpsc; + use tower::{service_fn, util::BoxService, ServiceExt}; + use zebra_chain::block; + use zebra_chain::serialization::ZcashDeserializeInto; + use zebra_network::zakura::testkit::{TraceCapture, TraceValue}; + use zebra_network::zakura::{ + commit_state_trace as cs_trace, BlockApplyResult, BlockSizeEstimate, BlockSyncAction, + BlockSyncBlockMeta, BlockSyncEvent, BlockSyncFrontiers, BlockSyncMisbehavior, + HeaderSyncCommitFailureKind, HeaderSyncFrontiers, Peer as ZakuraPeer, + Service as ZakuraService, Stream as ZakuraStream, ZakuraHeaderSyncDriverStartup, + COMMIT_STATE_TABLE, DEFAULT_HS_RANGE, + }; + use zebra_test::vectors::{BLOCK_MAINNET_1_BYTES, BLOCK_MAINNET_2_BYTES}; + + use super::zakura::{ + apply_block_sync_body, block_apply_class, block_sync_chain_tip_event, + block_sync_misbehavior_is_hard, block_sync_missing_body_window, + block_sync_needed_blocks_from_state, block_verify_error_is_duplicate, + body_sizes_for_served_header_range, coalesce_stale_needed_block_queries, + commit_block_sync_body, drive_block_sync_actions, drive_zakura_header_sync_actions, + header_range_commit_failure_kind, notify_block_sync_header_tip, query_block_sync_frontiers, + verified_block_tip_from_state, BlockApplyClass, ZakuraHeaderSyncDriverHandles, + ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL, ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, + }; + + fn mainnet_block(bytes: &[u8]) -> Arc { + Arc::new(bytes.zcash_deserialize_into().expect("block vector parses")) + } + + #[derive(Debug)] + struct NoopZakuraService; + + impl ZakuraService for NoopZakuraService { + fn name(&self) -> &'static str { + "noop" + } + + fn streams(&self) -> &[ZakuraStream] { + &[] + } + + fn add_peer(&self, _peer: ZakuraPeer) {} + + fn remove_peer(&self, _peer: &zebra_network::zakura::ZakuraPeerId) {} + } + + fn block_sync_startup_for_test() -> zebra_network::zakura::BlockSyncStartup { + let (tip_tx, tip_rx) = + tokio::sync::watch::channel((block::Height(0), block::Hash([0; 32]))); + drop(tip_tx); + zebra_network::zakura::BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(0), block::Hash([0; 32])), + tip_rx, + zebra_network::zakura::ZakuraBlockSyncConfig::default(), + ) + } + + #[test] + fn zakura_block_sync_replaces_chain_sync_when_v2_p2p_is_enabled() { + let mut config = zebra_network::Config::default(); + + assert!(use_zakura_block_sync(&config)); + + config.v2_p2p = false; + assert!(!use_zakura_block_sync(&config)); + } + + #[test] + fn missing_genesis_anchor_is_local_header_sync_commit_failure() { + let error = zebra_state::CommitHeaderRangeError::MissingGenesisAnchor { + anchor: block::Hash([0; 32]), + }; + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + + #[test] + fn served_header_body_size_hints_align_with_served_heights() { + let start = block::Height(10); + let header_heights = [ + block::Height(10), + block::Height(11), + block::Height(12), + block::Height(13), + ]; + let body_size_hints = [ + (block::Height(10), Some(100)), + (block::Height(11), None), + (block::Height(12), Some(300)), + (block::Height(13), Some(400)), + ]; + + assert_eq!( + body_sizes_for_served_header_range(start, header_heights, &body_size_hints), + vec![100, 0, 300, 400], + ); + + assert_eq!( + body_sizes_for_served_header_range(start, header_heights, &[]), + vec![0, 0, 0, 0], + ); + } #[test] fn block_verify_error_duplicate_classifier_detects_router_and_block_errors() { @@ -2360,4 +2071,2027 @@ mod zakura_header_sync_driver_tests { }; assert!(!block_verify_error_is_duplicate(&invalid_block_error)); } + + #[test] + fn block_sync_missing_body_window_stays_inside_reorg_bound() { + assert_eq!( + block_sync_missing_body_window(block::Height(10), block::Height(10)), + None + ); + assert_eq!( + block_sync_missing_body_window(block::Height(10), block::Height(12)), + Some((block::Height(11), 2)) + ); + assert_eq!( + block_sync_missing_body_window( + block::Height(10), + block::Height(10 + zebra_state::MAX_BLOCK_REORG_HEIGHT + 100) + ), + Some((block::Height(11), zebra_state::MAX_BLOCK_REORG_HEIGHT)) + ); + assert_eq!( + block_sync_missing_body_window(block::Height(u32::MAX - 1), block::Height(u32::MAX)), + Some((block::Height(u32::MAX), 1)) + ); + assert_eq!( + block_sync_missing_body_window(block::Height(u32::MAX), block::Height(u32::MAX)), + None + ); + } + + #[test] + fn block_sync_needed_blocks_align_missing_hashes_and_size_hints() { + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let headers = vec![ + (block::Height(1), block1.hash(), block1.header.clone()), + (block::Height(2), block2.hash(), block2.header.clone()), + ]; + let hints = vec![(block::Height(1), Some(0)), (block::Height(2), Some(42))]; + + let needed = block_sync_needed_blocks_from_state( + vec![block::Height(1), block::Height(2), block::Height(3)], + headers, + hints, + ); + + assert_eq!( + needed, + vec![ + BlockSyncBlockMeta { + height: block::Height(1), + hash: block1.hash(), + size: BlockSizeEstimate::Unknown, + }, + BlockSyncBlockMeta { + height: block::Height(2), + hash: block2.hash(), + size: BlockSizeEstimate::Advertised(42), + }, + ] + ); + } + + #[test] + fn block_sync_misbehavior_classifier_keeps_soft_reasons_soft() { + assert!(!block_sync_misbehavior_is_hard( + BlockSyncMisbehavior::SizeMismatch + )); + assert!(!block_sync_misbehavior_is_hard( + BlockSyncMisbehavior::RangeUnavailable + )); + assert!(!block_sync_misbehavior_is_hard( + BlockSyncMisbehavior::GetBlocksSpam + )); + assert!(block_sync_misbehavior_is_hard( + BlockSyncMisbehavior::InvalidBlock + )); + } + + #[test] + fn block_sync_chain_tip_action_mapping_preserves_reset_vs_grow() { + let frontiers = BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(1), + verified_block_hash: block::Hash([1; 32]), + }; + let tip_block = zebra_state::ChainTipBlock { + hash: block::Hash([1; 32]), + height: block::Height(1), + time: chrono::Utc::now(), + transactions: Vec::new(), + transaction_hashes: Arc::<[zebra_chain::transaction::Hash]>::from([]), + previous_block_hash: block::Hash([0; 32]), + }; + + assert!(matches!( + block_sync_chain_tip_event( + &zebra_state::TipAction::Grow { + block: tip_block.clone() + }, + frontiers + ), + BlockSyncEvent::ChainTipGrow(mapped) if mapped == frontiers + )); + assert!(matches!( + block_sync_chain_tip_event( + &zebra_state::TipAction::Reset { + height: block::Height(1), + hash: block::Hash([1; 32]), + }, + frontiers + ), + BlockSyncEvent::ChainTipReset(mapped) if mapped == frontiers + )); + } + + #[test] + fn verified_block_tip_from_state_prefers_highest_frontier_with_matching_hash() { + let empty = (block::Height(0), block::Hash([0; 32])); + + assert_eq!( + verified_block_tip_from_state( + Some((block::Height(2800), block::Hash([28; 32]))), + Some((block::Height(2561), block::Hash([25; 32]))), + empty, + ), + (block::Height(2800), block::Hash([28; 32])) + ); + + assert_eq!( + verified_block_tip_from_state( + Some((block::Height(2400), block::Hash([24; 32]))), + Some((block::Height(2801), block::Hash([29; 32]))), + empty, + ), + (block::Height(2801), block::Hash([29; 32])) + ); + + assert_eq!( + verified_block_tip_from_state(None, None, empty), + (block::Height(0), block::Hash([0; 32])) + ); + + assert_eq!( + verified_block_tip_from_state( + None, + Some((block::Height(3), block::Hash([3; 32]))), + empty + ), + (block::Height(3), block::Hash([3; 32])) + ); + } + + #[test] + fn block_apply_class_checkpoint_boundary_is_inclusive() { + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES); + + assert_eq!( + block_apply_class(&block1, block::Height(1)), + BlockApplyClass::Checkpoint + ); + assert_eq!( + block_apply_class(&block2, block::Height(1)), + BlockApplyClass::Full + ); + } + + #[test] + fn zakura_checkpoint_static_limits_cover_checkpoint_gap() { + const { + assert!( + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT + >= zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP + ); + } + assert!( + usize::try_from(DEFAULT_HS_RANGE) + .expect("DEFAULT_HS_RANGE fits usize on supported targets") + >= zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP + ); + assert!( + usize::try_from(zebra_state::MAX_BLOCK_REORG_HEIGHT) + .expect("MAX_BLOCK_REORG_HEIGHT fits usize on supported targets") + >= zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP + ); + } + + #[tokio::test] + async fn header_tip_notification_drives_block_sync_needed_query() { + let startup = block_sync_startup_for_test(); + let (block_sync, mut reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let header_hash = block::Hash([3; 32]); + + notify_block_sync_header_tip( + Some(&block_sync), + block::Height(3), + header_hash, + &zebra_network::zakura::ZakuraTrace::noop(), + ) + .await; + + let action = tokio::time::timeout(Duration::from_secs(5), reactor_actions.recv()) + .await + .expect("reactor emits a needed-block query after a header tip") + .expect("reactor action channel remains open"); + + assert!(matches!( + action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(3), + } + )); + + reactor_task.abort(); + } + + #[tokio::test] + async fn header_sync_driver_header_advanced_updates_exchange_header_only() { + let network = zebra_chain::parameters::Network::Mainnet; + let genesis_hash = network.genesis_hash(); + let mut config = zebra_network::Config { + network: network.clone(), + ..zebra_network::Config::default() + }; + config.zakura.listen_addr = None; + let endpoint = zebra_network::zakura::spawn_zakura_endpoint_with_header_sync_driver( + &config, + |_supervisor, _trace| Arc::new(NoopZakuraService) as Arc, + Some(ZakuraHeaderSyncDriverStartup { + frontiers: HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: genesis_hash, + }, + best_header_tip: Some((block::Height(0), genesis_hash)), + verified_block_tip_hash: genesis_hash, + }), + ) + .await + .expect("Zakura endpoint starts") + .expect("v2_p2p starts an endpoint"); + + let initial = endpoint + .current_sync_frontier() + .expect("driver startup initializes exchange"); + assert_eq!(initial.frontier.verified_body.height, block::Height(0)); + assert_eq!(initial.frontier.best_header.height, block::Height(0)); + + let (action_tx, action_rx) = mpsc::channel(4); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let handles = ZakuraHeaderSyncDriverHandles { + endpoint: endpoint.clone(), + header_sync: endpoint + .header_sync() + .expect("driver startup starts header sync"), + }; + let state = service_fn(|request: zebra_state::Request| async move { + panic!("unexpected state request from HeaderAdvanced: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_state::BoxError>(zebra_state::Response::Committed(block::Hash([0; 32]))) + }); + let read_state = service_fn(|request: zebra_state::ReadRequest| async move { + panic!("unexpected read request from HeaderAdvanced: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_state::BoxError>(zebra_state::ReadResponse::Tip(None)) + }); + let verifier = service_fn(|request: zebra_consensus::Request| async move { + panic!("unexpected verifier request from HeaderAdvanced: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_consensus::BoxError>(block::Hash([0; 32])) + }); + let driver = tokio::spawn(drive_zakura_header_sync_actions( + action_rx, + handles, + state, + read_state, + verifier, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + let advanced_hash = block::Hash([5; 32]); + action_tx + .send(zebra_network::zakura::HeaderSyncAction::HeaderAdvanced { + height: block::Height(5), + hash: advanced_hash, + }) + .await + .expect("driver action channel stays open"); + + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let update = endpoint + .current_sync_frontier() + .expect("exchange remains available"); + if update.frontier.best_header.height == block::Height(5) { + assert_eq!( + update.change, + zebra_network::zakura::FrontierChange::HeaderAdvanced + ); + assert_eq!(update.frontier.best_header.hash, advanced_hash); + assert_eq!( + update.frontier.verified_body, + initial.frontier.verified_body + ); + assert_eq!(update.frontier.finalized, initial.frontier.finalized); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("HeaderAdvanced publishes to the exchange"); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + endpoint.shutdown().await; + } + + #[tokio::test] + async fn block_sync_driver_coalesces_stale_needed_queries() { + let (action_tx, mut action_rx) = mpsc::channel(8); + action_tx + .send(BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(1), + }) + .await + .expect("first query queues"); + action_tx + .send(BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(2), + }) + .await + .expect("stale query queues"); + let deferred_peer = + zebra_network::zakura::ZakuraPeerId::new(vec![7; 32]).expect("test peer id is valid"); + action_tx + .send(BlockSyncAction::Misbehavior { + peer: deferred_peer.clone(), + reason: BlockSyncMisbehavior::StatusSpam, + }) + .await + .expect("non-query action queues"); + action_tx + .send(BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(8), + }) + .await + .expect("latest query queues"); + + let first = action_rx.recv().await.expect("first action remains queued"); + let mut deferred_actions = VecDeque::new(); + let action = + coalesce_stale_needed_block_queries(first, &mut action_rx, &mut deferred_actions); + + assert!(matches!( + action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(8), + } + )); + assert!(matches!( + deferred_actions.pop_front(), + Some(BlockSyncAction::Misbehavior { + peer, + reason: BlockSyncMisbehavior::StatusSpam, + }) if peer == deferred_peer + )); + assert!(deferred_actions.is_empty()); + assert!(action_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn block_sync_driver_answers_needed_block_queries_from_state() { + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let (action_tx, action_rx) = mpsc::channel(8); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + + let expected_headers = Arc::new(vec![ + (block::Height(1), block1.hash(), block1.header.clone()), + (block::Height(2), block2.hash(), block2.header.clone()), + ]); + let expected_hints = Arc::new(vec![ + (block::Height(1), Some(0)), + (block::Height(2), Some(42)), + ]); + let read_requests = Arc::new(Mutex::new(Vec::new())); + let read_requests_for_service = read_requests.clone(); + let read_headers = expected_headers.clone(); + let read_hints = expected_hints.clone(); + let block1_hash = block1.hash(); + let read_state = service_fn(move |request: zebra_state::ReadRequest| { + let read_requests = read_requests_for_service.clone(); + let read_headers = read_headers.clone(); + let read_hints = read_hints.clone(); + async move { + read_requests + .lock() + .expect("test read request log is not poisoned") + .push(request.clone()); + match request { + zebra_state::ReadRequest::MissingBlockBodies { from, limit } => { + assert_eq!(from, block::Height(1)); + assert_eq!(limit, 2); + Ok(zebra_state::ReadResponse::MissingBlockBodies(vec![ + block::Height(1), + block::Height(2), + ])) + } + zebra_state::ReadRequest::HeadersByHeightRange { start, count } => { + assert_eq!(start, block::Height(1)); + assert_eq!(count, 2); + Ok(zebra_state::ReadResponse::Headers((*read_headers).clone())) + } + zebra_state::ReadRequest::BlockSizeHints { from, count } => { + assert_eq!(from, block::Height(1)); + assert_eq!(count, 2); + Ok(zebra_state::ReadResponse::BlockSizeHints( + (*read_hints).clone(), + )) + } + zebra_state::ReadRequest::FinalizedTip => { + Ok(zebra_state::ReadResponse::FinalizedTip(None)) + } + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some(( + block::Height(1), + block1_hash, + )))), + request => panic!("unexpected read request: {request:?}"), + } + } + }); + + let (commit_tx, mut commit_rx) = mpsc::channel(1); + let verifier = service_fn(move |request: zebra_consensus::Request| { + let commit_tx = commit_tx.clone(); + async move { + match request { + zebra_consensus::Request::Commit(block) => { + let hash = block.hash(); + commit_tx + .send(hash) + .await + .expect("test commit receiver stays open"); + Ok::<_, zebra_consensus::BoxError>(hash) + } + request => panic!("unexpected consensus request: {request:?}"), + } + } + }); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + zebra_chain::chain_tip::NoChainTip, + read_state, + verifier, + block::Height::MAX, + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + action_tx + .send(BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(2), + }) + .await + .expect("driver action channel stays open"); + + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if read_requests + .lock() + .expect("test read request log is not poisoned") + .len() + >= 3 + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("driver answers QueryNeededBlocks through state reads"); + + assert_eq!( + read_requests + .lock() + .expect("test read request log is not poisoned") + .as_slice(), + &[ + zebra_state::ReadRequest::MissingBlockBodies { + from: block::Height(1), + limit: 2, + }, + zebra_state::ReadRequest::HeadersByHeightRange { + start: block::Height(1), + count: 2, + }, + zebra_state::ReadRequest::BlockSizeHints { + from: block::Height(1), + count: 2, + }, + ] + ); + + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 1, + block: block1.clone(), + }) + .await + .expect("driver action channel stays open"); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), commit_rx.recv()) + .await + .expect("commit arrives after query"), + Some(block1.hash()) + ); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + #[tokio::test] + async fn block_sync_driver_commits_parent_first_and_ignores_outbound_actions() { + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let (action_tx, action_rx) = mpsc::channel(8); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let (commit_tx, mut commit_rx) = mpsc::channel(8); + let verifier = service_fn(move |request: zebra_consensus::Request| { + let commit_tx = commit_tx.clone(); + async move { + match request { + zebra_consensus::Request::Commit(block) => { + let hash = block.hash(); + commit_tx + .send(hash) + .await + .expect("test commit receiver stays open"); + Ok::<_, zebra_consensus::BoxError>(hash) + } + request => panic!("unexpected consensus request: {request:?}"), + } + } + }); + let block2_hash = block2.hash(); + let read_state = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::FinalizedTip => Ok::<_, zebra_state::BoxError>( + zebra_state::ReadResponse::FinalizedTip(Some((block::Height(2), block2_hash))), + ), + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some(( + block::Height(2), + block2_hash, + )))), + request => panic!("unexpected read request: {request:?}"), + } + }); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + zebra_chain::chain_tip::NoChainTip, + read_state, + verifier, + block::Height::MAX, + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + let peer = + zebra_network::zakura::ZakuraPeerId::new(vec![8; 32]).expect("test peer id is valid"); + action_tx + .send(BlockSyncAction::SendMessage { + peer, + msg: zebra_network::zakura::BlockSyncMessage::Status( + zebra_network::zakura::BlockSyncStatus::default(), + ), + }) + .await + .expect("driver action channel stays open"); + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 1, + block: block1.clone(), + }) + .await + .expect("driver action channel stays open"); + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 2, + block: block2.clone(), + }) + .await + .expect("driver action channel stays open"); + + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), commit_rx.recv()) + .await + .expect("first commit arrives"), + Some(block1.hash()) + ); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), commit_rx.recv()) + .await + .expect("second commit arrives"), + Some(block2.hash()) + ); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + #[tokio::test] + async fn block_sync_driver_submits_checkpoint_blocks_without_waiting_for_first_commit() { + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let (action_tx, action_rx) = mpsc::channel(8); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let (commit_tx, mut commit_rx) = mpsc::channel(8); + let release_first = Arc::new(tokio::sync::Notify::new()); + let verifier = { + let release_first = release_first.clone(); + service_fn(move |request: zebra_consensus::Request| { + let commit_tx = commit_tx.clone(); + let release_first = release_first.clone(); + async move { + match request { + zebra_consensus::Request::Commit(block) => { + let height = block.coinbase_height().expect("test block has height"); + let hash = block.hash(); + commit_tx + .send(height) + .await + .expect("test commit receiver stays open"); + if height == block::Height(1) { + release_first.notified().await; + } + Ok::<_, zebra_consensus::BoxError>(hash) + } + request => panic!("unexpected consensus request: {request:?}"), + } + } + }) + }; + let block2_hash = block2.hash(); + let read_state = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::FinalizedTip => Ok::<_, zebra_state::BoxError>( + zebra_state::ReadResponse::FinalizedTip(Some((block::Height(2), block2_hash))), + ), + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some(( + block::Height(2), + block2_hash, + )))), + request => panic!("unexpected read request: {request:?}"), + } + }); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + zebra_chain::chain_tip::NoChainTip, + read_state, + verifier, + block::Height(2), + 2, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 1, + block: block1.clone(), + }) + .await + .expect("driver action channel stays open"); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), commit_rx.recv()) + .await + .expect("first checkpoint commit starts"), + Some(block::Height(1)) + ); + + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 2, + block: block2.clone(), + }) + .await + .expect("driver action channel stays open"); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), commit_rx.recv()) + .await + .expect("second checkpoint commit starts while first is pending"), + Some(block::Height(2)) + ); + + release_first.notify_waiters(); + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + /// A checkpoint-class commit must wait for the checkpoint verifier to + /// assemble a full contiguous range and must never be torn down by the + /// driver timeout, while a full (post-checkpoint) commit still times out. + /// + /// Regression test for the from-scratch mainnet stall: the checkpoint + /// verifier buffers every body below the first checkpoint (height 400) until + /// the whole range arrives, so a per-block commit timeout fired before + /// height 400 was reached and rolled the partial range back, freezing sync + /// at genesis. + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn checkpoint_commit_waits_past_driver_timeout_unlike_full_commit() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + + // A verifier that never answers a commit, mimicking the checkpoint + // verifier buffering a block until its range completes. + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(_) => { + std::future::pending::>().await + } + request => panic!("unexpected consensus request: {request:?}"), + } + }); + + // A full-class commit gives up after the driver timeout. (`verifier` is + // a capture-free `service_fn`, so it is `Copy` and reused below as-is.) + assert_eq!( + commit_block_sync_body(verifier, block.clone(), BlockApplyClass::Full).await, + BlockApplyResult::TimedOut, + "full commit should time out when the verifier never answers" + ); + + // A checkpoint-class commit keeps waiting: an outer timeout several times + // longer than the driver timeout must elapse with the commit still + // unresolved. If the driver timeout still applied to checkpoint commits, + // this would instead resolve to Ok(TimedOut) long before the outer + // timeout fired. + let waited = tokio::time::timeout( + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT * 4, + commit_block_sync_body(verifier, block, BlockApplyClass::Checkpoint), + ) + .await; + assert!( + waited.is_err(), + "checkpoint commit must keep waiting past the driver timeout, got {waited:?}" + ); + } + + #[tokio::test] + async fn checkpoint_commit_success_refreshes_block_sync_frontiers() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block_hash = block.hash(); + let (tip_tx, tip_rx) = + tokio::sync::watch::channel((block::Height(10), block::Hash([10; 32]))); + let _tip_tx = tip_tx; + let startup = zebra_network::zakura::BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(10), block::Hash([10; 32])), + tip_rx, + zebra_network::zakura::ZakuraBlockSyncConfig::default(), + ); + let (block_sync, mut reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + + let startup_action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv()) + .await + .expect("reactor emits startup action") + .expect("reactor action channel remains open"); + assert!( + matches!( + startup_action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(10), + } + ), + "test setup should start with an initial body query, got {startup_action:?}" + ); + + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(block) => { + Ok::<_, zebra_consensus::BoxError>(block.hash()) + } + request => panic!("unexpected consensus request: {request:?}"), + } + }); + let read_state = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::FinalizedTip => { + Ok::<_, zebra_state::BoxError>(zebra_state::ReadResponse::FinalizedTip(Some(( + block::Height(2), + block::Hash([2; 32]), + )))) + } + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some(( + block::Height(1), + block_hash, + )))), + request => panic!("unexpected read request: {request:?}"), + } + }); + + apply_block_sync_body( + verifier, + zebra_chain::chain_tip::NoChainTip, + None, + read_state, + block_sync.clone(), + 1, + block, + BlockApplyClass::Checkpoint, + zebra_network::zakura::ZakuraTrace::noop(), + ) + .await; + + let action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv()) + .await + .expect("reactor emits action after refreshed checkpoint frontier") + .expect("reactor action channel remains open"); + assert!( + matches!( + action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(2), + best_header_tip: block::Height(10), + } + ), + "checkpoint commit success must refresh state frontiers and query the next body window, got {action:?}" + ); + + reactor_task.abort(); + } + + #[tokio::test] + async fn block_sync_apply_emits_commit_state_trace_rows() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block_hash = block.hash(); + let block_height = block.coinbase_height().expect("test block has height"); + let mut capture = + TraceCapture::for_test("block_sync_apply_emits_commit_state_trace_rows").unwrap(); + let trace = zebra_network::zakura::ZakuraTrace::new(capture.tracer(), "01"); + + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(block) => { + Ok::<_, zebra_consensus::BoxError>(block.hash()) + } + request => panic!("unexpected consensus request: {request:?}"), + } + }); + let read_state = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::FinalizedTip => { + Ok::<_, zebra_state::BoxError>(zebra_state::ReadResponse::FinalizedTip(None)) + } + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some(( + block_height, + block_hash, + )))), + request => panic!("unexpected read request: {request:?}"), + } + }); + + apply_block_sync_body( + verifier, + zebra_chain::chain_tip::NoChainTip, + None, + read_state, + block_sync, + 77, + block, + BlockApplyClass::Full, + trace, + ) + .await; + + capture.flush().await; + let reader = capture.reader().unwrap(); + let commit_state = reader.table(COMMIT_STATE_TABLE.table()); + let hash_label = format!("{block_hash}"); + let common = [ + (cs_trace::APPLY_TOKEN, TraceValue::U64(77)), + (cs_trace::HEIGHT, TraceValue::U64(u64::from(block_height.0))), + (cs_trace::HASH, TraceValue::Str(&hash_label)), + ]; + commit_state.assert_row(cs_trace::COMMIT_START, &common); + commit_state.assert_row( + cs_trace::COMMIT_FINISH, + &[ + (cs_trace::APPLY_TOKEN, TraceValue::U64(77)), + (cs_trace::RESULT, TraceValue::Str("committed")), + ], + ); + commit_state.assert_row( + cs_trace::REACTOR_EVENT_SENT, + &[ + (cs_trace::APPLY_TOKEN, TraceValue::U64(77)), + (cs_trace::RESULT, TraceValue::Str("committed")), + ], + ); + + let _ = capture.finish().await.unwrap(); + reactor_task.abort(); + } + + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn block_sync_pending_checkpoint_apply_emits_stalled_trace_without_finishing() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block_hash = block.hash(); + let block_height = block.coinbase_height().expect("test block has height"); + let mut capture = TraceCapture::for_test( + "block_sync_pending_checkpoint_apply_emits_stalled_trace_without_finishing", + ) + .unwrap(); + let trace = zebra_network::zakura::ZakuraTrace::new(capture.tracer(), "01"); + + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(_block) => { + future::pending::>().await + } + request => panic!("unexpected consensus request: {request:?}"), + } + }); + let read_state = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::FinalizedTip => { + Ok::<_, zebra_state::BoxError>(zebra_state::ReadResponse::FinalizedTip(None)) + } + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some(( + block_height, + block_hash, + )))), + request => panic!("unexpected read request: {request:?}"), + } + }); + + let apply_task = tokio::spawn(apply_block_sync_body( + verifier, + zebra_chain::chain_tip::NoChainTip, + None, + read_state, + block_sync, + 88, + block, + BlockApplyClass::Checkpoint, + trace, + )); + + tokio::task::yield_now().await; + tokio::time::advance(ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT).await; + tokio::task::yield_now().await; + + capture.flush().await; + let reader = capture.reader().unwrap(); + let commit_state = reader.table(COMMIT_STATE_TABLE.table()); + commit_state.assert_row( + cs_trace::COMMIT_START, + &[(cs_trace::APPLY_TOKEN, TraceValue::U64(88))], + ); + commit_state.assert_row( + cs_trace::COMMIT_STALLED, + &[(cs_trace::APPLY_TOKEN, TraceValue::U64(88))], + ); + assert_eq!( + commit_state.count(cs_trace::COMMIT_FINISH), + 0, + "pending checkpoint verifier must not produce a finish row before it resolves" + ); + + apply_task.abort(); + let _ = capture.finish().await.unwrap(); + reactor_task.abort(); + } + + #[tokio::test] + async fn block_sync_pending_checkpoint_apply_does_not_block_control_plane_actions() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let (action_tx, action_rx) = mpsc::channel(8); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(_block) => { + future::pending::>().await + } + request => panic!("unexpected consensus request: {request:?}"), + } + }); + let (query_seen_tx, query_seen_rx) = oneshot::channel(); + let query_seen_tx = Arc::new(Mutex::new(Some(query_seen_tx))); + let read_state = service_fn(move |request: zebra_state::ReadRequest| { + let query_seen_tx = query_seen_tx.clone(); + async move { + match request { + zebra_state::ReadRequest::MissingBlockBodies { from, limit } => { + assert_eq!(from, block::Height(1)); + assert_eq!(limit, 1); + if let Some(query_seen_tx) = query_seen_tx + .lock() + .expect("query seen sender mutex is not poisoned") + .take() + { + let _ = query_seen_tx.send(()); + } + Ok::<_, zebra_state::BoxError>( + zebra_state::ReadResponse::MissingBlockBodies(Vec::new()), + ) + } + request => panic!("unexpected read request: {request:?}"), + } + } + }); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + zebra_chain::chain_tip::NoChainTip, + read_state, + verifier, + block::Height::MAX, + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + action_tx + .send(BlockSyncAction::SubmitBlock { token: 1, block }) + .await + .expect("driver action channel stays open"); + action_tx + .send(BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(1), + }) + .await + .expect("driver action channel stays open"); + + tokio::time::timeout(Duration::from_secs(1), query_seen_rx) + .await + .expect("driver processes unrelated query while checkpoint apply is pending") + .expect("read service reports query"); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + #[tokio::test] + async fn block_sync_checkpoint_apply_limit_is_capped_to_one_checkpoint_gap() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let (action_tx, action_rx) = mpsc::channel(zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP + 8); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let commit_count = Arc::new(AtomicUsize::new(0)); + let verifier_count = commit_count.clone(); + let verifier = service_fn(move |request: zebra_consensus::Request| { + let verifier_count = verifier_count.clone(); + async move { + match request { + zebra_consensus::Request::Commit(_block) => { + verifier_count.fetch_add(1, Ordering::SeqCst); + future::pending::>().await + } + request => panic!("unexpected consensus request: {request:?}"), + } + } + }); + let read_state = service_fn(move |request: zebra_state::ReadRequest| async move { + panic!("unexpected read request while checkpoint applies are pending: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_state::BoxError>(zebra_state::ReadResponse::Tip(None)) + }); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + zebra_chain::chain_tip::NoChainTip, + read_state, + verifier, + block::Height::MAX, + zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP * 2, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + for token in 0..=zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP { + action_tx + .send(BlockSyncAction::SubmitBlock { + token: u64::try_from(token).expect("test token fits in u64"), + block: block.clone(), + }) + .await + .expect("driver action channel stays open"); + } + + tokio::time::timeout(Duration::from_secs(1), async { + while commit_count.load(Ordering::SeqCst) < zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP { + tokio::task::yield_now().await; + } + }) + .await + .expect("driver starts exactly one checkpoint gap of applies"); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + commit_count.load(Ordering::SeqCst), + zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP, + "driver must not submit a second checkpoint range before the first range completes" + ); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn delayed_checkpoint_frontier_refresh_sends_committed_height() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block_hash = block.hash(); + let (_tip_tx, tip_rx) = + tokio::sync::watch::channel((block::Height(10), block::Hash([10; 32]))); + let startup = zebra_network::zakura::BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(10), block::Hash([10; 32])), + tip_rx, + zebra_network::zakura::ZakuraBlockSyncConfig::default(), + ); + let (block_sync, mut reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + + let startup_action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv()) + .await + .expect("reactor emits startup action") + .expect("reactor action channel remains open"); + assert!( + matches!( + startup_action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(10), + } + ), + "test setup should start with an initial body query, got {startup_action:?}" + ); + + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(block) => { + Ok::<_, zebra_consensus::BoxError>(block.hash()) + } + request => panic!("unexpected consensus request: {request:?}"), + } + }); + let (mut tip_sender, latest_chain_tip, _tip_change) = + zebra_state::ChainTipSender::new(None, &zebra_chain::parameters::Network::Mainnet); + let read_count = Arc::new(AtomicUsize::new(0)); + let read_state = service_fn(move |request: zebra_state::ReadRequest| { + let read_index = read_count.fetch_add(1, Ordering::SeqCst); + async move { + let visible_tip = if read_index < 2 { + None + } else { + Some((block::Height(1), block_hash)) + }; + + match request { + zebra_state::ReadRequest::FinalizedTip => Ok::<_, zebra_state::BoxError>( + zebra_state::ReadResponse::FinalizedTip(visible_tip), + ), + zebra_state::ReadRequest::Tip => { + Ok(zebra_state::ReadResponse::Tip(visible_tip)) + } + request => panic!("unexpected read request: {request:?}"), + } + } + }); + + let (action_tx, action_rx) = mpsc::channel(8); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync.clone(), + latest_chain_tip, + read_state, + verifier, + block::Height::MAX, + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + action_tx + .send(BlockSyncAction::SubmitBlock { token: 1, block }) + .await + .expect("driver action channel stays open"); + tokio::task::yield_now().await; + + assert!( + tokio::time::timeout(Duration::from_millis(1), reactor_actions.recv()) + .await + .is_err(), + "the immediate refresh should not advance before state exposes the checkpoint" + ); + + tip_sender.set_finalized_tip(zebra_state::ChainTipBlock { + hash: block_hash, + height: block::Height(1), + time: chrono::Utc::now(), + transactions: Vec::new(), + transaction_hashes: Arc::from([]), + previous_block_hash: block::Hash([0; 32]), + }); + tokio::time::advance(ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL).await; + + let action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv()) + .await + .expect("reactor emits action after delayed checkpoint frontier") + .expect("reactor action channel remains open"); + assert!( + matches!( + action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(1), + best_header_tip: block::Height(10), + } + ), + "delayed checkpoint refresh must send the committed height once state catches up, got {action:?}" + ); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn delayed_checkpoint_frontier_refresh_is_coalesced_across_commits() { + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let block2 = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let block2_hash = block2.hash(); + let (_tip_tx, tip_rx) = + tokio::sync::watch::channel((block::Height(10), block::Hash([10; 32]))); + let startup = zebra_network::zakura::BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: block::Hash([0; 32]), + }, + (block::Height(10), block::Hash([10; 32])), + tip_rx, + zebra_network::zakura::ZakuraBlockSyncConfig::default(), + ); + let (block_sync, mut reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let _startup_action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv()) + .await + .expect("reactor emits startup action") + .expect("reactor action channel remains open"); + + let mut capture = TraceCapture::for_test( + "delayed_checkpoint_frontier_refresh_is_coalesced_across_commits", + ) + .unwrap(); + let trace = zebra_network::zakura::ZakuraTrace::new(capture.tracer(), "01"); + let visible = Arc::new(AtomicBool::new(false)); + let visible_for_reads = visible.clone(); + let read_count = Arc::new(AtomicUsize::new(0)); + let read_count_for_service = read_count.clone(); + let read_state = service_fn(move |request: zebra_state::ReadRequest| { + let visible = visible_for_reads.load(Ordering::SeqCst); + let read_count = read_count_for_service.clone(); + async move { + read_count.fetch_add(1, Ordering::SeqCst); + let visible_tip = visible.then_some((block::Height(2), block2_hash)); + match request { + zebra_state::ReadRequest::FinalizedTip => Ok::<_, zebra_state::BoxError>( + zebra_state::ReadResponse::FinalizedTip(visible_tip), + ), + zebra_state::ReadRequest::Tip => { + Ok(zebra_state::ReadResponse::Tip(visible_tip)) + } + request => panic!("unexpected read request: {request:?}"), + } + } + }); + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(block) => { + Ok::<_, zebra_consensus::BoxError>(block.hash()) + } + request => panic!("unexpected consensus request: {request:?}"), + } + }); + let (action_tx, action_rx) = mpsc::channel(8); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync.clone(), + zebra_chain::chain_tip::NoChainTip, + read_state, + verifier, + block::Height::MAX, + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + trace, + async move { + let _ = shutdown_rx.await; + }, + )); + + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 1, + block: block1, + }) + .await + .expect("driver action channel stays open"); + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 2, + block: block2, + }) + .await + .expect("driver action channel stays open"); + + tokio::time::timeout(Duration::from_secs(1), async { + while read_count.load(Ordering::SeqCst) < 4 { + tokio::task::yield_now().await; + } + }) + .await + .expect("both immediate post-commit frontier reads complete"); + + visible.store(true, Ordering::SeqCst); + tokio::time::advance(ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL).await; + + let action = tokio::time::timeout(Duration::from_secs(1), reactor_actions.recv()) + .await + .expect("reactor emits action after coalesced delayed checkpoint frontier") + .expect("reactor action channel remains open"); + assert!( + matches!( + action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(2), + best_header_tip: block::Height(10), + } + ), + "coalesced delayed checkpoint refresh must send the committed height once state catches up, got {action:?}" + ); + + capture.flush().await; + let reader = capture.reader().unwrap(); + let commit_state = reader.table(COMMIT_STATE_TABLE.table()); + assert_eq!( + commit_state.count(cs_trace::CHECKPOINT_REFRESH_ATTEMPT), + 1, + "multiple committed checkpoint bodies should share one delayed frontier refresh attempt" + ); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + let _ = capture.finish().await.unwrap(); + reactor_task.abort(); + } + + #[tokio::test] + async fn block_sync_driver_treats_duplicate_commit_as_idempotent_and_keeps_draining() { + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let (action_tx, action_rx) = mpsc::channel(8); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let attempts = Arc::new(AtomicUsize::new(0)); + let verifier_attempts = attempts.clone(); + let (commit_tx, mut commit_rx) = mpsc::channel(8); + let verifier = service_fn(move |request: zebra_consensus::Request| { + let attempts = verifier_attempts.clone(); + let commit_tx = commit_tx.clone(); + async move { + match request { + zebra_consensus::Request::Commit(block) => { + let hash = block.hash(); + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(zebra_consensus::RouterError::Block { + source: Box::new(zebra_consensus::VerifyBlockError::Block { + source: zebra_consensus::BlockError::AlreadyInChain( + hash, + zebra_state::KnownBlock::BestChain, + ), + }), + }); + } + commit_tx + .send(hash) + .await + .expect("test commit receiver stays open"); + Ok(hash) + } + request => panic!("unexpected consensus request: {request:?}"), + } + } + }); + let read_requests = Arc::new(Mutex::new(Vec::new())); + let read_requests_for_service = read_requests.clone(); + let block_hash = block.hash(); + let read_state = service_fn(move |request: zebra_state::ReadRequest| { + let read_requests = read_requests_for_service.clone(); + async move { + read_requests + .lock() + .expect("test read request log is not poisoned") + .push(request.clone()); + match request { + zebra_state::ReadRequest::FinalizedTip => { + Ok(zebra_state::ReadResponse::FinalizedTip(None)) + } + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some(( + block::Height(1), + block_hash, + )))), + request => panic!("unexpected read request: {request:?}"), + } + } + }); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + zebra_chain::chain_tip::NoChainTip, + read_state, + verifier, + block::Height::MAX, + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 1, + block: block.clone(), + }) + .await + .expect("driver action channel stays open"); + action_tx + .send(BlockSyncAction::SubmitBlock { + token: 2, + block: block.clone(), + }) + .await + .expect("driver action channel stays open"); + + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), commit_rx.recv()) + .await + .expect("second commit arrives after duplicate"), + Some(block.hash()) + ); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + assert!( + read_requests + .lock() + .expect("test read request log is not poisoned") + .iter() + .any(|request| matches!(request, zebra_state::ReadRequest::Tip)), + "duplicate commit should refresh block-sync frontiers from state" + ); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + /// Drives the block-sync apply loop against the *real* checkpoint verifier and a *real* + /// ephemeral state, reproducing the checkpoint-range batch-commit that Regtest (genesis + /// checkpoint only) cannot exercise. + /// + /// A checkpoint is placed at height 10 so an 11-block range covers a full checkpoint gap + /// without 400 real blocks. The whole range is submitted except one mid-range body, which + /// the verifier holds the entire range for (it commits nothing until the range is + /// contiguous to the next checkpoint). Delivering the withheld body must let the whole + /// range commit — i.e. a transiently-missing body recovers instead of wedging the floor, + /// which is the production "drop-through" failure mode. + #[tokio::test] + async fn block_sync_driver_recovers_checkpoint_range_after_withheld_body() { + const CHECKPOINT_HEIGHT: u32 = 10; + const WITHHELD: u32 = 5; + + // Real, contiguous mainnet blocks 0..=10 (valid PoW, merkle roots, and parent + // linkage), so the real checkpoint verifier accepts them with no synthesis. + let chain: Vec<(block::Height, Arc)> = (0..=CHECKPOINT_HEIGHT) + .map(|height| { + let bytes: &[u8] = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .get(&height) + .copied() + .expect("a contiguous mainnet block vector exists for heights 0..=10"); + let block = mainnet_block(bytes); + assert_eq!( + block.coinbase_height(), + Some(block::Height(height)), + "mainnet block vector height matches its coinbase height", + ); + (block::Height(height), block) + }) + .collect(); + let genesis_hash = chain[0].1.hash(); + let checkpoint_hash = chain[CHECKPOINT_HEIGHT as usize].1.hash(); + + let network = zebra_chain::parameters::Network::Mainnet; + let (write_state, read_state, _latest_tip, _tip_change) = + zebra_state::init_test_services(&network).await; + + // A low checkpoint at height 10 turns the 11-block range into one checkpoint batch. + let checkpoint_verifier = zebra_consensus::CheckpointVerifier::from_list( + [ + (block::Height(0), genesis_hash), + (block::Height(CHECKPOINT_HEIGHT), checkpoint_hash), + ], + &network, + None, + write_state, + ) + .expect("a checkpoint list with genesis and one mid-chain checkpoint is valid"); + + // Adapt the checkpoint verifier (`Service>`) to the driver's + // `Service` bound. + let checkpoint_verifier = + tower::buffer::Buffer::new(BoxService::new(checkpoint_verifier), 16); + let verifier = service_fn(move |request: zebra_consensus::Request| { + let checkpoint_verifier = checkpoint_verifier.clone(); + async move { + match request { + zebra_consensus::Request::Commit(block) => { + checkpoint_verifier.oneshot(block).await + } + request => panic!("unexpected consensus request: {request:?}"), + } + } + }); + + let (action_tx, action_rx) = mpsc::channel(64); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + _latest_tip, + read_state.clone(), + verifier, + // Every block 0..=10 is at or below the checkpoint, so all are Checkpoint-class + // (indefinite-wait) commits — the path that wedges in production. + block::Height(CHECKPOINT_HEIGHT), + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + let finalized_tip = || { + let read_state = read_state.clone(); + async move { + match read_state + .oneshot(zebra_state::ReadRequest::FinalizedTip) + .await + .expect("finalized tip read succeeds") + { + zebra_state::ReadResponse::FinalizedTip(tip) => { + tip.map(|(height, _hash)| height) + } + response => panic!("unexpected FinalizedTip response: {response:?}"), + } + } + }; + + // Submit the whole checkpoint range except the withheld mid-range body. + for (height, block) in &chain { + if height.0 == WITHHELD { + continue; + } + action_tx + .send(BlockSyncAction::SubmitBlock { + token: u64::from(height.0), + block: block.clone(), + }) + .await + .expect("driver action channel stays open"); + } + + // While the body is missing the range cannot commit, and the driver must keep the + // checkpoint-class commits pending (not time them out): the tip never reaches the + // checkpoint. + tokio::time::sleep(Duration::from_millis(250)).await; + assert_ne!( + finalized_tip().await, + Some(block::Height(CHECKPOINT_HEIGHT)), + "checkpoint range must not commit while a mid-range body is missing", + ); + + // Deliver the withheld body; the verifier can now commit the whole range. + let (withheld_height, withheld_block) = chain + .iter() + .find(|(height, _)| height.0 == WITHHELD) + .expect("withheld block is part of the test chain"); + action_tx + .send(BlockSyncAction::SubmitBlock { + token: u64::from(withheld_height.0), + block: withheld_block.clone(), + }) + .await + .expect("driver action channel stays open"); + + // Recovery: the entire range commits, so the finalized tip reaches the checkpoint. + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if finalized_tip().await == Some(block::Height(CHECKPOINT_HEIGHT)) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("delivering the withheld body must let the checkpoint range commit to the tip"); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + /// Drives a from-scratch body sync across two synthetic checkpoint boundaries using + /// real contiguous mainnet blocks, real ephemeral state, the real checkpoint verifier, + /// the block-sync reactor, and the block-sync driver apply loop. + #[tokio::test] + async fn block_sync_driver_finalizes_across_two_checkpoint_boundaries() { + const FIRST_CHECKPOINT_HEIGHT: u32 = 5; + const SECOND_CHECKPOINT_HEIGHT: u32 = 10; + + let chain: Vec<(block::Height, Arc)> = (0..=SECOND_CHECKPOINT_HEIGHT) + .map(|height| { + let bytes: &[u8] = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .get(&height) + .copied() + .expect("a contiguous mainnet block vector exists for heights 0..=10"); + let block = mainnet_block(bytes); + assert_eq!( + block.coinbase_height(), + Some(block::Height(height)), + "mainnet block vector height matches its coinbase height", + ); + (block::Height(height), block) + }) + .collect(); + let genesis_hash = chain[0].1.hash(); + let first_checkpoint_hash = chain[FIRST_CHECKPOINT_HEIGHT as usize].1.hash(); + let second_checkpoint_height = block::Height(SECOND_CHECKPOINT_HEIGHT); + let second_checkpoint_hash = chain[SECOND_CHECKPOINT_HEIGHT as usize].1.hash(); + + let network = zebra_chain::parameters::Network::Mainnet; + let (write_state, read_state, latest_tip, _tip_change) = + zebra_state::init_test_services(&network).await; + + let checkpoint_verifier = zebra_consensus::CheckpointVerifier::from_list( + [ + (block::Height(0), genesis_hash), + ( + block::Height(FIRST_CHECKPOINT_HEIGHT), + first_checkpoint_hash, + ), + (second_checkpoint_height, second_checkpoint_hash), + ], + &network, + None, + write_state, + ) + .expect("a checkpoint list with two low checkpoint boundaries is valid"); + let checkpoint_verifier = + tower::buffer::Buffer::new(BoxService::new(checkpoint_verifier), 32); + let verifier = service_fn(move |request: zebra_consensus::Request| { + let checkpoint_verifier = checkpoint_verifier.clone(); + async move { + match request { + zebra_consensus::Request::Commit(block) => { + checkpoint_verifier.oneshot(block).await + } + request => panic!("unexpected consensus request: {request:?}"), + } + } + }); + + let (action_tx, action_rx) = mpsc::channel(64); + let startup = block_sync_startup_for_test(); + let (block_sync, _reactor_actions, reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let driver = tokio::spawn(drive_block_sync_actions( + action_rx, + zebra_network::zakura::ZakuraSupervisorHandle::new(1), + None, + block_sync, + latest_tip, + read_state.clone(), + verifier, + second_checkpoint_height, + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + sync::MIN_CONCURRENCY_LIMIT, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + for (height, block) in &chain { + action_tx + .send(BlockSyncAction::SubmitBlock { + token: u64::from(height.0), + block: block.clone(), + }) + .await + .expect("driver action channel stays open"); + } + + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let finalized_tip = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::FinalizedTip) + .await + .expect("finalized tip read succeeds") + { + zebra_state::ReadResponse::FinalizedTip(tip) => tip, + response => panic!("unexpected FinalizedTip response: {response:?}"), + }; + + if finalized_tip == Some((second_checkpoint_height, second_checkpoint_hash)) { + break; + } + + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("driver must finalize through both checkpoint boundaries"); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + reactor_task.abort(); + } + + #[tokio::test] + async fn block_sync_restart_reloads_checkpoint_frontier_after_missed_live_update() { + const CHECKPOINT_HEIGHT: u32 = 10; + + // Real, contiguous mainnet blocks 0..=10, matching the low-checkpoint + // setup in the withheld-body regression above. + let chain: Vec<(block::Height, Arc)> = (0..=CHECKPOINT_HEIGHT) + .map(|height| { + let bytes: &[u8] = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .get(&height) + .copied() + .expect("a contiguous mainnet block vector exists for heights 0..=10"); + let block = mainnet_block(bytes); + assert_eq!( + block.coinbase_height(), + Some(block::Height(height)), + "mainnet block vector height matches its coinbase height", + ); + (block::Height(height), block) + }) + .collect(); + let genesis_hash = chain[0].1.hash(); + let checkpoint_height = block::Height(CHECKPOINT_HEIGHT); + let checkpoint_hash = chain[CHECKPOINT_HEIGHT as usize].1.hash(); + let best_header_tip = (block::Height(20), block::Hash([20; 32])); + + let network = zebra_chain::parameters::Network::Mainnet; + let (write_state, read_state, _latest_tip, _tip_change) = + zebra_state::init_test_services(&network).await; + + // Start a live reactor from the stale genesis frontier, with headers + // already above the checkpoint. + let (_tip_tx, tip_rx) = tokio::sync::watch::channel(best_header_tip); + let startup = zebra_network::zakura::BlockSyncStartup::new( + BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: genesis_hash, + }, + best_header_tip, + tip_rx, + zebra_network::zakura::ZakuraBlockSyncConfig::default(), + ); + let (stale_block_sync, mut stale_actions, stale_reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(startup); + + let startup_action = tokio::time::timeout(Duration::from_secs(1), stale_actions.recv()) + .await + .expect("stale reactor emits startup action") + .expect("stale reactor action channel remains open"); + assert!( + matches!( + startup_action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(20), + } + ), + "stale reactor should start querying from genesis, got {startup_action:?}" + ); + + // Commit the low checkpoint range through the real checkpoint verifier, + // but intentionally do not notify the live block-sync reactor. + let checkpoint_verifier = zebra_consensus::CheckpointVerifier::from_list( + [ + (block::Height(0), genesis_hash), + (checkpoint_height, checkpoint_hash), + ], + &network, + None, + write_state, + ) + .expect("a checkpoint list with genesis and one mid-chain checkpoint is valid"); + let checkpoint_verifier = + tower::buffer::Buffer::new(BoxService::new(checkpoint_verifier), 16); + let mut commits = FuturesUnordered::new(); + for (_height, block) in chain { + let checkpoint_verifier = checkpoint_verifier.clone(); + commits.push(async move { checkpoint_verifier.oneshot(block).await }); + } + while let Some(result) = commits.next().await { + result.expect("checkpoint verifier commits the contiguous range"); + } + + let finalized_tip = || { + let read_state = read_state.clone(); + async move { + match read_state + .oneshot(zebra_state::ReadRequest::FinalizedTip) + .await + .expect("finalized tip read succeeds") + { + zebra_state::ReadResponse::FinalizedTip(tip) => tip, + response => panic!("unexpected FinalizedTip response: {response:?}"), + } + } + }; + + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if finalized_tip() + .await + .is_some_and(|(height, _hash)| height == checkpoint_height) + { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("checkpoint range must reach durable finalized state"); + + assert_eq!( + stale_block_sync.local_status().servable_high, + block::Height(0), + "without a live frontier event, the old reactor remains stale" + ); + notify_block_sync_header_tip( + Some(&stale_block_sync), + best_header_tip.0, + best_header_tip.1, + &zebra_network::zakura::ZakuraTrace::noop(), + ) + .await; + let stale_nudge = tokio::time::timeout(Duration::from_secs(1), stale_actions.recv()) + .await + .expect("stale reactor emits query after a header-tip nudge") + .expect("stale reactor action channel remains open"); + assert!( + matches!( + stale_nudge, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(0), + best_header_tip: block::Height(20), + } + ), + "without a live frontier event, the old reactor keeps querying from genesis, got {stale_nudge:?}" + ); + + stale_reactor_task.abort(); + + let restart_read_state = { + let read_state = read_state.clone(); + service_fn(move |request: zebra_state::ReadRequest| { + let read_state = read_state.clone(); + async move { + match request { + zebra_state::ReadRequest::FinalizedTip => read_state.oneshot(request).await, + zebra_state::ReadRequest::Tip => Ok(zebra_state::ReadResponse::Tip(Some( + (block::Height(0), genesis_hash), + ))), + request => panic!("unexpected restart read request: {request:?}"), + } + } + }) + }; + let restart_frontiers = + query_block_sync_frontiers(restart_read_state, zebra_chain::chain_tip::NoChainTip) + .await + .expect("restart reads block-sync frontiers from durable state"); + assert_eq!(restart_frontiers.finalized_height, checkpoint_height); + assert_eq!(restart_frontiers.verified_block_tip, checkpoint_height); + assert_eq!(restart_frontiers.verified_block_hash, checkpoint_hash); + + let (_restart_tip_tx, restart_tip_rx) = tokio::sync::watch::channel(best_header_tip); + let restart_startup = zebra_network::zakura::BlockSyncStartup::new( + restart_frontiers, + best_header_tip, + restart_tip_rx, + zebra_network::zakura::ZakuraBlockSyncConfig::default(), + ); + let (_fresh_block_sync, mut fresh_actions, fresh_reactor_task) = + zebra_network::zakura::spawn_block_sync_reactor(restart_startup); + let restart_action = tokio::time::timeout(Duration::from_secs(1), fresh_actions.recv()) + .await + .expect("fresh reactor emits startup action") + .expect("fresh reactor action channel remains open"); + assert!( + matches!( + restart_action, + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: block::Height(10), + best_header_tip: block::Height(20), + } + ), + "fresh reactor should query from the durable checkpoint frontier, got {restart_action:?}" + ); + + fresh_reactor_task.abort(); + } } diff --git a/zebrad/src/commands/start/zakura/block_sync_driver.rs b/zebrad/src/commands/start/zakura/block_sync_driver.rs new file mode 100644 index 00000000000..547fb72d1b9 --- /dev/null +++ b/zebrad/src/commands/start/zakura/block_sync_driver.rs @@ -0,0 +1,1211 @@ +use std::{ + collections::{HashMap, VecDeque}, + future::Future, + sync::Arc, + time::{Duration, Instant}, +}; + +use futures::{ + future::BoxFuture, + stream::{FuturesUnordered, StreamExt}, + FutureExt, +}; +use tokio::time::Instant as TokioInstant; +use tokio::{pin, select, sync::mpsc}; +use tower::{Service, ServiceExt}; +use tracing::{debug, warn}; + +use zebra_chain::{block, chain_tip::ChainTip}; +use zebra_network::zakura::{ + commit_state_trace as cs_trace, BlockApplyResult, BlockApplyToken, BlockSizeEstimate, + BlockSyncAction, BlockSyncBlockMeta, BlockSyncEvent, BlockSyncHandle, BlockSyncMessage, + BlockSyncMisbehavior, Frontier, FrontierChange, ZakuraEndpoint, ZakuraTrace, +}; + +use crate::components::sync; + +use super::{ + block_apply_result_label, block_verify_error_is_duplicate, emit_commit_state, insert_cs_bool, + insert_cs_frontiers, insert_cs_hash, insert_cs_height, insert_cs_peer, insert_cs_str, + insert_cs_u64, query_block_sync_frontiers, ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, +}; + +pub(crate) const ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL: Duration = + Duration::from_secs(5); +const ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_ATTEMPTS: usize = 24; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum BlockApplyClass { + Checkpoint, + Full, +} + +#[derive(Clone, Debug)] +struct PendingBlockApply { + token: BlockApplyToken, + class: BlockApplyClass, + block: Arc, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) struct BlockApplyCompletion { + class: BlockApplyClass, + checkpoint_refresh_floor: Option, +} + +#[derive(Clone, Debug, Default)] +struct CheckpointFrontierRefresh { + highest_sent: Option, + attempts_remaining: usize, + next_attempt_at: Option, +} + +impl CheckpointFrontierRefresh { + fn observe_checkpoint_commit(&mut self, highest_observed_at_apply: block::Height) { + self.highest_sent = Some( + self.highest_sent + .map(|height| height.max(highest_observed_at_apply)) + .unwrap_or(highest_observed_at_apply), + ); + self.attempts_remaining = ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_ATTEMPTS; + if self.next_attempt_at.is_none() { + self.next_attempt_at = + Some(TokioInstant::now() + ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL); + } + } + + fn next_attempt_at(&self) -> Option { + (self.attempts_remaining > 0) + .then_some(self.next_attempt_at) + .flatten() + } + + fn finish_attempt(&mut self, highest_sent: block::Height) { + self.highest_sent = Some(highest_sent); + self.attempts_remaining = self.attempts_remaining.saturating_sub(1); + self.next_attempt_at = (self.attempts_remaining > 0).then_some( + TokioInstant::now() + ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL, + ); + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn drive_block_sync_actions( + mut actions: mpsc::Receiver, + supervisor: zebra_network::zakura::ZakuraSupervisorHandle, + endpoint: Option, + block_sync: BlockSyncHandle, + latest_chain_tip: impl ChainTip + Clone + Send + Sync + 'static, + read_state: ReadState, + block_verifier: BlockVerifier, + max_checkpoint_height: block::Height, + checkpoint_apply_limit: usize, + full_apply_limit: usize, + trace: ZakuraTrace, + shutdown: impl Future + Send + 'static, +) where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, + BlockVerifier: + Service + Clone + Send + 'static, + BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, + BlockVerifier::Future: Send + 'static, +{ + pin!(shutdown); + const { + assert!( + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT <= zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP + ); + } + let checkpoint_apply_limit = checkpoint_apply_limit.clamp( + sync::MIN_CHECKPOINT_CONCURRENCY_LIMIT, + zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP, + ); + let full_apply_limit = full_apply_limit.max(sync::MIN_CONCURRENCY_LIMIT); + let mut pending_applies = VecDeque::new(); + let mut in_flight_applies: FuturesUnordered> = + FuturesUnordered::new(); + let mut checkpoint_in_flight = 0usize; + let mut full_in_flight = 0usize; + let mut deferred_actions = VecDeque::new(); + let mut checkpoint_frontier_refresh = CheckpointFrontierRefresh::default(); + + loop { + let action = if let Some(action) = deferred_actions.pop_front() { + action + } else { + select! { + _ = &mut shutdown => return, + completed = in_flight_applies.next(), if !in_flight_applies.is_empty() => { + let Some(completed) = completed else { + continue; + }; + match completed.class { + BlockApplyClass::Checkpoint => { + checkpoint_in_flight = checkpoint_in_flight.saturating_sub(1); + } + BlockApplyClass::Full => { + full_in_flight = full_in_flight.saturating_sub(1); + } + } + if let Some(highest_observed_at_apply) = completed.checkpoint_refresh_floor { + checkpoint_frontier_refresh + .observe_checkpoint_commit(highest_observed_at_apply); + } + drain_pending_block_applies( + &mut pending_applies, + &mut in_flight_applies, + &mut checkpoint_in_flight, + &mut full_in_flight, + checkpoint_apply_limit, + full_apply_limit, + latest_chain_tip.clone(), + endpoint.clone(), + read_state.clone(), + block_verifier.clone(), + block_sync.clone(), + trace.clone(), + ); + continue; + } + _ = async { + match checkpoint_frontier_refresh.next_attempt_at() { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } + }, if checkpoint_frontier_refresh.next_attempt_at().is_some() => { + refresh_block_sync_frontiers_for_checkpoint_window( + read_state.clone(), + latest_chain_tip.clone(), + endpoint.clone(), + Some(block_sync.clone()), + trace.clone(), + &mut checkpoint_frontier_refresh, + ).await; + continue; + } + action = actions.recv() => { + let Some(action) = action else { + return; + }; + action + } + } + }; + let action = + coalesce_stale_needed_block_queries(action, &mut actions, &mut deferred_actions); + + trace_block_driver_action(&trace, &action); + match action { + BlockSyncAction::SendMessage { .. } => {} + BlockSyncAction::Misbehavior { peer, reason } => { + if block_sync_misbehavior_is_hard(reason) { + debug!( + ?peer, + ?reason, + "disconnecting peer for Zakura block-sync violation" + ); + let _ = supervisor.disconnect_peer(&peer).await; + } else { + debug!( + ?peer, + ?reason, + "recorded soft Zakura block-sync peer violation" + ); + } + } + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_START, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "query_needed_blocks"); + insert_cs_height(row, cs_trace::VERIFIED_BLOCK_TIP, verified_block_tip); + insert_cs_height(row, cs_trace::BEST_HEADER_TIP, best_header_tip); + }, + ); + let started = Instant::now(); + match query_block_sync_needed_blocks( + read_state.clone(), + verified_block_tip, + best_header_tip, + ) + .await + { + Ok(blocks) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_SUCCESS, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "query_needed_blocks"); + insert_cs_u64(row, cs_trace::RANGE_COUNT, blocks.len() as u64); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + let _ = block_sync.send(BlockSyncEvent::NeededBlocks(blocks)).await; + emit_commit_state( + &trace, + cs_trace::REACTOR_EVENT_SENT, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "needed_blocks"); + }, + ); + } + Err(error) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_ERROR, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "query_needed_blocks"); + insert_cs_str(row, cs_trace::RESULT, "error"); + insert_cs_str(row, cs_trace::REASON, &format!("{error}")); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + warn!( + ?verified_block_tip, + ?best_header_tip, + ?error, + "failed to query Zakura block-sync needed blocks" + ); + } + } + } + BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_START, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "query_blocks_by_height_range"); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + }, + ); + let started = Instant::now(); + match tokio::time::timeout( + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, + read_state + .clone() + .oneshot(zebra_state::ReadRequest::BlocksByHeightRange { start, count }), + ) + .await + { + Ok(Ok(zebra_state::ReadResponse::Blocks(blocks))) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_SUCCESS, + "block_sync_driver", + |row| { + insert_cs_str( + row, + cs_trace::ACTION, + "query_blocks_by_height_range", + ); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, blocks.len() as u64); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + emit_commit_state( + &trace, + cs_trace::REACTOR_EVENT_SENT, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "block_range_response_ready"); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + }, + ); + let _ = block_sync + .send(BlockSyncEvent::BlockRangeResponseReady { + peer, + start_height: start, + requested_count: count, + blocks, + }) + .await; + } + Ok(Ok(response)) => { + trace_block_range_error( + &trace, + &peer, + start, + count, + "unexpected_response", + started, + ); + warn!(?peer, ?response, "unexpected BlocksByHeightRange response"); + trace_block_range_finished(&trace, &peer, start, count, 0); + let _ = block_sync + .send(BlockSyncEvent::BlockRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + Ok(Err(error)) => { + trace_block_range_error( + &trace, + &peer, + start, + count, + &format!("{error}"), + started, + ); + warn!( + ?peer, + ?error, + "failed to read Zakura Blocks response from state" + ); + trace_block_range_finished(&trace, &peer, start, count, 0); + let _ = block_sync + .send(BlockSyncEvent::BlockRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + Err(_elapsed) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_TIMEOUT, + "block_sync_driver", + |row| { + insert_cs_str( + row, + cs_trace::ACTION, + "query_blocks_by_height_range", + ); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + warn!(?peer, "timed out reading Zakura block-sync serving range"); + trace_block_range_finished(&trace, &peer, start, count, 0); + let _ = block_sync + .send(BlockSyncEvent::BlockRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + } + } + BlockSyncAction::SubmitBlock { token, block } => { + let class = block_apply_class(block.as_ref(), max_checkpoint_height); + emit_commit_state( + &trace, + cs_trace::BLOCK_SUBMIT_QUEUED, + "block_sync_driver", + |row| { + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_str(row, cs_trace::APPLY_CLASS, block_apply_class_label(class)); + insert_cs_hash(row, cs_trace::HASH, block.hash()); + if let Some(height) = block.coinbase_height() { + insert_cs_height(row, cs_trace::HEIGHT, height); + } + insert_cs_u64(row, cs_trace::QUEUE_LEN, pending_applies.len() as u64); + insert_cs_u64( + row, + cs_trace::IN_FLIGHT_COUNT, + (checkpoint_in_flight.saturating_add(full_in_flight)) as u64, + ); + }, + ); + pending_applies.push_back(PendingBlockApply { + token, + class, + block, + }); + drain_pending_block_applies( + &mut pending_applies, + &mut in_flight_applies, + &mut checkpoint_in_flight, + &mut full_in_flight, + checkpoint_apply_limit, + full_apply_limit, + latest_chain_tip.clone(), + endpoint.clone(), + read_state.clone(), + block_verifier.clone(), + block_sync.clone(), + trace.clone(), + ); + } + } + } +} + +pub(crate) fn coalesce_stale_needed_block_queries( + action: BlockSyncAction, + actions: &mut mpsc::Receiver, + deferred_actions: &mut VecDeque, +) -> BlockSyncAction { + let BlockSyncAction::QueryNeededBlocks { + mut verified_block_tip, + mut best_header_tip, + } = action + else { + return action; + }; + + let mut coalesced_count = 0u64; + while let Ok(action) = actions.try_recv() { + match action { + BlockSyncAction::QueryNeededBlocks { + verified_block_tip: latest_verified_block_tip, + best_header_tip: latest_best_header_tip, + } => { + verified_block_tip = latest_verified_block_tip; + best_header_tip = latest_best_header_tip; + coalesced_count = coalesced_count.saturating_add(1); + } + action => deferred_actions.push_back(action), + } + } + + if coalesced_count > 0 { + metrics::counter!("sync.block.needed_query.coalesced").increment(coalesced_count); + } + + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } +} + +#[allow(clippy::too_many_arguments)] +fn drain_pending_block_applies( + pending_applies: &mut VecDeque, + in_flight_applies: &mut FuturesUnordered>, + checkpoint_in_flight: &mut usize, + full_in_flight: &mut usize, + checkpoint_apply_limit: usize, + full_apply_limit: usize, + latest_chain_tip: impl ChainTip + Clone + Send + Sync + 'static, + endpoint: Option, + read_state: ReadState, + block_verifier: BlockVerifier, + block_sync: BlockSyncHandle, + trace: ZakuraTrace, +) where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, + BlockVerifier: + Service + Clone + Send + 'static, + BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, + BlockVerifier::Future: Send + 'static, +{ + while let Some(index) = pending_applies + .iter() + .position(|pending| match pending.class { + BlockApplyClass::Checkpoint => *checkpoint_in_flight < checkpoint_apply_limit, + BlockApplyClass::Full => *full_in_flight < full_apply_limit, + }) + { + let pending = pending_applies + .remove(index) + .expect("pending apply index was found in queue"); + + match pending.class { + BlockApplyClass::Checkpoint => { + *checkpoint_in_flight = checkpoint_in_flight.saturating_add(1); + } + BlockApplyClass::Full => { + *full_in_flight = full_in_flight.saturating_add(1); + } + } + + let class = pending.class; + in_flight_applies.push( + apply_block_sync_body( + block_verifier.clone(), + latest_chain_tip.clone(), + endpoint.clone(), + read_state.clone(), + block_sync.clone(), + pending.token, + pending.block, + class, + trace.clone(), + ) + .boxed(), + ); + } +} + +pub(crate) fn block_apply_class( + block: &block::Block, + max_checkpoint_height: block::Height, +) -> BlockApplyClass { + if block + .coinbase_height() + .is_some_and(|height| height <= max_checkpoint_height) + { + BlockApplyClass::Checkpoint + } else { + BlockApplyClass::Full + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn apply_block_sync_body( + block_verifier: BlockVerifier, + latest_chain_tip: impl ChainTip + Clone + Send + Sync + 'static, + endpoint: Option, + read_state: ReadState, + block_sync: BlockSyncHandle, + token: BlockApplyToken, + block: Arc, + class: BlockApplyClass, + trace: ZakuraTrace, +) -> BlockApplyCompletion +where + BlockVerifier: + Service + Clone + Send + 'static, + BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, + BlockVerifier::Future: Send + 'static, + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + let expected_hash = block.hash(); + let Some(height) = block.coinbase_height() else { + warn!( + ?expected_hash, + "Zakura block sync cannot apply body without coinbase height" + ); + return BlockApplyCompletion { + class, + checkpoint_refresh_floor: None, + }; + }; + + emit_commit_state(&trace, cs_trace::COMMIT_START, "block_sync_driver", |row| { + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_str(row, cs_trace::APPLY_CLASS, block_apply_class_label(class)); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, expected_hash); + }); + let started = Instant::now(); + let result = commit_block_sync_body_with_stall_trace( + block_verifier.clone(), + block, + class, + &trace, + token, + height, + expected_hash, + ) + .await; + emit_commit_state( + &trace, + cs_trace::COMMIT_FINISH, + "block_sync_driver", + |row| { + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_str(row, cs_trace::APPLY_CLASS, block_apply_class_label(class)); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, expected_hash); + insert_cs_str(row, cs_trace::RESULT, block_apply_result_label(result)); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + emit_commit_state( + &trace, + cs_trace::FRONTIER_QUERY_START, + "block_sync_driver", + |row| { + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, expected_hash); + }, + ); + let local_frontier = + query_block_sync_frontiers(read_state.clone(), latest_chain_tip.clone()).await; + if let Some(frontiers) = local_frontier { + let change = + if result == BlockApplyResult::Committed || result == BlockApplyResult::Duplicate { + FrontierChange::VerifiedGrow + } else { + FrontierChange::Snapshot + }; + publish_body_frontier(endpoint.as_ref(), frontiers, change); + } + emit_commit_state( + &trace, + cs_trace::FRONTIER_QUERY_FINISH, + "block_sync_driver", + |row| { + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, expected_hash); + insert_cs_bool(row, cs_trace::LOCAL_FRONTIER, local_frontier.is_some()); + if let Some(frontiers) = &local_frontier { + insert_cs_frontiers(row, frontiers); + } + }, + ); + + let _ = block_sync + .send(BlockSyncEvent::BlockApplyFinished { + token, + height, + hash: expected_hash, + result, + local_frontier, + }) + .await; + emit_commit_state( + &trace, + cs_trace::REACTOR_EVENT_SENT, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "block_apply_finished"); + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, expected_hash); + insert_cs_str(row, cs_trace::RESULT, block_apply_result_label(result)); + insert_cs_bool(row, cs_trace::LOCAL_FRONTIER, local_frontier.is_some()); + }, + ); + + BlockApplyCompletion { + class, + checkpoint_refresh_floor: (class == BlockApplyClass::Checkpoint + && result == BlockApplyResult::Committed) + .then(|| { + local_frontier + .map(|frontiers| frontiers.verified_block_tip) + .unwrap_or_else(|| height.previous().unwrap_or(height)) + }), + } +} + +pub(crate) fn block_sync_misbehavior_is_hard(reason: BlockSyncMisbehavior) -> bool { + matches!( + reason, + BlockSyncMisbehavior::MalformedMessage + | BlockSyncMisbehavior::UnsolicitedBlock + | BlockSyncMisbehavior::GetBlocksTooLong + | BlockSyncMisbehavior::InvalidBlock + | BlockSyncMisbehavior::InvalidStatus + | BlockSyncMisbehavior::UnsolicitedDone + | BlockSyncMisbehavior::StatusSpam + ) +} + +#[cfg(test)] +pub(crate) async fn commit_block_sync_body( + block_verifier: BlockVerifier, + block: Arc, + class: BlockApplyClass, +) -> BlockApplyResult +where + BlockVerifier: + Service + Clone + Send + 'static, + BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, + BlockVerifier::Future: Send + 'static, +{ + let expected_hash = block.hash(); + let height = block.coinbase_height(); + let commit = block_verifier + .clone() + .oneshot(zebra_consensus::Request::Commit(block)); + match class { + BlockApplyClass::Checkpoint => block_commit_result(height, expected_hash, commit.await), + BlockApplyClass::Full => { + match tokio::time::timeout(ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, commit).await { + Ok(outcome) => block_commit_result(height, expected_hash, outcome), + Err(_elapsed) => block_commit_timed_out(height, expected_hash), + } + } + } +} + +#[allow(clippy::too_many_arguments)] +async fn commit_block_sync_body_with_stall_trace( + block_verifier: BlockVerifier, + block: Arc, + class: BlockApplyClass, + trace: &ZakuraTrace, + token: BlockApplyToken, + height: block::Height, + expected_hash: block::Hash, +) -> BlockApplyResult +where + BlockVerifier: + Service + Clone + Send + 'static, + BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, + BlockVerifier::Future: Send + 'static, +{ + let commit = block_verifier + .clone() + .oneshot(zebra_consensus::Request::Commit(block)); + + match class { + BlockApplyClass::Checkpoint => { + tokio::pin!(commit); + tokio::select! { + outcome = &mut commit => block_commit_result(Some(height), expected_hash, outcome), + _ = tokio::time::sleep(ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT) => { + emit_commit_state( + trace, + cs_trace::COMMIT_STALLED, + "block_sync_driver", + |row| { + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_str(row, cs_trace::APPLY_CLASS, block_apply_class_label(class)); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, expected_hash); + insert_cs_u64( + row, + cs_trace::ELAPSED_MS, + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT.as_millis().try_into().unwrap_or(u64::MAX), + ); + }, + ); + block_commit_result(Some(height), expected_hash, commit.await) + } + } + } + BlockApplyClass::Full => { + match tokio::time::timeout(ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, commit).await { + Ok(outcome) => block_commit_result(Some(height), expected_hash, outcome), + Err(_elapsed) => block_commit_timed_out(Some(height), expected_hash), + } + } + } +} + +fn block_commit_result( + height: Option, + expected_hash: block::Hash, + outcome: Result, +) -> BlockApplyResult +where + E: std::fmt::Debug + Send + Sync + 'static, +{ + match outcome { + Ok(committed_hash) if committed_hash == expected_hash => { + debug!( + ?height, + ?committed_hash, + "Zakura block sync committed block body through verifier" + ); + BlockApplyResult::Committed + } + Ok(committed_hash) => { + warn!( + ?height, + ?expected_hash, + ?committed_hash, + "Zakura block-sync verifier returned an unexpected hash" + ); + BlockApplyResult::Rejected + } + Err(error) => { + if block_verify_error_is_duplicate(&error) { + debug!( + ?height, + ?expected_hash, + ?error, + "Zakura block-sync body was already known by the block verifier" + ); + BlockApplyResult::Duplicate + } else { + debug!( + ?height, + ?expected_hash, + ?error, + "Zakura block-sync body rejected by block verifier" + ); + BlockApplyResult::Rejected + } + } + } +} + +fn block_commit_timed_out( + height: Option, + expected_hash: block::Hash, +) -> BlockApplyResult { + warn!( + ?height, + ?expected_hash, + "timed out committing Zakura block-sync body" + ); + BlockApplyResult::TimedOut +} + +async fn refresh_block_sync_frontiers_for_checkpoint_window( + read_state: ReadState, + latest_chain_tip: impl ChainTip + Clone + Send + Sync + 'static, + endpoint: Option, + block_sync: Option, + trace: ZakuraTrace, + refresh: &mut CheckpointFrontierRefresh, +) where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + let Some(mut highest_sent) = refresh.highest_sent else { + return; + }; + + emit_commit_state( + &trace, + cs_trace::CHECKPOINT_REFRESH_ATTEMPT, + "block_sync_driver", + |row| { + insert_cs_u64(row, "attempts_remaining", refresh.attempts_remaining as u64); + insert_cs_height(row, cs_trace::VERIFIED_BLOCK_TIP, highest_sent); + }, + ); + let Some(frontiers) = + query_block_sync_frontiers(read_state.clone(), latest_chain_tip.clone()).await + else { + refresh.finish_attempt(highest_sent); + return; + }; + + if frontiers.verified_block_tip <= highest_sent { + refresh.finish_attempt(highest_sent); + return; + } + + highest_sent = frontiers.verified_block_tip; + publish_body_frontier(endpoint.as_ref(), frontiers, FrontierChange::VerifiedGrow); + if let Some(block_sync) = &block_sync { + let _ = block_sync + .send(BlockSyncEvent::ChainTipGrow(frontiers)) + .await; + } + emit_commit_state( + &trace, + cs_trace::CHECKPOINT_REFRESH_SENT, + "block_sync_driver", + |row| { + insert_cs_frontiers(row, &frontiers); + }, + ); + refresh.finish_attempt(highest_sent); +} + +fn publish_body_frontier( + endpoint: Option<&ZakuraEndpoint>, + frontiers: zebra_network::zakura::BlockSyncFrontiers, + change: FrontierChange, +) { + let Some(endpoint) = endpoint else { + return; + }; + let Some(mut update) = endpoint.current_sync_frontier() else { + return; + }; + if frontiers.finalized_height == frontiers.verified_block_tip { + update.frontier.finalized = + Frontier::new(frontiers.finalized_height, frontiers.verified_block_hash); + } + update.frontier.verified_body = + Frontier::new(frontiers.verified_block_tip, frontiers.verified_block_hash); + update.change = change; + endpoint.publish_sync_frontier_from(update, "block_sync_driver"); +} + +async fn query_block_sync_needed_blocks( + read_state: ReadState, + verified_block_tip: block::Height, + best_header_tip: block::Height, +) -> Result, zebra_state::BoxError> +where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + let Some((from, limit)) = block_sync_missing_body_window(verified_block_tip, best_header_tip) + else { + return Ok(Vec::new()); + }; + + let missing = match tokio::time::timeout( + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, + read_state + .clone() + .oneshot(zebra_state::ReadRequest::MissingBlockBodies { from, limit }), + ) + .await + { + Ok(Ok(zebra_state::ReadResponse::MissingBlockBodies(heights))) => heights, + Ok(Ok(response)) => { + warn!(?response, "unexpected MissingBlockBodies response"); + return Ok(Vec::new()); + } + Ok(Err(error)) => return Err(error), + Err(elapsed) => return Err(Box::new(elapsed)), + }; + + let Some(first) = missing.first().copied() else { + return Ok(Vec::new()); + }; + let Some(last) = missing.last().copied() else { + return Ok(Vec::new()); + }; + let span = last.0.saturating_sub(first.0).saturating_add(1); + + let headers = match tokio::time::timeout( + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, + read_state + .clone() + .oneshot(zebra_state::ReadRequest::HeadersByHeightRange { + start: first, + count: span, + }), + ) + .await + { + Ok(Ok(zebra_state::ReadResponse::Headers(headers))) => headers, + Ok(Ok(response)) => { + warn!(?response, "unexpected HeadersByHeightRange response"); + return Ok(Vec::new()); + } + Ok(Err(error)) => return Err(error), + Err(elapsed) => return Err(Box::new(elapsed)), + }; + + let size_hints = match tokio::time::timeout( + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, + read_state.oneshot(zebra_state::ReadRequest::BlockSizeHints { + from: first, + count: span, + }), + ) + .await + { + Ok(Ok(zebra_state::ReadResponse::BlockSizeHints(hints))) => hints, + Ok(Ok(response)) => { + warn!(?response, "unexpected BlockSizeHints response"); + Vec::new() + } + Ok(Err(error)) => return Err(error), + Err(elapsed) => return Err(Box::new(elapsed)), + }; + + Ok(block_sync_needed_blocks_from_state( + missing, headers, size_hints, + )) +} + +pub(crate) fn block_sync_missing_body_window( + verified_block_tip: block::Height, + best_header_tip: block::Height, +) -> Option<(block::Height, u32)> { + if best_header_tip <= verified_block_tip { + return None; + } + + let from = block::Height(verified_block_tip.0.saturating_add(1)); + let limit = best_header_tip + .0 + .saturating_sub(verified_block_tip.0) + .clamp(1, zebra_state::MAX_BLOCK_REORG_HEIGHT); + Some((from, limit)) +} + +pub(crate) fn block_sync_needed_blocks_from_state( + missing: Vec, + headers: Vec<(block::Height, block::Hash, Arc)>, + size_hints: Vec<(block::Height, Option)>, +) -> Vec { + let headers: HashMap<_, _> = headers + .into_iter() + .map(|(height, hash, _header)| (height, hash)) + .collect(); + let size_hints: HashMap<_, _> = size_hints.into_iter().collect(); + + missing + .into_iter() + .filter_map(|height| { + let hash = *headers.get(&height)?; + let size = size_hints + .get(&height) + .copied() + .flatten() + .filter(|size| *size > 0) + .map(BlockSizeEstimate::Advertised) + .unwrap_or(BlockSizeEstimate::Unknown); + + Some(BlockSyncBlockMeta { height, hash, size }) + }) + .collect() +} + +fn trace_block_driver_action(trace: &ZakuraTrace, action: &BlockSyncAction) { + emit_commit_state( + trace, + cs_trace::ACTION_RECEIVED, + "block_sync_driver", + |row| match action { + BlockSyncAction::SendMessage { peer, msg } => { + insert_cs_str(row, cs_trace::ACTION, "send_message"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_str(row, cs_trace::REASON, block_sync_message_label(msg)); + } + BlockSyncAction::Misbehavior { peer, reason } => { + insert_cs_str(row, cs_trace::ACTION, "misbehavior"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_str(row, cs_trace::REASON, block_sync_misbehavior_label(*reason)); + } + BlockSyncAction::QueryNeededBlocks { + verified_block_tip, + best_header_tip, + } => { + insert_cs_str(row, cs_trace::ACTION, "query_needed_blocks"); + insert_cs_height(row, cs_trace::VERIFIED_BLOCK_TIP, *verified_block_tip); + insert_cs_height(row, cs_trace::BEST_HEADER_TIP, *best_header_tip); + } + BlockSyncAction::QueryBlocksByHeightRange { peer, start, count } => { + insert_cs_str(row, cs_trace::ACTION, "query_blocks_by_height_range"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::RANGE_START, *start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(*count)); + } + BlockSyncAction::SubmitBlock { token, block } => { + insert_cs_str(row, cs_trace::ACTION, "submit_block"); + insert_cs_u64(row, cs_trace::APPLY_TOKEN, *token); + insert_cs_hash(row, cs_trace::HASH, block.hash()); + if let Some(height) = block.coinbase_height() { + insert_cs_height(row, cs_trace::HEIGHT, height); + } + } + }, + ); +} + +fn trace_block_range_error( + trace: &ZakuraTrace, + peer: &zebra_network::zakura::ZakuraPeerId, + start: block::Height, + count: u32, + reason: &str, + started: Instant, +) { + emit_commit_state( + trace, + cs_trace::STATE_READ_ERROR, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "query_blocks_by_height_range"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + insert_cs_str(row, cs_trace::RESULT, "error"); + insert_cs_str(row, cs_trace::REASON, reason); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); +} + +fn trace_block_range_finished( + trace: &ZakuraTrace, + peer: &zebra_network::zakura::ZakuraPeerId, + start: block::Height, + requested_count: u32, + returned_count: u32, +) { + emit_commit_state( + trace, + cs_trace::REACTOR_EVENT_SENT, + "block_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "block_range_response_finished"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(returned_count)); + insert_cs_u64(row, "requested_count", u64::from(requested_count)); + }, + ); +} + +fn block_apply_class_label(class: BlockApplyClass) -> &'static str { + match class { + BlockApplyClass::Checkpoint => "checkpoint", + BlockApplyClass::Full => "full", + } +} + +fn block_sync_message_label(msg: &BlockSyncMessage) -> &'static str { + match msg { + BlockSyncMessage::Status(_) => "status", + BlockSyncMessage::Block(_) => "block", + BlockSyncMessage::BlocksDone { .. } => "blocks_done", + BlockSyncMessage::RangeUnavailable { .. } => "range_unavailable", + BlockSyncMessage::GetBlocks { .. } => "get_blocks", + } +} + +fn block_sync_misbehavior_label(reason: BlockSyncMisbehavior) -> &'static str { + match reason { + BlockSyncMisbehavior::MalformedMessage => "malformed_message", + BlockSyncMisbehavior::UnsolicitedBlock => "unsolicited_block", + BlockSyncMisbehavior::GetBlocksTooLong => "get_blocks_too_long", + BlockSyncMisbehavior::GetBlocksSpam => "get_blocks_spam", + BlockSyncMisbehavior::InvalidBlock => "invalid_block", + BlockSyncMisbehavior::SizeMismatch => "size_mismatch", + BlockSyncMisbehavior::InvalidStatus => "invalid_status", + BlockSyncMisbehavior::UnsolicitedDone => "unsolicited_done", + BlockSyncMisbehavior::RangeUnavailable => "range_unavailable", + BlockSyncMisbehavior::StatusSpam => "status_spam", + } +} + +fn elapsed_ms(started: Instant) -> u64 { + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) +} diff --git a/zebrad/src/commands/start/zakura/frontier.rs b/zebrad/src/commands/start/zakura/frontier.rs new file mode 100644 index 00000000000..fb611acef15 --- /dev/null +++ b/zebrad/src/commands/start/zakura/frontier.rs @@ -0,0 +1,106 @@ +use tower::{Service, ServiceExt}; +use tracing::warn; + +use zebra_chain::{block, chain_tip::ChainTip}; +use zebra_network::zakura::BlockSyncFrontiers; + +use super::ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT; + +pub(crate) fn verified_block_tip_from_state( + finalized_tip: Option<(block::Height, block::Hash)>, + tip: Option<(block::Height, block::Hash)>, + empty_state_tip: (block::Height, block::Hash), +) -> (block::Height, block::Hash) { + let finalized_tip = finalized_tip.unwrap_or(empty_state_tip); + let tip = tip.unwrap_or(empty_state_tip); + + if finalized_tip.0 > tip.0 { + finalized_tip + } else { + tip + } +} + +pub(crate) async fn query_block_sync_frontiers( + read_state: ReadState, + latest_chain_tip: impl ChainTip + Clone + Send + Sync + 'static, +) -> Option +where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + let latest_tip = latest_chain_tip.best_tip_height_and_hash(); + let finalized_tip = match tokio::time::timeout( + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, + read_state + .clone() + .oneshot(zebra_state::ReadRequest::FinalizedTip), + ) + .await + { + Ok(Ok(zebra_state::ReadResponse::FinalizedTip(tip))) => tip, + Ok(Ok(response)) => { + warn!(?response, "unexpected FinalizedTip response"); + None + } + Ok(Err(error)) => { + warn!( + ?error, + "failed to refresh Zakura block-sync finalized frontier" + ); + None + } + Err(_elapsed) => { + warn!("timed out refreshing Zakura block-sync finalized frontier"); + None + } + }; + + match tokio::time::timeout( + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, + read_state.oneshot(zebra_state::ReadRequest::Tip), + ) + .await + { + Ok(Ok(zebra_state::ReadResponse::Tip(tip))) => { + let state_tip = (finalized_tip.is_some() || tip.is_some()).then(|| { + verified_block_tip_from_state( + finalized_tip, + tip, + latest_tip.unwrap_or((block::Height(0), block::Hash([0; 32]))), + ) + }); + let (height, hash) = match (state_tip, latest_tip) { + (Some(state_tip), latest_tip) => { + verified_block_tip_from_state(Some(state_tip), latest_tip, state_tip) + } + (None, Some(latest_tip)) => latest_tip, + (None, None) => return None, + }; + let finalized_height = finalized_tip.map_or(block::Height(0), |(height, _)| height); + Some(BlockSyncFrontiers { + finalized_height, + verified_block_tip: height, + verified_block_hash: hash, + }) + } + Ok(Ok(response)) => { + warn!(?response, "unexpected Tip response"); + None + } + Ok(Err(error)) => { + warn!(?error, "failed to refresh Zakura block-sync body frontier"); + None + } + Err(_elapsed) => { + warn!("timed out refreshing Zakura block-sync body frontier"); + None + } + } +} diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs new file mode 100644 index 00000000000..aa3f44356cf --- /dev/null +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -0,0 +1,1286 @@ +use std::{future::Future, time::Instant}; + +use color_eyre::eyre::{eyre, Report}; +use tokio::{pin, select, sync::mpsc}; +use tower::{Service, ServiceExt}; +use tracing::{debug, warn}; + +use zebra_chain::{ + block::{self}, + chain_tip::ChainTip, +}; +use zebra_network::zakura::{ + commit_state_trace as cs_trace, BlockSyncFrontiers, Frontier, FrontierChange, HeaderSyncAction, + HeaderSyncCommitFailureKind, HeaderSyncEvent, HeaderSyncFrontiers, ZakuraEndpoint, + ZakuraHeaderSyncDriverStartup, ZakuraTrace, DEFAULT_HS_RANGE, +}; + +#[cfg(test)] +use zebra_network::zakura::{BlockSyncEvent, BlockSyncHandle}; + +use super::{ + block_verify_error_is_duplicate, emit_commit_state, insert_cs_frontiers, insert_cs_hash, + insert_cs_height, insert_cs_peer, insert_cs_str, insert_cs_u64, verified_block_tip_from_state, +}; + +pub(crate) async fn zakura_header_sync_driver_startup( + read_state: zebra_state::ReadStateService, + network: &zebra_chain::parameters::Network, +) -> Result { + let best_header_tip = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::BestHeaderTip) + .await + .map_err(|error| eyre!("{error}"))? + { + zebra_state::ReadResponse::BestHeaderTip(tip) => tip, + response => Err(eyre!("unexpected BestHeaderTip response: {response:?}"))?, + }; + + let finalized_tip = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::FinalizedTip) + .await + .map_err(|error| eyre!("{error}"))? + { + zebra_state::ReadResponse::FinalizedTip(tip) => tip, + response => Err(eyre!("unexpected FinalizedTip response: {response:?}"))?, + }; + + let verified_block_tip = match read_state + .oneshot(zebra_state::ReadRequest::Tip) + .await + .map_err(|error| eyre!("{error}"))? + { + zebra_state::ReadResponse::Tip(tip) => tip, + response => Err(eyre!("unexpected Tip response: {response:?}"))?, + }; + + let empty_state_tip = (block::Height(0), network.genesis_hash()); + let finalized_height = finalized_tip.map_or(block::Height(0), |(height, _)| height); + let verified_block_tip = + verified_block_tip_from_state(finalized_tip, verified_block_tip, empty_state_tip); + Ok(ZakuraHeaderSyncDriverStartup { + frontiers: HeaderSyncFrontiers { + finalized_height, + verified_block_tip: verified_block_tip.0, + verified_block_hash: verified_block_tip.1, + }, + best_header_tip: Some(best_header_tip.unwrap_or(empty_state_tip)), + verified_block_tip_hash: verified_block_tip.1, + }) +} + +#[derive(Clone)] +pub(crate) struct ZakuraHeaderSyncDriverHandles { + pub(crate) endpoint: ZakuraEndpoint, + pub(crate) header_sync: zebra_network::zakura::HeaderSyncHandle, +} + +pub(crate) async fn drive_zakura_header_sync_actions( + mut actions: mpsc::Receiver, + handles: ZakuraHeaderSyncDriverHandles, + state: State, + read_state: ReadState, + block_verifier: BlockVerifier, + trace: ZakuraTrace, + shutdown: impl Future + Send + 'static, +) where + State: Service< + zebra_state::Request, + Response = zebra_state::Response, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + State::Future: Send + 'static, + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, + BlockVerifier: + Service + Clone + Send + 'static, + BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, + BlockVerifier::Future: Send + 'static, +{ + pin!(shutdown); + loop { + let action = select! { + _ = &mut shutdown => return, + action = actions.recv() => { + let Some(action) = action else { + return; + }; + action + } + }; + + trace_header_driver_action(&trace, &action); + match action { + HeaderSyncAction::Misbehavior { peer, reason } => { + debug!( + ?peer, + ?reason, + "disconnecting peer for Zakura header-sync violation" + ); + let _ = handles.endpoint.supervisor().disconnect_peer(&peer).await; + } + HeaderSyncAction::NewBlockReceived { + peer, + height, + hash, + block, + } => { + emit_commit_state( + &trace, + cs_trace::COMMIT_START, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "new_block"); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + }, + ); + let started = Instant::now(); + match block_verifier + .clone() + .oneshot(zebra_consensus::Request::Commit(block.clone())) + .await + { + Ok(committed_hash) if committed_hash == hash => { + trace_header_commit_finish( + &trace, + "new_block", + &peer, + height, + hash, + "accepted", + started, + ); + trace_header_reactor_event( + &trace, + "new_block_accepted", + Some(&peer), + height, + hash, + 1, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::NewBlockAccepted { + peer, + height, + hash, + block, + }) + .await; + } + Ok(committed_hash) => { + trace_header_commit_finish( + &trace, + "new_block", + &peer, + height, + hash, + "rejected", + started, + ); + warn!( + ?peer, + ?hash, + ?committed_hash, + "Zakura NewBlock verifier returned an unexpected hash" + ); + trace_header_reactor_event( + &trace, + "new_block_rejected", + Some(&peer), + height, + hash, + 1, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::NewBlockRejected { peer, hash }) + .await; + } + Err(error) => { + if block_verify_error_is_duplicate(&error) { + trace_header_commit_finish( + &trace, + "new_block", + &peer, + height, + hash, + "duplicate", + started, + ); + debug!( + ?peer, + ?height, + ?hash, + ?error, + "Zakura NewBlock was already known by the block verifier" + ); + trace_header_reactor_event( + &trace, + "new_block_duplicate", + Some(&peer), + height, + hash, + 1, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::NewBlockDuplicate { peer, height, hash }) + .await; + continue; + } + + trace_header_commit_finish( + &trace, + "new_block", + &peer, + height, + hash, + "rejected", + started, + ); + debug!( + ?peer, + ?hash, + ?error, + "Zakura NewBlock rejected by block verifier" + ); + trace_header_reactor_event( + &trace, + "new_block_rejected", + Some(&peer), + height, + hash, + 1, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::NewBlockRejected { peer, hash }) + .await; + } + } + } + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + trace_state_read_start( + &trace, + "query_headers_by_height_range", + Some(&peer), + start, + count, + ); + let started = Instant::now(); + match read_state + .clone() + .oneshot(zebra_state::ReadRequest::HeadersByHeightRange { start, count }) + .await + { + Ok(zebra_state::ReadResponse::Headers(headers)) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_SUCCESS, + "header_sync_driver", + |row| { + insert_cs_str( + row, + cs_trace::ACTION, + "query_headers_by_height_range", + ); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, headers.len() as u64); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + trace_state_read_start( + &trace, + "block_size_hints", + Some(&peer), + start, + count, + ); + let body_size_hints = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::BlockSizeHints { + from: start, + count, + }) + .await + { + Ok(zebra_state::ReadResponse::BlockSizeHints(hints)) => hints, + Ok(response) => { + trace_state_read_error( + &trace, + "block_size_hints", + Some(&peer), + start, + count, + "unexpected_response", + started, + ); + warn!(?peer, ?response, "unexpected BlockSizeHints response"); + Vec::new() + } + Err(error) => { + trace_state_read_error( + &trace, + "block_size_hints", + Some(&peer), + start, + count, + &format!("{error}"), + started, + ); + warn!( + ?peer, + ?error, + "failed to read Zakura BlockSizeHints response from state" + ); + Vec::new() + } + }; + let body_sizes = body_sizes_for_served_header_range( + start, + headers.iter().map(|(height, _, _)| *height), + &body_size_hints, + ); + let headers = headers + .into_iter() + .map(|(_height, _hash, header)| header) + .collect(); + trace_header_reactor_event( + &trace, + "header_range_response_ready", + Some(&peer), + start, + block::Hash([0; 32]), + count, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::HeaderRangeResponseReady { + peer, + start_height: start, + requested_count: count, + headers, + body_sizes, + }) + .await; + } + Ok(response) => { + trace_state_read_error( + &trace, + "query_headers_by_height_range", + Some(&peer), + start, + count, + "unexpected_response", + started, + ); + warn!(?peer, ?response, "unexpected HeadersByHeightRange response"); + trace_header_range_finished(&trace, &peer, start, count, 0); + let _ = handles + .header_sync + .send(HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + Err(error) => { + trace_state_read_error( + &trace, + "query_headers_by_height_range", + Some(&peer), + start, + count, + &format!("{error}"), + started, + ); + warn!( + ?peer, + ?error, + "failed to read Zakura Headers response from state" + ); + trace_header_range_finished(&trace, &peer, start, count, 0); + let _ = handles + .header_sync + .send(HeaderSyncEvent::HeaderRangeResponseFinished { + peer, + start_height: start, + requested_count: count, + returned_count: 0, + }) + .await; + } + } + } + HeaderSyncAction::CommitHeaderRange { + peer, + anchor, + start_height, + headers, + body_sizes, + finalized: _finalized, + } => { + let count = u32::try_from(headers.len()).unwrap_or(u32::MAX); + emit_commit_state( + &trace, + cs_trace::COMMIT_START, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "commit_header_range"); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start_height); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + insert_cs_hash(row, cs_trace::HASH, anchor); + }, + ); + let started = Instant::now(); + match state + .clone() + .oneshot(zebra_state::Request::CommitHeaderRange { + anchor, + headers, + body_sizes, + }) + .await + { + Ok(zebra_state::Response::Committed(tip_hash)) => { + emit_commit_state( + &trace, + cs_trace::COMMIT_FINISH, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "commit_header_range"); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start_height); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + insert_cs_str(row, cs_trace::RESULT, "committed"); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + let tip_height = + block::Height(start_height.0.saturating_add(count.saturating_sub(1))); + let _ = handles + .header_sync + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height, + tip_height, + tip_hash, + }) + .await; + trace_header_reactor_event( + &trace, + "header_range_committed", + None, + tip_height, + tip_hash, + count, + ); + publish_header_frontier( + &handles.endpoint, + tip_height, + tip_hash, + FrontierChange::HeaderAdvanced, + &trace, + ); + } + Ok(response) => { + emit_commit_state( + &trace, + cs_trace::COMMIT_FINISH, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "commit_header_range"); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start_height); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + insert_cs_str(row, cs_trace::RESULT, "unexpected_response"); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + warn!(?peer, ?response, "unexpected CommitHeaderRange response"); + trace_header_reactor_event( + &trace, + "header_range_commit_failed", + Some(&peer), + start_height, + block::Hash([0; 32]), + count, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind: HeaderSyncCommitFailureKind::Local, + }) + .await; + } + Err(error) => { + let kind = header_range_commit_failure_kind(error.as_ref()); + emit_commit_state( + &trace, + cs_trace::COMMIT_FINISH, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "commit_header_range"); + insert_cs_peer(row, cs_trace::PEER, &peer); + insert_cs_height(row, cs_trace::RANGE_START, start_height); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + insert_cs_str( + row, + cs_trace::RESULT, + commit_failure_result_label(kind), + ); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + debug!( + ?peer, + ?start_height, + ?count, + ?kind, + ?error, + "Zakura header range commit failed" + ); + trace_header_reactor_event( + &trace, + "header_range_commit_failed", + Some(&peer), + start_height, + block::Hash([0; 32]), + count, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count, + kind, + }) + .await; + } + } + } + HeaderSyncAction::QueryBestHeaderTip => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_START, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "query_best_header_tip"); + }, + ); + match read_state + .clone() + .oneshot(zebra_state::ReadRequest::BestHeaderTip) + .await + { + Ok(zebra_state::ReadResponse::BestHeaderTip(Some((tip_height, tip_hash)))) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_SUCCESS, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "query_best_header_tip"); + insert_cs_height(row, cs_trace::BEST_HEADER_TIP, tip_height); + insert_cs_hash(row, cs_trace::HASH, tip_hash); + }, + ); + let _ = handles + .header_sync + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: tip_height, + tip_height, + tip_hash, + }) + .await; + publish_header_frontier( + &handles.endpoint, + tip_height, + tip_hash, + FrontierChange::HeaderAdvanced, + &trace, + ); + } + Ok(zebra_state::ReadResponse::BestHeaderTip(None)) => {} + Ok(response) => { + trace_state_read_error( + &trace, + "query_best_header_tip", + None, + block::Height(0), + 0, + "unexpected_response", + Instant::now(), + ); + warn!(?response, "unexpected BestHeaderTip response") + } + Err(error) => { + trace_state_read_error( + &trace, + "query_best_header_tip", + None, + block::Height(0), + 0, + &format!("{error}"), + Instant::now(), + ); + warn!(?error, "failed to query Zakura best header tip") + } + } + } + HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { + log_missing_block_bodies(read_state.clone(), from, limit, &trace).await; + } + HeaderSyncAction::BodyGaps { from, to } => { + let limit = + to.0.saturating_sub(from.0) + .saturating_add(1) + .min(DEFAULT_HS_RANGE); + log_missing_block_bodies(read_state.clone(), from, limit, &trace).await; + } + HeaderSyncAction::HeaderAdvanced { height, hash } => { + publish_header_frontier( + &handles.endpoint, + height, + hash, + FrontierChange::HeaderAdvanced, + &trace, + ); + } + HeaderSyncAction::HeaderReanchored { old: _, new } => { + publish_header_frontier( + &handles.endpoint, + new.0, + new.1, + FrontierChange::HeaderReanchored, + &trace, + ); + } + } + } +} + +pub(crate) fn publish_header_frontier( + endpoint: &ZakuraEndpoint, + height: block::Height, + hash: block::Hash, + change: FrontierChange, + trace: &ZakuraTrace, +) { + let Some(mut update) = endpoint.current_sync_frontier() else { + return; + }; + + update.frontier.best_header = Frontier::new(height, hash); + update.change = change; + endpoint.publish_sync_frontier_from(update, "header_sync_driver"); + emit_commit_state( + trace, + cs_trace::BLOCK_SYNC_NOTIFY_SENT, + "header_sync_driver", + |row| { + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + }, + ); +} + +#[cfg(test)] +pub(crate) async fn notify_block_sync_header_tip( + block_sync: Option<&BlockSyncHandle>, + height: block::Height, + hash: block::Hash, + trace: &ZakuraTrace, +) { + if let Some(block_sync) = block_sync { + let _ = block_sync + .send(BlockSyncEvent::HeaderTipChanged { height, hash }) + .await; + emit_commit_state( + trace, + cs_trace::BLOCK_SYNC_NOTIFY_SENT, + "header_sync_driver", + |row| { + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + }, + ); + } +} + +pub(crate) fn body_sizes_for_served_header_range( + start: block::Height, + header_heights: impl IntoIterator, + body_size_hints: &[(block::Height, Option)], +) -> Vec { + header_heights + .into_iter() + .map(|height| { + let Some(offset) = usize::try_from(height - start).ok() else { + return 0; + }; + + body_size_hints + .get(offset) + .and_then(|(hint_height, size)| { + (*hint_height == height).then_some(size.unwrap_or(0)) + }) + .unwrap_or(0) + }) + .collect() +} + +async fn log_missing_block_bodies( + read_state: ReadState, + from: block::Height, + limit: u32, + trace: &ZakuraTrace, +) where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Send + + 'static, + ReadState::Future: Send + 'static, +{ + trace_state_read_start(trace, "missing_block_bodies", None, from, limit); + let started = Instant::now(); + match read_state + .oneshot(zebra_state::ReadRequest::MissingBlockBodies { from, limit }) + .await + { + Ok(zebra_state::ReadResponse::MissingBlockBodies(heights)) => { + emit_commit_state( + trace, + cs_trace::STATE_READ_SUCCESS, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "missing_block_bodies"); + insert_cs_height(row, cs_trace::RANGE_START, from); + insert_cs_u64(row, cs_trace::RANGE_COUNT, heights.len() as u64); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + let first = heights.first().copied(); + let last = heights.last().copied(); + let count = heights.len(); + debug!( + ?from, + ?limit, + ?count, + ?first, + ?last, + "Zakura header-known body gaps from state" + ); + } + Ok(response) => { + trace_state_read_error( + trace, + "missing_block_bodies", + None, + from, + limit, + "unexpected_response", + started, + ); + warn!(?response, "unexpected MissingBlockBodies response") + } + Err(error) => { + trace_state_read_error( + trace, + "missing_block_bodies", + None, + from, + limit, + &format!("{error}"), + started, + ); + warn!(?error, "failed to query Zakura missing block bodies") + } + } +} + +pub(crate) fn header_range_commit_failure_kind( + error: &(dyn std::error::Error + Send + Sync + 'static), +) -> HeaderSyncCommitFailureKind { + let Some(error) = error.downcast_ref::() else { + return HeaderSyncCommitFailureKind::Local; + }; + + match error { + zebra_state::CommitHeaderRangeError::StorageWriteError { .. } + | zebra_state::CommitHeaderRangeError::MissingGenesisAnchor { .. } + | zebra_state::CommitHeaderRangeError::SendCommitRequestFailed + | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { + HeaderSyncCommitFailureKind::Local + } + zebra_state::CommitHeaderRangeError::EmptyRange + | zebra_state::CommitHeaderRangeError::RangeTooLong { .. } + | zebra_state::CommitHeaderRangeError::UnknownAnchor { .. } + | zebra_state::CommitHeaderRangeError::HeightOverflow + | zebra_state::CommitHeaderRangeError::ImmutableConflict { .. } + | zebra_state::CommitHeaderRangeError::ReorgTooDeep { .. } + | zebra_state::CommitHeaderRangeError::CheckpointConflict { .. } + | zebra_state::CommitHeaderRangeError::ConflictingFullBlockHeader { .. } + | zebra_state::CommitHeaderRangeError::ValidateContextError(_) => { + HeaderSyncCommitFailureKind::InvalidPeerRange + } + _ => HeaderSyncCommitFailureKind::Local, + } +} + +pub(crate) async fn mirror_zakura_full_block_commits( + mut chain_tip_change: zebra_state::ChainTipChange, + latest_chain_tip: zebra_state::LatestChainTip, + read_state: ReadState, + header_sync: zebra_network::zakura::HeaderSyncHandle, + endpoint: ZakuraEndpoint, + trace: ZakuraTrace, + shutdown: impl Future + Send + 'static, +) where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + pin!(shutdown); + loop { + let action = select! { + _ = &mut shutdown => return, + action = chain_tip_change.wait_for_tip_change() => { + let Ok(action) = action else { + return; + }; + action + } + }; + let height = action.best_tip_height(); + let hash = action.best_tip_hash(); + emit_commit_state( + &trace, + cs_trace::CHAIN_TIP_ACTION, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, tip_action_label(&action)); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + }, + ); + + let finalized_tip = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::FinalizedTip) + .await + { + Ok(zebra_state::ReadResponse::FinalizedTip(tip)) => tip, + Ok(response) => { + warn!(?response, "unexpected FinalizedTip response"); + None + } + Err(error) => { + warn!(?error, "failed to query Zakura finalized frontier"); + None + } + }; + let finalized_height = finalized_tip.map_or(block::Height(0), |(height, _)| height); + emit_commit_state( + &trace, + cs_trace::STATE_READ_SUCCESS, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "finalized_tip"); + insert_cs_height(row, cs_trace::FINALIZED_HEIGHT, finalized_height); + }, + ); + let action_tip = Some((height, hash)); + let verified_block_tip = + verified_block_tip_from_state(finalized_tip, action_tip, (height, hash)); + let verified_block_tip = verified_block_tip_from_state( + Some(verified_block_tip), + latest_chain_tip.best_tip_height_and_hash(), + verified_block_tip, + ); + + emit_commit_state( + &trace, + cs_trace::FRONTIER_DERIVED, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "sync_exchange_frontier_derived"); + insert_cs_height(row, cs_trace::FINALIZED_HEIGHT, finalized_height); + insert_cs_height(row, cs_trace::VERIFIED_BLOCK_TIP, verified_block_tip.0); + insert_cs_hash(row, cs_trace::VERIFIED_BLOCK_HASH, verified_block_tip.1); + }, + ); + if let Some(mut update) = endpoint.current_sync_frontier() { + if let Some((finalized_height, finalized_hash)) = finalized_tip { + update.frontier.finalized = Frontier::new(finalized_height, finalized_hash); + } + update.frontier.verified_body = + Frontier::new(verified_block_tip.0, verified_block_tip.1); + update.change = match action { + zebra_state::TipAction::Grow { .. } => FrontierChange::VerifiedGrow, + zebra_state::TipAction::Reset { .. } => FrontierChange::VerifiedReset, + }; + endpoint.publish_sync_frontier_from(update, "chain_tip_mirror"); + emit_commit_state( + &trace, + cs_trace::FRONTIER_DERIVED, + "chain_tip_mirror", + |row| { + let frontiers = BlockSyncFrontiers { + finalized_height, + verified_block_tip: verified_block_tip.0, + verified_block_hash: verified_block_tip.1, + }; + insert_cs_str(row, cs_trace::ACTION, "sync_exchange_frontier_sent"); + insert_cs_frontiers(row, &frontiers); + }, + ); + } + + emit_commit_state( + &trace, + cs_trace::STATE_READ_START, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "committed_tip_block"); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + }, + ); + match read_state + .clone() + .oneshot(zebra_state::ReadRequest::Block(hash.into())) + .await + { + Ok(zebra_state::ReadResponse::Block(Some(block))) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_SUCCESS, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "committed_tip_block"); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + insert_cs_str(row, cs_trace::RESULT, "found"); + }, + ); + let _ = header_sync + .send(HeaderSyncEvent::FullBlockCommitted { + height, + hash, + header: block.header.clone(), + }) + .await; + emit_commit_state( + &trace, + cs_trace::REACTOR_EVENT_SENT, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "full_block_committed"); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + }, + ); + } + Ok(zebra_state::ReadResponse::Block(None)) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_SUCCESS, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "committed_tip_block"); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + insert_cs_str(row, cs_trace::RESULT, "missing"); + }, + ); + debug!( + ?height, + ?hash, + "Zakura full-block mirror could not find committed tip block" + ); + } + Ok(response) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_ERROR, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "committed_tip_block"); + insert_cs_str(row, cs_trace::REASON, "unexpected_response"); + }, + ); + warn!(?response, "unexpected block lookup response") + } + Err(error) => { + emit_commit_state( + &trace, + cs_trace::STATE_READ_ERROR, + "chain_tip_mirror", + |row| { + insert_cs_str(row, cs_trace::ACTION, "committed_tip_block"); + insert_cs_str(row, cs_trace::REASON, &format!("{error}")); + }, + ); + warn!(?error, "failed to mirror Zakura full-block commit") + } + } + } +} + +#[cfg(test)] +pub(crate) fn block_sync_chain_tip_event( + action: &zebra_state::TipAction, + frontiers: BlockSyncFrontiers, +) -> BlockSyncEvent { + match action { + zebra_state::TipAction::Grow { .. } => BlockSyncEvent::ChainTipGrow(frontiers), + zebra_state::TipAction::Reset { .. } => BlockSyncEvent::ChainTipReset(frontiers), + } +} + +fn trace_header_driver_action(trace: &ZakuraTrace, action: &HeaderSyncAction) { + emit_commit_state( + trace, + cs_trace::ACTION_RECEIVED, + "header_sync_driver", + |row| match action { + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + .. + } => { + insert_cs_str(row, cs_trace::ACTION, "commit_header_range"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::RANGE_START, *start_height); + insert_cs_u64(row, cs_trace::RANGE_COUNT, headers.len() as u64); + } + HeaderSyncAction::QueryBestHeaderTip => { + insert_cs_str(row, cs_trace::ACTION, "query_best_header_tip"); + } + HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count } => { + insert_cs_str(row, cs_trace::ACTION, "query_headers_by_height_range"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::RANGE_START, *start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(*count)); + } + HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { + insert_cs_str(row, cs_trace::ACTION, "query_missing_block_bodies"); + insert_cs_height(row, cs_trace::RANGE_START, *from); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(*limit)); + } + HeaderSyncAction::Misbehavior { peer, reason } => { + insert_cs_str(row, cs_trace::ACTION, "misbehavior"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_str(row, cs_trace::REASON, header_misbehavior_label(*reason)); + } + HeaderSyncAction::BodyGaps { from, to } => { + insert_cs_str(row, cs_trace::ACTION, "body_gaps"); + insert_cs_height(row, cs_trace::RANGE_START, *from); + insert_cs_u64( + row, + cs_trace::RANGE_COUNT, + u64::from(to.0.saturating_sub(from.0).saturating_add(1)), + ); + } + HeaderSyncAction::HeaderAdvanced { height, hash } => { + insert_cs_str(row, cs_trace::ACTION, "header_advanced"); + insert_cs_height(row, cs_trace::HEIGHT, *height); + insert_cs_hash(row, cs_trace::HASH, *hash); + } + HeaderSyncAction::HeaderReanchored { old, new } => { + insert_cs_str(row, cs_trace::ACTION, "header_reanchored"); + insert_cs_height(row, cs_trace::BEST_HEADER_TIP, old.0); + insert_cs_height(row, cs_trace::HEIGHT, new.0); + insert_cs_hash(row, cs_trace::HASH, new.1); + } + HeaderSyncAction::NewBlockReceived { + peer, height, hash, .. + } => { + insert_cs_str(row, cs_trace::ACTION, "new_block_received"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::HEIGHT, *height); + insert_cs_hash(row, cs_trace::HASH, *hash); + } + }, + ); +} + +fn trace_header_commit_finish( + trace: &ZakuraTrace, + action: &'static str, + peer: &zebra_network::zakura::ZakuraPeerId, + height: block::Height, + hash: block::Hash, + result: &'static str, + started: Instant, +) { + emit_commit_state( + trace, + cs_trace::COMMIT_FINISH, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, action); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + insert_cs_str(row, cs_trace::RESULT, result); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); +} + +fn trace_header_reactor_event( + trace: &ZakuraTrace, + action: &'static str, + peer: Option<&zebra_network::zakura::ZakuraPeerId>, + height: block::Height, + hash: block::Hash, + count: u32, +) { + emit_commit_state( + trace, + cs_trace::REACTOR_EVENT_SENT, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, action); + if let Some(peer) = peer { + insert_cs_peer(row, cs_trace::PEER, peer); + } + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, hash); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + }, + ); +} + +fn trace_header_range_finished( + trace: &ZakuraTrace, + peer: &zebra_network::zakura::ZakuraPeerId, + start: block::Height, + requested_count: u32, + returned_count: u32, +) { + emit_commit_state( + trace, + cs_trace::REACTOR_EVENT_SENT, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "header_range_response_finished"); + insert_cs_peer(row, cs_trace::PEER, peer); + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(returned_count)); + insert_cs_u64(row, "requested_count", u64::from(requested_count)); + }, + ); +} + +fn trace_state_read_start( + trace: &ZakuraTrace, + action: &'static str, + peer: Option<&zebra_network::zakura::ZakuraPeerId>, + start: block::Height, + count: u32, +) { + emit_commit_state( + trace, + cs_trace::STATE_READ_START, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, action); + if let Some(peer) = peer { + insert_cs_peer(row, cs_trace::PEER, peer); + } + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + }, + ); +} + +fn trace_state_read_error( + trace: &ZakuraTrace, + action: &'static str, + peer: Option<&zebra_network::zakura::ZakuraPeerId>, + start: block::Height, + count: u32, + reason: &str, + started: Instant, +) { + emit_commit_state( + trace, + cs_trace::STATE_READ_ERROR, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, action); + if let Some(peer) = peer { + insert_cs_peer(row, cs_trace::PEER, peer); + } + insert_cs_height(row, cs_trace::RANGE_START, start); + insert_cs_u64(row, cs_trace::RANGE_COUNT, u64::from(count)); + insert_cs_str(row, cs_trace::REASON, reason); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); +} + +fn commit_failure_result_label(kind: HeaderSyncCommitFailureKind) -> &'static str { + match kind { + HeaderSyncCommitFailureKind::InvalidPeerRange => "invalid_peer_range", + HeaderSyncCommitFailureKind::Local => "local_error", + } +} + +fn header_misbehavior_label(reason: zebra_network::zakura::HeaderSyncMisbehavior) -> &'static str { + match reason { + zebra_network::zakura::HeaderSyncMisbehavior::InvalidStatus => "invalid_status", + zebra_network::zakura::HeaderSyncMisbehavior::UnsolicitedHeaders => "unsolicited_headers", + zebra_network::zakura::HeaderSyncMisbehavior::EmptyHeaders => "empty_headers", + zebra_network::zakura::HeaderSyncMisbehavior::ResponseTooLong => "response_too_long", + zebra_network::zakura::HeaderSyncMisbehavior::InvalidRange => "invalid_range", + zebra_network::zakura::HeaderSyncMisbehavior::MalformedMessage => "malformed_message", + zebra_network::zakura::HeaderSyncMisbehavior::StatusSpam => "status_spam", + zebra_network::zakura::HeaderSyncMisbehavior::NewBlockSpam => "new_block_spam", + zebra_network::zakura::HeaderSyncMisbehavior::GetHeadersSpam => "get_headers_spam", + zebra_network::zakura::HeaderSyncMisbehavior::GetHeadersTooLong => "get_headers_too_long", + zebra_network::zakura::HeaderSyncMisbehavior::UnknownPeer => "unknown_peer", + zebra_network::zakura::HeaderSyncMisbehavior::InvalidNewBlock => "invalid_new_block", + } +} + +fn tip_action_label(action: &zebra_state::TipAction) -> &'static str { + match action { + zebra_state::TipAction::Grow { .. } => "grow", + zebra_state::TipAction::Reset { .. } => "reset", + } +} + +fn elapsed_ms(started: Instant) -> u64 { + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) +} diff --git a/zebrad/src/commands/start/zakura/mod.rs b/zebrad/src/commands/start/zakura/mod.rs new file mode 100644 index 00000000000..9f7241875e3 --- /dev/null +++ b/zebrad/src/commands/start/zakura/mod.rs @@ -0,0 +1,130 @@ +use std::time::Duration; + +use serde_json::{Map, Number, Value}; +use zebra_chain::block; +use zebra_network::zakura::{ + commit_state_trace as cs_trace, zakura_trace_peer_label, BlockApplyResult, ZakuraPeerId, + ZakuraTrace, COMMIT_STATE_TABLE, +}; + +pub(crate) mod block_sync_driver; +pub(crate) mod frontier; +pub(crate) mod header_sync_driver; + +pub(crate) use block_sync_driver::drive_block_sync_actions; +#[cfg(test)] +pub(crate) use block_sync_driver::{ + apply_block_sync_body, block_apply_class, block_sync_misbehavior_is_hard, + block_sync_missing_body_window, block_sync_needed_blocks_from_state, + coalesce_stale_needed_block_queries, commit_block_sync_body, BlockApplyClass, + ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL, +}; +pub(crate) use frontier::{query_block_sync_frontiers, verified_block_tip_from_state}; +#[cfg(test)] +pub(crate) use header_sync_driver::{ + block_sync_chain_tip_event, body_sizes_for_served_header_range, + header_range_commit_failure_kind, notify_block_sync_header_tip, +}; +pub(crate) use header_sync_driver::{ + drive_zakura_header_sync_actions, mirror_zakura_full_block_commits, + zakura_header_sync_driver_startup, ZakuraHeaderSyncDriverHandles, +}; + +pub(crate) const ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT: Duration = Duration::from_secs(30); + +pub(crate) fn emit_commit_state( + trace: &ZakuraTrace, + event: &'static str, + source: &'static str, + build: impl FnOnce(&mut Map), +) { + trace.emit_with(COMMIT_STATE_TABLE, |row| { + row.insert( + cs_trace::EVENT.to_string(), + Value::String(event.to_string()), + ); + insert_cs_str(row, cs_trace::SOURCE, source); + build(row); + }); +} + +pub(crate) fn insert_cs_height( + row: &mut Map, + key: &'static str, + height: block::Height, +) { + insert_cs_u64(row, key, u64::from(height.0)); +} + +pub(crate) fn insert_cs_hash(row: &mut Map, key: &'static str, hash: block::Hash) { + row.insert(key.to_string(), Value::String(format!("{hash}"))); +} + +pub(crate) fn insert_cs_peer(row: &mut Map, key: &'static str, peer: &ZakuraPeerId) { + row.insert( + key.to_string(), + Value::String(zakura_trace_peer_label(peer)), + ); +} + +pub(crate) fn insert_cs_u64(row: &mut Map, key: &'static str, value: u64) { + row.insert(key.to_string(), Value::Number(Number::from(value))); +} + +pub(crate) fn insert_cs_bool(row: &mut Map, key: &'static str, value: bool) { + row.insert(key.to_string(), Value::Bool(value)); +} + +pub(crate) fn insert_cs_str(row: &mut Map, key: &'static str, value: &str) { + row.insert(key.to_string(), Value::String(value.to_string())); +} + +pub(crate) fn insert_cs_frontiers( + row: &mut Map, + frontiers: &zebra_network::zakura::BlockSyncFrontiers, +) { + insert_cs_height(row, cs_trace::FINALIZED_HEIGHT, frontiers.finalized_height); + insert_cs_height( + row, + cs_trace::VERIFIED_BLOCK_TIP, + frontiers.verified_block_tip, + ); + insert_cs_hash( + row, + cs_trace::VERIFIED_BLOCK_HASH, + frontiers.verified_block_hash, + ); +} + +pub(crate) fn block_apply_result_label(result: BlockApplyResult) -> &'static str { + match result { + BlockApplyResult::Committed => "committed", + BlockApplyResult::Duplicate => "duplicate", + BlockApplyResult::Rejected => "rejected", + BlockApplyResult::TimedOut => "timed_out", + } +} + +pub(crate) fn block_verify_error_is_duplicate(error: &Error) -> bool +where + Error: std::fmt::Debug + Send + Sync + 'static, +{ + let error = error as &dyn std::any::Any; + + error + .downcast_ref::() + .is_some_and(zebra_consensus::RouterError::is_duplicate_request) + || error + .downcast_ref::() + .is_some_and(zebra_consensus::VerifyBlockError::is_duplicate_request) + || error + .downcast_ref::() + .is_some_and(|error| { + error + .downcast_ref::() + .is_some_and(zebra_consensus::RouterError::is_duplicate_request) + || error + .downcast_ref::() + .is_some_and(zebra_consensus::VerifyBlockError::is_duplicate_request) + }) +} diff --git a/zebrad/src/components/mempool/tests/vector.rs b/zebrad/src/components/mempool/tests/vector.rs index 0555a919749..76b359fb877 100644 --- a/zebrad/src/components/mempool/tests/vector.rs +++ b/zebrad/src/components/mempool/tests/vector.rs @@ -613,16 +613,29 @@ async fn mempool_cancel_mined() -> Result<(), Report> { #[tokio::test(flavor = "multi_thread")] async fn mempool_cancel_downloads_after_network_upgrade() -> Result<(), Report> { - let block1: Arc = zebra_test::vectors::BLOCK_MAINNET_1_BYTES - .zcash_deserialize_into() - .unwrap(); - let block2: Arc = zebra_test::vectors::BLOCK_MAINNET_2_BYTES - .zcash_deserialize_into() - .unwrap(); - - // Using the mainnet for now - let network = Network::Mainnet; + // Use a configured Testnet that activates NU5 at `NU5_ACTIVATION_TEST_HEIGHT`, so a real + // network-upgrade `TipAction::Reset` is reachable from generated blocks. Under the + // `next_height` reset semantics, committing the block at `NU5_ACTIVATION_TEST_HEIGHT - 1` + // produces a `Reset` (since the next height is the NU5 activation height). + let network = nu_activation_test_network(); + + // Generate enough blocks to commit a chain that reaches the NU5 reset boundary, plus a spare + // block whose coinbase transaction we queue for download (it is never committed, so it is not + // in the best chain and a network download is attempted). + let blocks = generate_test_chain(&network, NU5_ACTIVATION_TEST_HEIGHT as usize + 2); + let reset_block = blocks[NU5_ACTIVATION_TEST_HEIGHT as usize - 1].clone(); + assert_eq!( + reset_block.coinbase_height(), + Some(Height(NU5_ACTIVATION_TEST_HEIGHT - 1)), + "reset block should be one below the NU5 activation height" + ); + let uncommitted_tx_id = blocks + .last() + .expect("generated chain is non-empty") + .transactions[0] + .unmined_id(); + // Don't commit the mainnet genesis block; we commit the generated chain instead. let ( mut mempool, mut peer_set, @@ -631,19 +644,29 @@ async fn mempool_cancel_downloads_after_network_upgrade() -> Result<(), Report> _tx_verifier, mut recent_syncs, _mempool_transaction_receiver, - ) = setup(&network, u64::MAX, true).await; + ) = setup(&network, u64::MAX, false).await; + + // Commit the chain up to (but not including) the NU5 reset boundary. These commits are the + // genesis reset followed by plain `Grow`s, leaving the tip at `NU5_ACTIVATION_TEST_HEIGHT - 2`. + for block in &blocks[..NU5_ACTIVATION_TEST_HEIGHT as usize - 1] { + commit_block_and_wait_for_tip_change( + &mut state_service, + &mut chain_tip_change, + block.clone(), + ) + .await; + } - // Enable the mempool + // Enable the mempool now that the tip is past genesis and stable. mempool.enable(&mut recent_syncs).await; assert!(mempool.is_enabled()); - // Queue transaction from block 2 for download - let txid = block2.transactions[0].unmined_id(); + // Queue the uncommitted transaction for download. let response = mempool .ready() .await .unwrap() - .call(Request::Queue(vec![txid.into()])) + .call(Request::Queue(vec![uncommitted_tx_id.into()])) .await .unwrap(); let queued_responses = match response { @@ -657,32 +680,10 @@ async fn mempool_cancel_downloads_after_network_upgrade() -> Result<(), Report> // Query the mempool to make it poll chain_tip_change mempool.dummy_call().await; - // Push block 1 to the state. This is considered a network upgrade, + // Commit the NU5 activation block. This is a network upgrade reset (`TipAction::Reset`), // and thus must cancel all pending transaction downloads. - state_service - .ready() - .await - .unwrap() - .call(zebra_state::Request::CommitCheckpointVerifiedBlock( - block1.clone().into(), - )) - .await - .unwrap(); - - // Wait for the chain tip update - if let Err(timeout_error) = timeout( - CHAIN_TIP_UPDATE_WAIT_LIMIT, - chain_tip_change.wait_for_tip_change(), - ) - .await - .map(|change_result| change_result.expect("unexpected chain tip update failure")) - { - info!( - timeout = ?humantime_seconds(CHAIN_TIP_UPDATE_WAIT_LIMIT), - ?timeout_error, - "timeout waiting for chain tip change after committing block" - ); - } + commit_block_and_wait_for_tip_change(&mut state_service, &mut chain_tip_change, reset_block) + .await; // Ignore all the previous network requests. while let Some(_request) = peer_set.try_next_request().await {} @@ -698,7 +699,7 @@ async fn mempool_cancel_downloads_after_network_upgrade() -> Result<(), Report> assert_eq!( request.request(), - &zebra_network::Request::TransactionsById(iter::once(txid).collect()), + &zebra_network::Request::TransactionsById(iter::once(uncommitted_tx_id).collect()), ); assert_eq!(mempool.tx_downloads().in_flight(), 1); @@ -881,18 +882,43 @@ async fn mempool_failed_download_is_not_rejected() -> Result<(), Report> { /// during verification. #[tokio::test(flavor = "multi_thread")] async fn mempool_reverifies_after_tip_change() -> Result<(), Report> { - let network = Network::Mainnet; + // Use a configured Testnet that activates NU5 at `NU5_ACTIVATION_TEST_HEIGHT`, so a real + // network-upgrade `TipAction::Reset` is reachable from generated blocks (see + // `nu_activation_test_network`). + let network = nu_activation_test_network(); + + // Generate enough blocks to commit a chain across the NU5 reset boundary and one block past it, + // plus a spare block whose coinbase transaction we queue for download (it is never committed, + // so it is not in the best chain and a network download is attempted). + let blocks = generate_test_chain(&network, NU5_ACTIVATION_TEST_HEIGHT as usize + 2); + // Committing this block produces a network-upgrade `TipAction::Reset` (its next height is the + // NU5 activation height). + let reset_block = blocks[NU5_ACTIVATION_TEST_HEIGHT as usize - 1].clone(); + assert_eq!( + reset_block.coinbase_height(), + Some(Height(NU5_ACTIVATION_TEST_HEIGHT - 1)), + "reset block should be one below the NU5 activation height" + ); + // Committing this block (the NU5 activation block) is a plain `TipAction::Grow` that increases + // the tip height past the height the queued tx was verified at. + let grow_block = blocks[NU5_ACTIVATION_TEST_HEIGHT as usize].clone(); + assert_eq!( + grow_block.coinbase_height(), + Some(Height(NU5_ACTIVATION_TEST_HEIGHT)), + "grow block should be at the NU5 activation height" + ); - let block1: Arc = zebra_test::vectors::BLOCK_MAINNET_1_BYTES - .zcash_deserialize_into() - .unwrap(); - let block2: Arc = zebra_test::vectors::BLOCK_MAINNET_2_BYTES - .zcash_deserialize_into() - .unwrap(); - let block3: Arc = zebra_test::vectors::BLOCK_MAINNET_3_BYTES - .zcash_deserialize_into() - .unwrap(); + // The queued transaction is the coinbase of a generated block that is never committed, so it is + // not in the best chain and a network download is attempted. All verification is mocked, so the + // transaction contents don't matter. + let tx = blocks + .last() + .expect("generated chain is non-empty") + .transactions[0] + .clone(); + let txid = tx.unmined_id(); + // Don't commit the mainnet genesis block; we commit the generated chain instead. let ( mut mempool, mut peer_set, @@ -901,15 +927,24 @@ async fn mempool_reverifies_after_tip_change() -> Result<(), Report> { mut tx_verifier, mut recent_syncs, _mempool_transaction_receiver, - ) = setup(&network, u64::MAX, true).await; + ) = setup(&network, u64::MAX, false).await; + + // Commit the chain up to (but not including) the NU5 reset boundary. These commits are the + // genesis reset followed by plain `Grow`s, leaving the tip at `NU5_ACTIVATION_TEST_HEIGHT - 2`. + for block in &blocks[..NU5_ACTIVATION_TEST_HEIGHT as usize - 1] { + commit_block_and_wait_for_tip_change( + &mut state_service, + &mut chain_tip_change, + block.clone(), + ) + .await; + } - // Enable the mempool + // Enable the mempool now that the tip is past genesis and stable. mempool.enable(&mut recent_syncs).await; assert!(mempool.is_enabled()); - // Queue transaction from block 3 for download - let tx = block3.transactions[0].clone(); - let txid = block3.transactions[0].unmined_id(); + // Queue the transaction for download let response = mempool .ready() .await @@ -959,24 +994,14 @@ async fn mempool_reverifies_after_tip_change() -> Result<(), Report> { }) .await; - // Push block 1 to the state. This is considered a network upgrade, - // and must cancel all pending transaction downloads with a `TipAction::Reset`. - state_service - .ready() - .await - .unwrap() - .call(zebra_state::Request::CommitCheckpointVerifiedBlock( - block1.clone().into(), - )) - .await - .unwrap(); - - // Wait for the chain tip update without a timeout - // (skipping the chain tip change here will fail the test) - chain_tip_change - .wait_for_tip_change() - .await - .expect("unexpected chain tip update failure"); + // Commit the NU5 activation block. This is a network upgrade reset (`TipAction::Reset`), + // and must cancel all pending transaction downloads and re-queue them for verification. + commit_block_and_wait_for_tip_change( + &mut state_service, + &mut chain_tip_change, + reset_block.clone(), + ) + .await; // Query the mempool to make it poll chain_tip_change and try reverifying its state for the `TipAction::Reset` mempool.dummy_call().await; @@ -1021,24 +1046,10 @@ async fn mempool_reverifies_after_tip_change() -> Result<(), Report> { }) .await; - // Push block 2 to the state. This will increase the tip height past the expected - // tip height that the tx was verified at. - state_service - .ready() - .await - .unwrap() - .call(zebra_state::Request::CommitCheckpointVerifiedBlock( - block2.clone().into(), - )) - .await - .unwrap(); - - // Wait for the chain tip update without a timeout - // (skipping the chain tip change here will fail the test) - chain_tip_change - .wait_for_tip_change() - .await - .expect("unexpected chain tip update failure"); + // Commit the next block (a plain `Grow`). This increases the tip height past the expected + // tip height that the tx was verified at, so the mempool must re-verify it. + commit_block_and_wait_for_tip_change(&mut state_service, &mut chain_tip_change, grow_block) + .await; // Query the mempool to make it poll tx_downloads.pending and try reverifying transactions // because the tip height has changed. @@ -1795,6 +1806,96 @@ fn pick_transaction_with_prevout(network: &Network) -> VerifiedUnminedTx { .expect("missing non-coinbase transaction") } +/// The height at which the network returned by [`nu_activation_test_network`] activates NU5. +/// +/// All earlier network upgrades activate at height 1, so the only non-genesis network upgrade +/// reset boundary is at NU5. Because [`ChainTipChange::action`] resets when the *next* height +/// (`tip + 1`) is an activation height, committing the block at height +/// `NU5_ACTIVATION_TEST_HEIGHT - 1` produces a [`TipAction::Reset`]. +const NU5_ACTIVATION_TEST_HEIGHT: u32 = 4; + +/// Build a configured Testnet that activates NU5 at [`NU5_ACTIVATION_TEST_HEIGHT`]. +/// +/// This makes a network-upgrade reset reachable from generated blocks: under the +/// `next_height` reset semantics in [`ChainTipChange::action`], committing the block at height +/// `NU5_ACTIVATION_TEST_HEIGHT - 1` yields a [`TipAction::Reset`], because the next height is the +/// NU5 activation height. The temporary Orchard-disabling soft fork is disabled so the only +/// non-genesis reset boundary is the NU5 activation. +fn nu_activation_test_network() -> Network { + use zebra_chain::parameters::testnet::{ + ConfiguredActivationHeights, Parameters as TestnetParameters, + }; + + TestnetParameters::build() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(NU5_ACTIVATION_TEST_HEIGHT), + ..Default::default() + }) + .expect("configured activation heights are valid") + .disable_temporary_orchard_disabling_soft_fork() + .extend_funding_streams() + .to_network() + .expect("configured network is valid") +} + +/// Generate a chain of `count` valid blocks (starting at the genesis block) for `network`. +/// +/// The blocks have valid commitments, so they can be committed to a state service via +/// [`zebra_state::Request::CommitCheckpointVerifiedBlock`]. +fn generate_test_chain(network: &Network, count: usize) -> Vec> { + use proptest::{ + strategy::{Strategy, ValueTree}, + test_runner::TestRunner, + }; + use zebra_chain::block::{arbitrary::allow_all_transparent_coinbase_spends, LedgerState}; + + let strategy = LedgerState::genesis_strategy(Some(network.clone()), None, None, false) + .prop_flat_map(move |init| { + Block::partial_chain_strategy( + init, + count, + allow_all_transparent_coinbase_spends, + // Generate valid commitments so the blocks pass the checkpoint commit checks. + true, + ) + }); + + let mut runner = TestRunner::deterministic(); + strategy + .new_tree(&mut runner) + .expect("partial chain strategy should produce a value tree") + .current() + .0 +} + +/// Commit a checkpoint-verified `block` to `state_service` and wait for the chain tip change. +async fn commit_block_and_wait_for_tip_change( + state_service: &mut StateService, + chain_tip_change: &mut ChainTipChange, + block: Arc, +) { + state_service + .ready() + .await + .unwrap() + .call(zebra_state::Request::CommitCheckpointVerifiedBlock( + block.into(), + )) + .await + .expect("unexpected block commit failure"); + + chain_tip_change + .wait_for_tip_change() + .await + .expect("unexpected chain tip update failure"); +} + /// Create a new [`Mempool`] instance using mocked services. async fn setup( network: &Network, diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index cf5dc9b1be0..c015ba0bda9 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -278,6 +278,25 @@ pub struct Config { /// If the number of logical cores can't be detected, Zebra uses one thread. /// For details, see [the `rayon` documentation](https://docs.rs/rayon/latest/rayon/struct.ThreadPoolBuilder.html#method.num_threads). pub parallel_cpu_threads: usize, + + /// Skip the Regtest direct genesis self-seed at startup, forcing this node to + /// obtain the genesis block from peers like a Mainnet/Testnet node. + /// + /// On Regtest, zebrad normally commits the genesis block directly at startup + /// (the genesis block is a known constant and there may be no peer to download + /// it from). When this is `true`, that shortcut is skipped and genesis must be + /// downloaded and verified from a peer instead. This exercises the production + /// genesis-bootstrap path — in particular a Zakura-only node + /// (`legacy_p2p = false`) that must fetch genesis over Zakura before native + /// header/body sync can advance from an empty state. + /// + /// Defaults to `false`, so standalone Regtest nodes keep self-seeding genesis. + /// + /// Skipped when serializing so `zebrad generate` output (and the stored config + /// compatibility snapshot) stays stable; it is only ever set by hand in + /// test/bootstrap configs. + #[serde(skip_serializing)] + pub debug_skip_regtest_genesis_self_seed: bool, } impl Default for Config { @@ -306,6 +325,10 @@ impl Default for Config { // If this causes tokio executor starvation, move CPU-intensive tasks to rayon threads, // or reserve a few cores for tokio threads, based on `num_cpus()`. parallel_cpu_threads: 0, + + // Standalone Regtest nodes self-seed genesis; only opt-in test/bootstrap + // setups download it from peers. + debug_skip_regtest_genesis_self_seed: false, } } } @@ -570,6 +593,20 @@ where } } + /// Downloads and verifies genesis, then parks this legacy syncer. + /// + /// Zakura block sync uses this bootstrap path because header range validation needs the + /// committed genesis header before native Zakura header/body sync can advance from scratch. + #[instrument(skip(self))] + pub async fn bootstrap_genesis_then_pause(mut self) -> Result<(), Report> { + self.request_genesis().await?; + info!( + "Zakura block sync replacement completed genesis bootstrap; parking legacy ChainSync" + ); + std::future::pending::<()>().await; + Ok(()) + } + /// Tries to synchronize the chain as far as it can. /// /// Obtains some prospective tips and iteratively tries to extend them and download the missing @@ -1044,6 +1081,14 @@ where Ok(Err(fatal_error)) => Err(fatal_error)?, // Handle timeouts and block errors Err(error) | Ok(Ok(Err(error))) => { + if self.is_duplicate_finalized_genesis_error(&error) { + info!( + ?error, + "genesis block is already finalized, continuing sync" + ); + return Ok(()); + } + // TODO: exit syncer on permanent service errors (NetworkError, VerifierError) if Self::should_restart_sync(&error) { warn!( @@ -1065,6 +1110,26 @@ where Ok(()) } + fn is_duplicate_finalized_genesis_error(&self, error: &BlockDownloadVerifyError) -> bool { + match error { + BlockDownloadVerifyError::Invalid { + error, + height, + hash, + .. + } => { + *height == Height(0) + && *hash == self.genesis_hash + && Self::is_duplicate_finalized_error(error) + } + _ => false, + } + } + + fn is_duplicate_finalized_error(error: &zebra_consensus::RouterError) -> bool { + error.duplicate_location() == Some(&zs::KnownBlock::Finalized) + } + /// Try to download and verify the genesis block once. /// /// Fatal errors are returned in the outer result, temporary errors in the inner one. @@ -1316,6 +1381,15 @@ where debug!(error = ?e, "block was already verified or committed, possibly from a previous sync run, continuing"); false } + BlockDownloadVerifyError::Invalid { error, .. } + if Self::is_duplicate_finalized_error(error) => + { + debug!( + error = ?e, + "block was already finalized, possibly from a previous sync run, continuing" + ); + false + } // Structural matches: direct BlockDownloadVerifyError::CancelledDuringDownload { .. } diff --git a/zebrad/src/components/sync/downloads.rs b/zebrad/src/components/sync/downloads.rs index 35b583e46c1..1093a82d06c 100644 --- a/zebrad/src/components/sync/downloads.rs +++ b/zebrad/src/components/sync/downloads.rs @@ -405,17 +405,28 @@ where }; let (block, advertiser_addr) = if let zn::Response::Blocks(blocks) = rsp { - assert_eq!( - blocks.len(), - 1, - "wrong number of blocks in response to a single hash" - ); - - blocks - .first() - .expect("just checked length") - .available() - .expect("unexpected missing block status: single block failures should be errors") + // A cooperating peer returns exactly one available block for a + // single-hash request. A response with a different count, or a + // `Missing` status, means the peer is misbehaving or raced us + // (e.g. the block was reorged away between advertisement and + // fetch). Treat it as a retryable download failure rather than + // asserting and bringing the whole node down. + match blocks.first().and_then(|block| block.available()) { + Some(available) if blocks.len() == 1 => available, + _ => { + metrics::histogram!("sync.block.download.duration_seconds", "result" => "failed") + .record(download_start.elapsed().as_secs_f64()); + return Err(BlockDownloadVerifyError::DownloadFailed { + error: format!( + "expected one available block in response to a \ + single-hash request, got {}", + blocks.len() + ) + .into(), + hash, + }); + } + } } else { unreachable!("wrong response to block request"); }; diff --git a/zebrad/src/components/sync/tests/timing.rs b/zebrad/src/components/sync/tests/timing.rs index 2e4e392def6..9bf82d58498 100644 --- a/zebrad/src/components/sync/tests/timing.rs +++ b/zebrad/src/components/sync/tests/timing.rs @@ -14,15 +14,13 @@ use zebra_chain::{ block::Height, parameters::{Network, POST_BLOSSOM_POW_TARGET_SPACING}, }; -use zebra_network::constants::{ - DEFAULT_CRAWL_NEW_PEER_INTERVAL, HANDSHAKE_TIMEOUT, INVENTORY_ROTATION_INTERVAL, -}; +use zebra_network::constants::HANDSHAKE_TIMEOUT; use zebra_state::ChainTipSender; use crate::{ components::sync::{ ChainSync, BLOCK_DOWNLOAD_RETRY_LIMIT, BLOCK_DOWNLOAD_TIMEOUT, BLOCK_VERIFY_TIMEOUT, - GENESIS_TIMEOUT_RETRY, SYNC_RESTART_DELAY, + GENESIS_TIMEOUT_RETRY, SYNC_RESTART_DELAY, SYNC_RESTART_SLEEP, }, config::ZebradConfig, }; @@ -32,10 +30,15 @@ use crate::{ fn ensure_timeouts_consistent() { let _init_guard = zebra_test::init(); - // This constraint clears the download pipeline during a restart + // This fork deliberately decouples the post-round idle (`SYNC_RESTART_SLEEP`) from the + // per-operation restart timeout (`SYNC_RESTART_DELAY`), and keeps both short to recover + // quickly on thin or flaky peer sets. It accepts that sync may re-enter while cancelled + // downloads from the previous round are still draining, so the upstream invariant + // `SYNC_RESTART_DELAY > 2 * BLOCK_DOWNLOAD_TIMEOUT` no longer applies. We instead require + // the post-round idle to stay shorter than the restart-operation timeout. assert!( - SYNC_RESTART_DELAY.as_secs() > 2 * BLOCK_DOWNLOAD_TIMEOUT.as_secs(), - "Sync restart should allow for pending and buffered requests to complete" + SYNC_RESTART_SLEEP.as_secs() < SYNC_RESTART_DELAY.as_secs(), + "the post-round sync sleep should be shorter than the sync restart timeout" ); // We multiply by 2, because the Hedge can wait up to BLOCK_DOWNLOAD_TIMEOUT @@ -82,28 +85,15 @@ fn ensure_timeouts_consistent() { "a syncer tip crawl should complete before most new blocks" ); - // This is a compromise between two failure modes: - // - some peers have the inventory, but they weren't ready last time we checked, - // so we want to retry soon - // - all peers are missing the inventory, so we want to wait for a while before retrying - assert!( - INVENTORY_ROTATION_INTERVAL < SYNC_RESTART_DELAY, - "we should expire some inventory every time the syncer resets" - ); - assert!( - SYNC_RESTART_DELAY < 2 * INVENTORY_ROTATION_INTERVAL, - "we should give the syncer at least one retry attempt, \ - before we expire all inventory" - ); - - // The default peer crawler interval should be at least - // `HANDSHAKE_TIMEOUT` lower than all other crawler intervals. - // - // See `DEFAULT_CRAWL_NEW_PEER_INTERVAL` for details. + // `SYNC_RESTART_DELAY` now bounds restart operations (obtaining tips and retrying the + // genesis block) rather than the inter-round cadence, so it must be long enough for peers + // to respond to a tip request — at least one handshake. The upstream invariants that + // compared the restart delay against the inventory-rotation and peer-crawl intervals + // assumed it was the much longer inter-round delay; this fork's short restart timeout + // intentionally no longer satisfies them (the inter-round idle is `SYNC_RESTART_SLEEP`). assert!( - DEFAULT_CRAWL_NEW_PEER_INTERVAL.as_secs() + HANDSHAKE_TIMEOUT.as_secs() - < SYNC_RESTART_DELAY.as_secs(), - "an address crawl and peer connections should complete before most syncer tips crawls" + SYNC_RESTART_DELAY.as_secs() > HANDSHAKE_TIMEOUT.as_secs(), + "a sync restart should allow at least one peer handshake to complete" ); } diff --git a/zebrad/src/components/sync/tests/vectors.rs b/zebrad/src/components/sync/tests/vectors.rs index df3e0585e41..86592990d74 100644 --- a/zebrad/src/components/sync/tests/vectors.rs +++ b/zebrad/src/components/sync/tests/vectors.rs @@ -2,17 +2,27 @@ #![allow(clippy::unwrap_in_result)] -use std::{collections::HashMap, iter, sync::Arc, time::Duration}; +use std::{ + collections::HashMap, + iter, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; use color_eyre::Report; -use futures::{Future, FutureExt}; +use futures::{Future, FutureExt, StreamExt}; use zebra_chain::{ block::{self, Block, Height}, chain_tip::mock::{MockChainTip, MockChainTipSender}, serialization::ZcashDeserializeInto, }; -use zebra_consensus::{Config as ConsensusConfig, RouterError, VerifyBlockError}; +use zebra_consensus::{ + Config as ConsensusConfig, RouterError, VerifyBlockError, VerifyCheckpointError, +}; use zebra_network::{InventoryResponse, PeerSocketAddr}; use zebra_state::Config as StateConfig; use zebra_test::mock_service::{MockService, PanicAssertion}; @@ -22,7 +32,11 @@ use zebra_state as zs; use crate::{ components::{ - sync::{self, downloads::BlockDownloadVerifyError, SyncStatus}, + sync::{ + self, + downloads::{BlockDownloadVerifyError, Downloads}, + SyncStatus, + }, ChainSync, }, config::ZebradConfig, @@ -1027,6 +1041,132 @@ async fn should_restart_sync_returns_false() { ); } +/// A scratch state can have finalized genesis tip metadata before +/// `KnownBlock(genesis)` can find a block body. In that state, committing the +/// downloaded genesis block returns duplicate/finalized; the genesis bootstrap +/// loop must treat that as success instead of retrying forever. +#[tokio::test] +async fn request_genesis_accepts_duplicate_finalized_genesis() -> Result<(), crate::BoxError> { + let block0: Arc = + zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES.zcash_deserialize_into()?; + let block0_hash = block0.hash(); + + let state_requests = Arc::new(AtomicUsize::new(0)); + let state_requests_in_service = Arc::clone(&state_requests); + let state_service = tower::service_fn(move |request| { + state_requests_in_service.fetch_add(1, Ordering::SeqCst); + async move { + assert_eq!(request, zs::Request::KnownBlock(block0_hash)); + Ok::<_, crate::BoxError>(zs::Response::KnownBlock(None)) + } + }); + + let peer_requests = Arc::new(AtomicUsize::new(0)); + let peer_requests_in_service = Arc::clone(&peer_requests); + let peer_block = block0.clone(); + let peer_set = tower::service_fn(move |request| { + peer_requests_in_service.fetch_add(1, Ordering::SeqCst); + let peer_block = peer_block.clone(); + async move { + assert_eq!( + request, + zn::Request::BlocksByHash(iter::once(block0_hash).collect()) + ); + Ok::<_, crate::BoxError>(zn::Response::Blocks(vec![Available((peer_block, None))])) + } + }); + + let verifier_requests = Arc::new(AtomicUsize::new(0)); + let verifier_requests_in_service = Arc::clone(&verifier_requests); + let verifier_service = tower::service_fn(move |request| { + verifier_requests_in_service.fetch_add(1, Ordering::SeqCst); + async move { + let zebra_consensus::Request::Commit(block) = request else { + unreachable!("no other verifier request is allowed") + }; + assert_eq!(block.hash(), block0_hash); + + let duplicate = zs::CommitBlockError::Duplicate { + hash_or_height: None, + location: zs::KnownBlock::Finalized, + }; + let duplicate = zs::CommitCheckpointVerifiedError::from(duplicate); + let router_error = RouterError::Checkpoint { + source: Box::new(VerifyCheckpointError::CommitCheckpointVerified(Box::new( + duplicate, + ))), + }; + + Err::(Box::new(router_error) as crate::BoxError) + } + }); + + let consensus_config = ConsensusConfig::default(); + let state_config = StateConfig::ephemeral(); + let config = ZebradConfig { + consensus: consensus_config, + state: state_config, + ..Default::default() + }; + let (mock_chain_tip, _mock_chain_tip_sender) = MockChainTip::new(); + let (misbehavior_tx, _misbehavior_rx) = tokio::sync::mpsc::channel(1); + let (mut chain_sync, _sync_status) = ChainSync::new( + &config, + Height(0), + peer_set, + verifier_service, + state_service, + mock_chain_tip, + misbehavior_tx, + ); + + tokio::time::timeout(Duration::from_secs(2), chain_sync.request_genesis()) + .await + .expect("duplicate finalized genesis should not sleep and retry") + .expect("duplicate finalized genesis is accepted"); + + assert_eq!(state_requests.load(Ordering::SeqCst), 1); + assert_eq!(peer_requests.load(Ordering::SeqCst), 1); + assert_eq!(verifier_requests.load(Ordering::SeqCst), 1); + + Ok(()) +} + +/// In-flight checkpoint downloads can finish after a later contiguous range has +/// already reached finalized state. Those duplicate/finalized responses are +/// stale work, not a reason to restart the whole sync loop. +#[test] +fn duplicate_finalized_checkpoint_block_does_not_restart_sync() -> Result<(), crate::BoxError> { + let block1: Arc = zebra_test::vectors::BLOCK_MAINNET_1_BYTES.zcash_deserialize_into()?; + let block1_hash = block1.hash(); + + let duplicate = zs::CommitBlockError::Duplicate { + hash_or_height: None, + location: zs::KnownBlock::Finalized, + }; + let duplicate = zs::CommitCheckpointVerifiedError::from(duplicate); + let router_error = RouterError::Checkpoint { + source: Box::new(VerifyCheckpointError::CommitCheckpointVerified(Box::new( + duplicate, + ))), + }; + let err = BlockDownloadVerifyError::Invalid { + error: router_error, + height: Height(1), + hash: block1_hash, + advertiser_addr: None, + }; + + let restart = TestChainSync::should_restart_sync(&err); + + assert!( + !restart, + "duplicate finalized checkpoint blocks are stale in-flight work, not sync restarts" + ); + + Ok(()) +} + /// Verifies fix for GHSA-gvjc-3w7c-92jx: `AboveLookaheadHeightLimit` now has /// an explicit match arm in `should_restart_sync` that returns `false`. #[tokio::test] @@ -1332,3 +1472,80 @@ fn setup_chain_sync() -> ( fn not_found_block_error(_hash: block::Hash) -> crate::BoxError { zn::SharedPeerError::from(zn::PeerError::NotFoundResponse(Vec::new())).into() } + +#[test] +fn debug_skip_regtest_genesis_self_seed_defaults_off_and_is_opt_in() { + use crate::components::sync::Config; + + // Default is off, so standalone Regtest nodes keep self-seeding genesis. + assert!(!Config::default().debug_skip_regtest_genesis_self_seed); + + // Opt-in still parses despite `deny_unknown_fields`. + let config: Config = toml::from_str("debug_skip_regtest_genesis_self_seed = true") + .expect("sync config with the genesis-bootstrap flag parses"); + assert!(config.debug_skip_regtest_genesis_self_seed); + + // Skipped on serialize, so `zebrad generate` output (and the stored-config + // compatibility snapshot) stays stable. + let serialized = toml::to_string(&Config::default()).expect("sync config serializes"); + assert!( + !serialized.contains("debug_skip_regtest_genesis_self_seed"), + "debug bootstrap flag must not appear in generated config output" + ); +} + +/// A peer that returns *zero* blocks for a single-hash download request must be +/// treated as a retryable download failure, not crash the whole node. +/// +/// Regression for a `downloads.rs` `assert_eq!(blocks.len(), 1)` panic: a +/// gossiped single-hash fetch that raced an empty response took down a Zakura +/// node mid catch-up (`thread 'tokio-rt-worker' panicked ... wrong number of +/// blocks in response to a single hash`, propagated to a fatal syncer panic). A +/// misbehaving or racing peer must not be able to kill the node, so an +/// unexpected block count is surfaced as a `DownloadFailed` the syncer retries. +#[tokio::test] +async fn empty_block_response_is_retryable_download_failure() { + let _init_guard = zebra_test::init(); + + let mut peer_set = MockService::build().for_unit_tests::(); + let verifier = + MockService::build().for_unit_tests::(); + let (chain_tip, _chain_tip_sender) = MockChainTip::new(); + let (past_lookahead_limit_sender, _past_lookahead_limit_receiver) = + tokio::sync::watch::channel(false); + + let mut downloads = Downloads::new( + peer_set.clone(), + verifier, + chain_tip, + past_lookahead_limit_sender, + sync::MIN_CONCURRENCY_LIMIT, + Height(0), + ); + + let block0: Arc = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into() + .expect("test vector deserializes"); + let hash = block0.hash(); + + downloads + .download_and_verify(hash) + .await + .expect("queuing a fresh hash succeeds"); + + // The peer responds to the single-hash request with an empty block list. + peer_set + .expect_request(zn::Request::BlocksByHash(iter::once(hash).collect())) + .await + .respond(zn::Response::Blocks(vec![])); + + let result = downloads + .next() + .await + .expect("the download task produces a result instead of panicking"); + + assert!( + matches!(result, Err(BlockDownloadVerifyError::DownloadFailed { .. })), + "an empty block response must be a retryable DownloadFailed, got {result:?}", + ); +} diff --git a/zebrad/tests/common/configs/v4.5.0-zakura-blocksync.toml b/zebrad/tests/common/configs/v4.5.0-zakura-blocksync.toml new file mode 100644 index 00000000000..6df3948a28c --- /dev/null +++ b/zebrad/tests/common/configs/v4.5.0-zakura-blocksync.toml @@ -0,0 +1,143 @@ +# Default configuration for zebrad. +# +# This file can be used as a skeleton for custom configs. +# +# Unspecified fields use default values. Optional fields are Some(field) if the +# field is present and None if it is absent. +# +# This file is generated as an example using zebrad's current defaults. +# You should set only the config options you want to keep, and delete the rest. +# Only a subset of fields are present in the skeleton, since optional values +# whose default is None are omitted. +# +# The config format (including a complete list of sections and fields) is +# documented here: +# https://docs.rs/zebrad/latest/zebrad/config/struct.ZebradConfig.html +# +# CONFIGURATION SOURCES (in order of precedence, highest to lowest): +# +# 1. Environment variables with ZEBRA_ prefix (highest precedence) +# - Format: ZEBRA_SECTION__KEY (double underscore for nested keys) +# - Examples: +# - ZEBRA_NETWORK__NETWORK=Testnet +# - ZEBRA_RPC__LISTEN_ADDR=127.0.0.1:8232 +# - ZEBRA_STATE__CACHE_DIR=/path/to/cache +# - ZEBRA_TRACING__FILTER=debug +# - ZEBRA_METRICS__ENDPOINT_ADDR=0.0.0.0:9999 +# +# 2. Configuration file (TOML format) +# - At the path specified via -c flag, e.g. `zebrad -c myconfig.toml start`, or +# - At the default path in the user's preference directory (platform-dependent, see below) +# +# 3. Hard-coded defaults (lowest precedence) +# +# The user's preference directory and the default path to the `zebrad` config are platform dependent, +# based on `dirs::preference_dir`, see https://docs.rs/dirs/latest/dirs/fn.preference_dir.html : +# +# | Platform | Value | Example | +# | -------- | ------------------------------------- | ---------------------------------------------- | +# | Linux | `$XDG_CONFIG_HOME` or `$HOME/.config` | `/home/alice/.config/zebrad.toml` | +# | macOS | `$HOME/Library/Preferences` | `/Users/Alice/Library/Preferences/zebrad.toml` | +# | Windows | `{FOLDERID_RoamingAppData}` | `C:\Users\Alice\AppData\Local\zebrad.toml` | + +[consensus] +checkpoint_sync = true + +[health] +enforce_on_test_networks = false +min_connected_peers = 1 +ready_max_blocks_behind = 2 +ready_max_tip_age = "5m" + +[mempool] +eviction_memory_time = "1h" +max_datacarrier_bytes = 83 +tx_cost_limit = 80000000 + +[metrics] + +[mining] +internal_miner = false + +[network] +cache_dir = true +crawl_new_peer_interval = "1m 1s" +initial_mainnet_peers = [ + "dnsseed.str4d.xyz:8233", + "dnsseed.z.cash:8233", + "mainnet.seeder.shieldedinfra.net:8233", + "mainnet.seeder.zfnd.org:8233", +] +initial_testnet_peers = [ + "dnsseed.testnet.z.cash:18233", + "testnet.seeder.zfnd.org:18233", +] +legacy_p2p = true +listen_addr = "[::]:8233" +max_connections_per_ip = 1 +network = "Mainnet" +peerset_initial_target_size = 25 +v2_p2p = true + +[network.zakura] +bootstrap_peers = [] +max_connections = 32 +max_pending_handshakes = 8 +message_rate_per_second = 128 +stream_open_rate_per_second = 16 + +[network.zakura.block_sync] +fanout = 3 +max_blocks_per_response = 16 +max_inflight_block_bytes = 268435456 +max_inflight_requests = 4 +max_response_bytes = 33554432 +replace_legacy_syncer = false +request_timeout = "30s" +size_deviation_tolerance = 200 +status_refresh_interval = "30s" + +[network.zakura.block_sync.peer_limits] +inbound_queue_depth = 128 +max_inbound_peers = 256 +max_outbound_peers = 256 +max_pending_escalations = 32 +outbound_queue_depth = 128 + +[network.zakura.header_sync] +max_headers_per_response = 1000 +max_inflight_requests = 10 +status_refresh_interval = "30s" + +[network.zakura.header_sync.peer_limits] +inbound_queue_depth = 128 +max_inbound_peers = 256 +max_outbound_peers = 256 +max_pending_escalations = 32 +outbound_queue_depth = 128 + +[rpc] +cookie_dir = "cache_dir" +debug_force_finished_sync = false +enable_cookie_auth = true +max_response_body_size = 52428800 +parallel_cpu_threads = 0 + +[state] +cache_dir = "cache_dir" +debug_skip_non_finalized_state_backup_task = false +delete_old_database = true +ephemeral = false +should_backup_non_finalized_state = true + +[sync] +checkpoint_verify_concurrency_limit = 1000 +download_concurrency_limit = 50 +full_verify_concurrency_limit = 20 +parallel_cpu_threads = 0 + +[tracing] +buffer_limit = 128000 +force_use_color = false +use_color = true +use_journald = false diff --git a/zebrad/tests/common/zcashd_compat/reorg.rs b/zebrad/tests/common/zcashd_compat/reorg.rs index f94a36aa2b9..ab46cbc8e43 100644 --- a/zebrad/tests/common/zcashd_compat/reorg.rs +++ b/zebrad/tests/common/zcashd_compat/reorg.rs @@ -195,6 +195,7 @@ pub async fn over_batch_branch_restart_recovers() -> Result<()> { wait_for_tips_match(&setup, STANDARD_SYNC_TIMEOUT).await?; force_zebra_reorg(&setup, 8, (SYNC_BATCH_SIZE_LIMIT + 1) as u32).await?; + wait_for_tips_match(&setup, DEEP_REORG_SYNC_TIMEOUT).await?; restart_zcashd_and_wait_for_tips(&setup).await?; diff --git a/zebrad/tests/config.rs b/zebrad/tests/config.rs index 5004154e408..47a6af1f3cb 100644 --- a/zebrad/tests/config.rs +++ b/zebrad/tests/config.rs @@ -9,6 +9,7 @@ use std::{env, fs, io::Write, path::PathBuf, sync::Mutex}; use tempfile::{Builder, TempDir}; +use zebra_network::zakura::DEFAULT_ZAKURA_LISTEN_ADDR; use zebrad::components::zcashd_compat::ConfigZcashdBinarySource; use zebrad::config::ZebradConfig; @@ -79,6 +80,10 @@ fn config_load_defaults() { let config = ZebradConfig::load(None).expect("Should load default config"); assert_eq!(config.network.network.to_string(), "Mainnet"); + assert_eq!( + config.network.zakura.listen_addr, + Some(DEFAULT_ZAKURA_LISTEN_ADDR) + ); assert_eq!(config.rpc.listen_addr, None); // RPC disabled by default assert_eq!(config.metrics.endpoint_addr, None); // Metrics disabled by default } diff --git a/zebrad/tests/zakura_regtest_e2e.rs b/zebrad/tests/zakura_regtest_e2e.rs index 3694d597e4a..73072e2d5c1 100644 --- a/zebrad/tests/zakura_regtest_e2e.rs +++ b/zebrad/tests/zakura_regtest_e2e.rs @@ -1,11 +1,10 @@ //! Opt-in Zakura dual-stack regtest e2e (lightweight, host-networked). //! -//! Ignored by default, and additionally skipped in CI and when Docker is -//! unavailable, so it never gates normal CI — the unit-test lane runs ignored -//! tests (`--run-ignored=all`) on runners that have Docker, but this -//! host-networked docker-compose e2e is environment sensitive and is meant for -//! local validation only. There is no image build: each node runs the -//! host-built `zebrad` binary bind-mounted into stock `debian:trixie-slim`. +//! Ignored by default, and additionally skipped in CI unless +//! `ZAKURA_REGTEST_E2E=1` is set. The unit-test workflow opts this test in as +//! a dedicated required job so failures are isolated from ordinary unit-test +//! output. There is no image build: each node runs the host-built `zebrad` +//! binary bind-mounted into stock `debian:trixie-slim`. //! //! To run it locally: //! @@ -13,7 +12,23 @@ //! cargo test -p zebrad --test zakura_regtest_e2e -- --ignored --nocapture //! ``` //! -//! To force it in a CI environment, set `ZAKURA_REGTEST_E2E=1`. +//! To force it in any CI environment, set `ZAKURA_REGTEST_E2E=1`. +//! +//! `ZAKURA_E2E_MODE` selects the shell harness mode: +//! +//! - `smoke` (default): the existing four-node flow with a short chain, reset +//! catch-up, and non-finalized reorg. +//! - `pr-gate`: a trimmed PR/merge-queue confidence gate that derives a short +//! checkpoint list, forces a from-zero kind-6 catch-up across a checkpoint to +//! full-verifier handoff, runs the trace oracle as a primary assertion layer, +//! and writes a compact `timeline.jsonl` artifact. +//! - `checkpoint-long`: mines a 4,000-block Regtest chain and derives +//! checkpoints every 400 blocks before node2's from-zero catch-up. +//! - `no-checkpoint-long`: mines the same long chain but configures node2 with +//! `checkpoints = false`, leaving only genesis before full verification. +//! - `restart-matrix`: uses the long checkpoint chain and restarts node2 around +//! height 0, 399/400/401, 2,000, near tip with a 1,000-block gap, and after +//! the non-finalized reorg. //! //! It shells out to `docker/zakura-regtest-e2e/run.sh`, which builds `zebrad` //! (debug) if needed, brings up four Regtest nodes sharing the host network — a @@ -21,7 +36,15 @@ //! only via the seed's `zakura.bootstrap_peers`, a legacy-only node, and a //! dual-stack node that upgrades — and asserts legacy TCP backwards //! compatibility, the legacy->Zakura upgrade handshake, and block propagation -//! to the pure-Zakura and legacy-only peers. The upgraded node4 propagation path +//! to the pure-Zakura and legacy-only peers. The pure-Zakura node also disables +//! the Regtest genesis self-seed (`sync.debug_skip_regtest_genesis_self_seed`), +//! so the run additionally proves it bootstraps the genesis block over Zakura +//! from an empty state — the production Mainnet/Testnet bootstrap path that was +//! previously stuck at height 0. After propagation, the pure-Zakura node is reset +//! to an empty state while the seed sits idle at the tip, proving it re-downloads +//! the whole chain over kind-6 block sync (gossip cannot help, since the seed +//! re-advertises nothing) — the production from-0 / restart-catch-up path. The +//! upgraded node4 propagation path //! remains a documented P2 known issue and can be made fatal with //! `ZAKURA_REGTEST_E2E_STRICT_UPGRADE=1`. See that script for the exact //! assertions. @@ -55,7 +78,7 @@ fn zakura_regtest_dual_stack_e2e() { .status() .expect("failed to spawn the Zakura regtest e2e script"); - assert!(status.success(), "Zakura regtest dual-stack e2e failed"); + assert!(status.success(), "Zakura regtest e2e failed"); } fn command_succeeds(command: &mut Command) -> bool {