From 6d4779b275cab5369941051014abafba5df3de78 Mon Sep 17 00:00:00 2001 From: roman Date: Thu, 2 Jul 2026 13:47:40 -0600 Subject: [PATCH] remaining --- .github/workflows/checkpoint-update.yml | 53 + .vscode/settings.json | 17 + CHANGELOG.md | 32 +- Cargo.lock | 20 + Cargo.toml | 1 + Makefile | 7 + deploy/runner/replay_run.sh | 179 ++ docs/design/verified-commitment-trees.md | 30 +- docs/specs/blocksync/congestion_control.md | 183 ++ make/perf.mk | 58 + zebra-consensus/src/block.rs | 10 +- zebra-consensus/src/block/request.rs | 73 +- zebra-consensus/src/block/tests.rs | 65 +- zebra-consensus/src/checkpoint.rs | 321 +-- zebra-consensus/src/config.rs | 95 +- zebra-consensus/src/lib.rs | 3 +- zebra-consensus/src/router.rs | 11 + zebra-consensus/src/router/tests.rs | 2 + zebra-network/src/config/tests/vectors.rs | 18 + zebra-network/src/peer/connection.rs | 38 +- .../src/peer/connection/tests/vectors.rs | 62 +- zebra-network/src/peer/handshake/tests.rs | 6 +- zebra-network/src/peer_set/set.rs | 90 +- .../src/peer_set/set/tests/vectors.rs | 78 + .../src/protocol/internal/request.rs | 36 +- zebra-network/src/zakura/testkit/cluster.rs | 199 +- zebra-replay-bench/Cargo.toml | 42 + zebra-replay-bench/README.md | 164 ++ zebra-replay-bench/RESULTS.md | 283 +++ zebra-replay-bench/src/apply.rs | 202 ++ zebra-replay-bench/src/apply_sequencer.rs | 353 ++++ zebra-replay-bench/src/apply_verifier.rs | 340 ++++ zebra-replay-bench/src/apply_worker.rs | 272 +++ zebra-replay-bench/src/cache.rs | 186 ++ zebra-replay-bench/src/config.rs | 26 + zebra-replay-bench/src/index.rs | 161 ++ zebra-replay-bench/src/main.rs | 296 +++ zebra-replay-bench/src/prefetch.rs | 106 + zebra-replay-bench/src/rollback.rs | 37 + zebra-replay-bench/src/roots_cache.rs | 110 + zebra-replay-bench/src/stats.rs | 56 + zebra-state/CHANGELOG.md | 32 +- zebra-state/src/error.rs | 137 ++ zebra-state/src/lib.rs | 7 +- zebra-state/src/request.rs | 308 ++- zebra-state/src/service.rs | 153 +- zebra-state/src/service/check.rs | 26 +- .../src/service/check/tests/nullifier.rs | 21 +- zebra-state/src/service/check/tests/utxo.rs | 8 +- zebra-state/src/service/finalized_state.rs | 766 ++++++- .../service/finalized_state/commitment_aux.rs | 391 +++- .../finalized_state/commitment_aux_verify.rs | 49 +- .../finalized_state/disk_format/chain.rs | 30 +- .../disk_format/tests/snapshot.rs | 2 +- .../upgrade/cache_genesis_roots.rs | 25 +- .../src/service/finalized_state/tests/prop.rs | 1771 ++++++++++++++++- .../service/finalized_state/tests/rollback.rs | 2 +- .../src/service/finalized_state/vct.rs | 675 +++++++ .../finalized_state/vct/mainnet-frontier.bin | Bin 0 -> 1675 bytes .../src/service/finalized_state/zebra_db.rs | 14 + .../service/finalized_state/zebra_db/block.rs | 35 +- .../zebra_db/block/tests/prune.rs | 48 +- .../zebra_db/block/tests/snapshot.rs | 2 +- .../zebra_db/block/tests/vectors.rs | 1 + .../service/finalized_state/zebra_db/chain.rs | 36 +- .../service/finalized_state/zebra_db/prune.rs | 2 +- .../finalized_state/zebra_db/rollback.rs | 74 + .../finalized_state/zebra_db/shielded.rs | 19 +- .../src/service/non_finalized_state.rs | 1 + zebra-state/src/service/read/block.rs | 8 +- zebra-state/src/service/tests.rs | 118 +- zebra-state/src/service/write.rs | 209 +- zebra-state/src/tests/setup.rs | 2 +- zebra-utils/Cargo.toml | 4 +- zebra-utils/src/bin/zebra-checkpoints/args.rs | 66 +- zebra-utils/src/bin/zebra-checkpoints/main.rs | 75 +- zebrad/src/commands/start.rs | 37 + zebrad/src/components/sync/tests/vectors.rs | 16 +- zebrad/tests/common/cached_state.rs | 4 +- zebrad/tests/common/checkpoints.rs | 9 +- 80 files changed, 8835 insertions(+), 669 deletions(-) create mode 100644 .vscode/settings.json create mode 100755 deploy/runner/replay_run.sh create mode 100644 docs/specs/blocksync/congestion_control.md create mode 100644 zebra-replay-bench/Cargo.toml create mode 100644 zebra-replay-bench/README.md create mode 100644 zebra-replay-bench/RESULTS.md create mode 100644 zebra-replay-bench/src/apply.rs create mode 100644 zebra-replay-bench/src/apply_sequencer.rs create mode 100644 zebra-replay-bench/src/apply_verifier.rs create mode 100644 zebra-replay-bench/src/apply_worker.rs create mode 100644 zebra-replay-bench/src/cache.rs create mode 100644 zebra-replay-bench/src/config.rs create mode 100644 zebra-replay-bench/src/index.rs create mode 100644 zebra-replay-bench/src/main.rs create mode 100644 zebra-replay-bench/src/prefetch.rs create mode 100644 zebra-replay-bench/src/rollback.rs create mode 100644 zebra-replay-bench/src/roots_cache.rs create mode 100644 zebra-replay-bench/src/stats.rs create mode 100644 zebra-state/src/service/finalized_state/vct.rs create mode 100644 zebra-state/src/service/finalized_state/vct/mainnet-frontier.bin diff --git a/.github/workflows/checkpoint-update.yml b/.github/workflows/checkpoint-update.yml index 61489e26ed3..28e33c8360f 100644 --- a/.github/workflows/checkpoint-update.yml +++ b/.github/workflows/checkpoint-update.yml @@ -35,6 +35,7 @@ jobs: env: MAINNET_CHECKPOINTS: zebra-chain/src/parameters/checkpoint/main-checkpoints.txt TESTNET_CHECKPOINTS: zebra-chain/src/parameters/checkpoint/test-checkpoints.txt + MAINNET_FRONTIER: zebra-state/src/service/finalized_state/vct/mainnet-frontier.bin EOS_FILE: zebrad/src/components/sync/end_of_support.rs steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 @@ -87,6 +88,15 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} continue-on-error: true + - name: Download mainnet frontier artifact + id: mainnet-frontier-artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 + with: + name: generate-checkpoints-mainnet-frontier + run-id: ${{ steps.resolve-run.outputs.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + - name: Download testnet checkpoint artifact id: testnet-artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 @@ -112,6 +122,11 @@ jobs: HAS_MAINNET="true" fi + if [ -f "mainnet-frontier.bin" ]; then + BYTES=$(wc -c < mainnet-frontier.bin | tr -d ' ') + echo "Mainnet frontier artifact: ${BYTES} bytes" + fi + if [ -f "test-checkpoints.txt" ]; then LINES=$(wc -l < test-checkpoints.txt | tr -d ' ') echo "Testnet artifact: ${LINES} checkpoint lines" @@ -130,6 +145,7 @@ jobs: # Append new mainnet checkpoints (entries with heights higher than current last) - name: Append new mainnet checkpoints + id: append-mainnet if: steps.check-artifacts.outputs.has_mainnet == 'true' run: | CURRENT_LAST=$(tail -1 "${MAINNET_CHECKPOINTS}" | awk '{print $1}') @@ -138,13 +154,48 @@ jobs: # Extract only new entries (height > current last) NEW_COUNT=$(awk -v last="$CURRENT_LAST" '$1 > last' main-checkpoints.txt | wc -l | tr -d ' ') echo "New mainnet checkpoints to append: ${NEW_COUNT}" + echo "new_count=${NEW_COUNT}" >> "$GITHUB_OUTPUT" if [ "$NEW_COUNT" -gt 0 ]; then awk -v last="$CURRENT_LAST" '$1 > last' main-checkpoints.txt >> "${MAINNET_CHECKPOINTS}" NEW_LAST=$(tail -1 "${MAINNET_CHECKPOINTS}" | awk '{print $1}') echo "Updated last mainnet checkpoint: ${NEW_LAST}" + echo "new_last=${NEW_LAST}" >> "$GITHUB_OUTPUT" + else + echo "new_last=${CURRENT_LAST}" >> "$GITHUB_OUTPUT" + fi + + - name: Update Mainnet VCT frontier + if: >- + steps.check-artifacts.outputs.has_mainnet == 'true' && + steps.append-mainnet.outputs.new_count != '0' + env: + EXPECTED_HEIGHT: ${{ steps.append-mainnet.outputs.new_last }} + run: | + if [ ! -s "mainnet-frontier.bin" ]; then + echo "Mainnet checkpoints advanced, but mainnet-frontier.bin is missing or empty" + exit 1 + fi + + FRONTIER_HEIGHT=$(python3 - <<'PY' + import struct + + with open("mainnet-frontier.bin", "rb") as frontier: + height_bytes = frontier.read(4) + if len(height_bytes) != 4: + raise SystemExit("frontier artifact is shorter than its height prefix") + print(struct.unpack(" Build + stage the bench binary as perf-artifacts/" @echo " perf-run [bin-name] Run an isolated bench (PERF_LABEL/PERF_STOP, optional staged bin)" @echo " perf-run-mainnet [bin-name] Run against public Mainnet Zakura peers (PERF_MAINNET_LABEL/PERF_STOP)" + @echo " perf-build-replay-bench Build the offline commit-replay bench (commit-metrics)" + @echo " perf-replay-index One-time: dump the REPLAY_START..REPLAY_END window to a cache" + @echo " perf-replay Replay the cache through the committer, no networking" + @echo " perf-replay-worker Replay through the real write worker (one rung up)" + @echo " perf-replay-verifier Replay through the checkpoint verifier (adds PoW + Merkle)" + @echo " perf-replay-sequencer Replay through the Zakura block-sync Sequencer (VCT-only," + @echo " pruned by default; REPLAY_ARCHIVE=1 for archive, REPLAY_TRACE_DIR= for traces)" @echo " perf-analyze Bottleneck attribution over the CSV window (PERF_LABEL/PERF_LO/PERF_HI)" @echo " perf-dashboard Live metrics dashboard for the running bench node" @echo " perf-verify-isolation Confirm the bench sees only the two cohort peers" diff --git a/deploy/runner/replay_run.sh b/deploy/runner/replay_run.sh new file mode 100755 index 00000000000..546ee9d9565 --- /dev/null +++ b/deploy/runner/replay_run.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Offline commit-pipeline replay bench (zebra-replay-bench). +# +# Replays real mainnet blocks through the state committer with NO networking, to +# benchmark the write-assembler + disk-writer in isolation. Forward model (no +# rollback): the base snapshot's tip must equal REPLAY_START-1, and blocks are +# replayed onto a fork of it. +# +# replay_run.sh index # fork the block source, dump the window to a cache +# replay_run.sh index-roots # fork the source, derive per-height roots -> VCT sidecar +# replay_run.sh run