draft: remaining feature branch changes we might want

.github/workflows/checkpoint-update.yml

@@ -35,6 +35,7 @@ jobs:
     env:
       MAINNET_CHECKPOINTS: zebra-chain/src/parameters/checkpoint/main-checkpoints.txt
       TESTNET_CHECKPOINTS: zebra-chain/src/parameters/checkpoint/test-checkpoints.txt
+      MAINNET_FRONTIER: zebra-state/src/service/finalized_state/vct/mainnet-frontier.bin
       EOS_FILE: zebrad/src/components/sync/end_of_support.rs
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
@@ -87,6 +88,15 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
         continue-on-error: true
 
+      - name: Download mainnet frontier artifact
+        id: mainnet-frontier-artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
+        with:
+          name: generate-checkpoints-mainnet-frontier
+          run-id: ${{ steps.resolve-run.outputs.run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+        continue-on-error: true
+
       - name: Download testnet checkpoint artifact
         id: testnet-artifact
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
@@ -112,6 +122,11 @@ jobs:
             HAS_MAINNET="true"
           fi
 
+          if [ -f "mainnet-frontier.bin" ]; then
+            BYTES=$(wc -c < mainnet-frontier.bin | tr -d ' ')
+            echo "Mainnet frontier artifact: ${BYTES} bytes"
+          fi
+
           if [ -f "test-checkpoints.txt" ]; then
             LINES=$(wc -l < test-checkpoints.txt | tr -d ' ')
             echo "Testnet artifact: ${LINES} checkpoint lines"
@@ -130,6 +145,7 @@ jobs:
 
       # Append new mainnet checkpoints (entries with heights higher than current last)
       - name: Append new mainnet checkpoints
+        id: append-mainnet
         if: steps.check-artifacts.outputs.has_mainnet == 'true'
         run: |
           CURRENT_LAST=$(tail -1 "${MAINNET_CHECKPOINTS}" | awk '{print $1}')
@@ -138,13 +154,48 @@ jobs:
           # Extract only new entries (height > current last)
           NEW_COUNT=$(awk -v last="$CURRENT_LAST" '$1 > last' main-checkpoints.txt | wc -l | tr -d ' ')
           echo "New mainnet checkpoints to append: ${NEW_COUNT}"
+          echo "new_count=${NEW_COUNT}" >> "$GITHUB_OUTPUT"
 
           if [ "$NEW_COUNT" -gt 0 ]; then
             awk -v last="$CURRENT_LAST" '$1 > last' main-checkpoints.txt >> "${MAINNET_CHECKPOINTS}"
             NEW_LAST=$(tail -1 "${MAINNET_CHECKPOINTS}" | awk '{print $1}')
             echo "Updated last mainnet checkpoint: ${NEW_LAST}"
+            echo "new_last=${NEW_LAST}" >> "$GITHUB_OUTPUT"
+          else
+            echo "new_last=${CURRENT_LAST}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Update Mainnet VCT frontier
+        if: >-
+          steps.check-artifacts.outputs.has_mainnet == 'true' &&
+          steps.append-mainnet.outputs.new_count != '0'
+        env:
+          EXPECTED_HEIGHT: ${{ steps.append-mainnet.outputs.new_last }}
+        run: |
+          if [ ! -s "mainnet-frontier.bin" ]; then
+            echo "Mainnet checkpoints advanced, but mainnet-frontier.bin is missing or empty"
+            exit 1
+          fi
+
+          FRONTIER_HEIGHT=$(python3 - <<'PY'
+          import struct
+
+          with open("mainnet-frontier.bin", "rb") as frontier:
+              height_bytes = frontier.read(4)
+          if len(height_bytes) != 4:
+              raise SystemExit("frontier artifact is shorter than its height prefix")
+          print(struct.unpack("<I", height_bytes)[0])
+          PY
+          )
+
+          if [ "${FRONTIER_HEIGHT}" != "${EXPECTED_HEIGHT}" ]; then
+            echo "Frontier height ${FRONTIER_HEIGHT} does not match updated Mainnet checkpoint ${EXPECTED_HEIGHT}"
+            exit 1
           fi
 
+          cp mainnet-frontier.bin "${MAINNET_FRONTIER}"
+          echo "Updated ${MAINNET_FRONTIER} for checkpoint height ${EXPECTED_HEIGHT}"
+
       # Append new testnet checkpoints
       - name: Append new testnet checkpoints
         if: steps.check-artifacts.outputs.has_testnet == 'true'
@@ -214,6 +265,7 @@ jobs:
             ### Changes
 
             - Updated mainnet and/or testnet checkpoint files with new entries
+            - Updated `mainnet-frontier.bin` when Mainnet checkpoints advanced
             - Updated `ESTIMATED_RELEASE_HEIGHT` in `end_of_support.rs` to match the latest mainnet checkpoint
 
             ### Validation
@@ -223,6 +275,7 @@ jobs:
             - Heights are monotonically increasing
             - No gaps exceed 400 blocks
             - No duplicate heights or hashes
+            - Mainnet frontier height matches the updated Mainnet checkpoint height, when present
 
             ### Review
 

.vscode/settings.json

@@ -0,0 +1,17 @@
+{
+  // Rust build artifacts (~140G in target/, ~5G in unity-node/target/) saturate
+  // the file watcher and index, which hangs the extension host (agents + terminal).
+  // These dirs are gitignored, so hiding them from the editor is safe.
+  "files.watcherExclude": {
+    "**/target/**": true,
+    "**/.git/objects/**": true
+  },
+  "search.exclude": {
+    "**/target": true
+  },
+  "files.exclude": {
+    "**/target": true
+  },
+  // Let rust-analyzer manage the workspace without a redundant cargo check storm.
+  "rust-analyzer.files.excludeDirs": ["target"]
+}

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Fixed
 
+- Reject `consensus.checkpoint_sync = false` with
+  `consensus.vct_fast_sync = true` at startup, because VCT fast sync relies on
+  the full checkpoint-sync verification range.
 - Fixed an out-of-memory crash during Zakura block sync when the header chain
   runs far ahead of the commit tip. The block-sync applying buffer holds decoded
   block bodies ahead of the in-order committer; its look-ahead budget counted
@@ -116,9 +119,6 @@ and this project adheres to [Semantic Versioning](https://semver.org).
   commitment roots and transaction counts.
 - Reject transactions that add net value to the Orchard pool after NU6.3
   activation.
-- V6 transactions with supported NU6.3-or-later consensus branch IDs now
-  serialize and deserialize successfully, while unsupported later placeholders
-  are rejected.
 - Route post-NU6.3 coinbase rewards for Orchard receivers in unified miner
   addresses to the Ironwood pool instead of rejecting them or falling back to a
   lower-priority receiver.
@@ -167,6 +167,24 @@ and this project adheres to [Semantic Versioning](https://semver.org).
   duplicate-peer handling scaffolding.
 - Added bounded Zakura header-sync stream-5 wire messages, stateless header
   validation, and the default `network.zakura.header_sync` config surface.
+- Verified-commitment-trees fast checkpoint sync. Below the last checkpoint Zebra
+  now fetches per-block Sapling/Orchard commitment roots from peers over a new
+  header-sync-aligned `tree_aux` stream, verifies each root against the node's own
+  checkpoint-committed block headers (the ZIP-221 ChainHistory MMR plus direct
+  below-Heartwood/below-NU5 checks), and folds the verified roots into the anchor
+  set and history tree — skipping the per-block note-commitment frontier recompute
+  that dominates checkpoint-sync CPU cost. At the checkpoint handoff an embedded
+  final frontier, verified against that block's proven root, is written as the tip
+  treestate and normal per-block recompute resumes. The resulting consensus state
+  is byte-identical to the legacy recompute; a root that cannot be obtained or
+  verified is rejected rather than recomputed against the stale frozen frontier, so
+  no untrusted data can influence consensus state. This is the default whenever
+  `consensus.checkpoint_sync = true` on a network with an embedded handoff frontier
+  (Mainnet), for both Archive and Pruned storage modes. The new
+  `consensus.vct_fast_sync` flag (default `true`) selects this fast path; set it to
+  `false` to keep checkpoint sync enabled while forcing the legacy per-block
+  recompute. Bumps the state database
+  format to 27.3.0 (new column families only; no data migration).
 - Include the `zebra-rollback-state` and `zebra-prune-state` utilities alongside
   `zebrad` in release Docker images and Docker CI builds.
 - Use the `5.0.0-rc.3` release identity for this fork's v5 rollback build.
@@ -225,6 +243,14 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Fixed
 
+- Stop the database format-validity check from panicking with "just checked for
+  genesis block" while a verified-commitment-trees fast sync is in progress. The
+  check runs on a background thread, concurrently with block commits, and could
+  read its `is_vct_synced()` guard as `false` and then read an absent genesis
+  note-commitment tree once a concurrent fast-sync commit set the marker in
+  between. It now treats an absent genesis tree as a (mid-flight) fast-synced
+  database — where the genesis-root-caching invariant does not apply — instead of
+  panicking.
 - Use network protocol version 170160 as the NU6.3 minimum on Mainnet, Testnet,
   and Regtest, matching Zebra's advertised current protocol version.
 - Avoid panics in the block write task when RPC users invalidate a non-finalized

Cargo.lock

@@ -9317,6 +9317,25 @@ dependencies = [
  "zebra-chain",
 ]
 
+[[package]]
+name = "zebra-replay-bench"
+version = "0.1.0"
+dependencies = [
+ "clap 4.6.1",
+ "color-eyre",
+ "futures",
+ "metrics-exporter-prometheus",
+ "tempfile",
+ "tokio",
+ "tower 0.4.13",
+ "tracing",
+ "tracing-subscriber",
+ "zebra-chain",
+ "zebra-consensus",
+ "zebra-network",
+ "zebra-state",
+]
+
 [[package]]
 name = "zebra-rpc"
 version = "9.0.0"
@@ -9503,6 +9522,7 @@ dependencies = [
  "zebra-chain",
  "zebra-node-services",
  "zebra-rpc",
+ "zebra-state",
 ]
 
 [[package]]

Cargo.toml

@@ -4,6 +4,7 @@ members = [
         "zebra-chain",
         "zebra-network",
         "zebra-state",
+        "zebra-replay-bench",
         "zebra-script",
         "zebra-consensus",
         "zebra-rpc",

Makefile

@@ -11,6 +11,13 @@ help:
 	@echo "  perf-build-stage-bin <name>      Build + stage the bench binary as perf-artifacts/<name>"
 	@echo "  perf-run [bin-name]              Run an isolated bench (PERF_LABEL/PERF_STOP, optional staged bin)"
 	@echo "  perf-run-mainnet [bin-name]      Run against public Mainnet Zakura peers (PERF_MAINNET_LABEL/PERF_STOP)"
+	@echo "  perf-build-replay-bench          Build the offline commit-replay bench (commit-metrics)"
+	@echo "  perf-replay-index                One-time: dump the REPLAY_START..REPLAY_END window to a cache"
+	@echo "  perf-replay                      Replay the cache through the committer, no networking"
+	@echo "  perf-replay-worker               Replay through the real write worker (one rung up)"
+	@echo "  perf-replay-verifier             Replay through the checkpoint verifier (adds PoW + Merkle)"
+	@echo "  perf-replay-sequencer            Replay through the Zakura block-sync Sequencer (VCT-only,"
+	@echo "                                   pruned by default; REPLAY_ARCHIVE=1 for archive, REPLAY_TRACE_DIR=<dir> for traces)"
 	@echo "  perf-analyze                     Bottleneck attribution over the CSV window (PERF_LABEL/PERF_LO/PERF_HI)"
 	@echo "  perf-dashboard                   Live metrics dashboard for the running bench node"
 	@echo "  perf-verify-isolation            Confirm the bench sees only the two cohort peers"