diff --git a/.cursor/skills/zakura-trace-plots/SKILL.md b/.cursor/skills/zakura-trace-plots/SKILL.md
index 24d1e9fa1c0..b4c51defea6 100644
--- a/.cursor/skills/zakura-trace-plots/SKILL.md
+++ b/.cursor/skills/zakura-trace-plots/SKILL.md
@@ -29,6 +29,20 @@ Default outputs:
Use the time plot for stall diagnosis. Height plots collapse zero-progress stalls onto one x-position.
+When the user is analyzing the verify -> commit pipeline and provides per-block
+timing traces such as `seq50k-commit-timing.jsonl` and
+`seq50k-verify-timing.jsonl`, also generate the by-height timing plot:
+
+```bash
+python3 .cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py \
+ COMMIT_TIMING.jsonl VERIFY_TIMING.jsonl --out-dir perf-artifacts
+```
+
+This graph is useful for separating verifier cost from committer cost. It plots
+commit phases (`spent_utxo_reads`, `address_reads`, `batch_assembly`,
+`batch_commit`), commit total latency, verifier phases (`pow`, `precompute`,
+`merkle`), and rolling implied throughput from each timing stream.
+
## Metrics Awareness
Use `block_sync.jsonl` as the source of truth for:
diff --git a/.cursor/skills/zakura-trace-plots/scripts/plot_ab_throughput_keys.py b/.cursor/skills/zakura-trace-plots/scripts/plot_ab_throughput_keys.py
new file mode 100644
index 00000000000..85b93f5cedd
--- /dev/null
+++ b/.cursor/skills/zakura-trace-plots/scripts/plot_ab_throughput_keys.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""Overlay two stage-timing runs: throughput and transparent keys/block by height.
+
+For A/B comparisons where each run has a stage-timing.jsonl (per-block keys_* and
+ts_us). Throughput comes from the stage timestamps, so it is NOT skewed by any
+commit-timing trace overhead — both runs are measured the same, lightweight way.
+
+Usage:
+ python3 plot_ab_throughput_keys.py LABEL_A=stageA.jsonl LABEL_B=stageB.jsonl \
+ --out-dir perf-artifacts --name skip-ab [--bins 90]
+"""
+import argparse
+import json
+import math
+from pathlib import Path
+
+COLORS = ["#d62728", "#2ca02c", "#1f77b4", "#ff7f0e"]
+
+
+def load(path):
+ ts, ktrans = {}, {}
+ for line in open(path):
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ r = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ s, h = r.get("stage", ""), r.get("height")
+ if h is None:
+ continue
+ if s.startswith("keys_"):
+ if s == "keys_transparent":
+ ktrans[h] = r.get("val", 0)
+ ts[h] = max(ts.get(h, 0), r.get("ts_us", 0))
+ return ts, ktrans
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("runs", nargs="+", help="LABEL=stage.jsonl")
+ ap.add_argument("--out-dir", default="perf-artifacts")
+ ap.add_argument("--name", default="ab")
+ ap.add_argument("--bins", type=int, default=90)
+ args = ap.parse_args()
+
+ runs = []
+ for spec in args.runs:
+ label, path = spec.split("=", 1)
+ ts, kt = load(path)
+ runs.append((label, ts, kt))
+
+ h0 = min(min(ts) for _, ts, _ in runs)
+ h1 = max(max(ts) for _, ts, _ in runs)
+ width = max(1, math.ceil((h1 - h0 + 1) / args.bins))
+
+ def throughput_bins(ts):
+ buckets = {}
+ for h, t in ts.items():
+ buckets.setdefault((h - h0) // width, []).append((h, t))
+ out = {}
+ for k, pts in buckets.items():
+ pts.sort()
+ if len(pts) >= 2 and pts[-1][1] > pts[0][1]:
+ out[k] = (pts[-1][0] - pts[0][0]) / ((pts[-1][1] - pts[0][1]) / 1e6)
+ return out
+
+ def key_bins(kt):
+ buckets = {}
+ for h, v in kt.items():
+ buckets.setdefault((h - h0) // width, []).append(v)
+ return {k: sum(v) / len(v) for k, v in buckets.items()}
+
+ series = []
+ for i, (label, ts, kt) in enumerate(runs):
+ series.append((label, COLORS[i % len(COLORS)], throughput_bins(ts), key_bins(kt),
+ (h1 - h0) / ((max(ts.values()) - min(ts.values())) / 1e6) if len(ts) > 1 else 0))
+
+ W, left, right = 1240, 96, 36
+ plot_w = W - left - right
+ panel_h, gap = 290, 92
+ top1, top2 = 104, 104 + 290 + 92
+ H = top2 + panel_h + 60
+
+ bps_max = max(max(s[2].values(), default=1) for s in series) * 1.05
+ key_max = max(max(s[3].values(), default=1) for s in series) * 1.05
+
+ def xm(k):
+ h = h0 + k * width + width // 2
+ return left + (h - h0) / (h1 - h0) * plot_w
+
+ def ymap(v, vmax, top):
+ return top + panel_h - min(max(v, 0), vmax) / vmax * panel_h
+
+ svg = [
+ f'")
+
+ out = Path(args.out_dir)
+ out.mkdir(parents=True, exist_ok=True)
+ p = out / f"{args.name}-throughput-keys-ab.svg"
+ p.write_text("\n".join(svg))
+ print(f"wrote {p}")
+ # time-weighted regional throughput
+ def rate(ts, lo, hi):
+ sel = sorted((h, t) for h, t in ts.items() if lo <= h < hi)
+ return (sel[-1][0] - sel[0][0]) / ((sel[-1][1] - sel[0][1]) / 1e6) if len(sel) > 1 and sel[-1][1] > sel[0][1] else None
+ print(f" {'region':>22} " + " ".join(f"{lbl:>10}" for lbl, *_ in series))
+ for name, lo, hi in [("light <1862900", h0, 1862900), ("sandblast 1862.9-1872k", 1862900, 1872000), ("recovery 1872k+", 1872000, h1 + 1)]:
+ vals = [rate(ts, lo, hi) for _, ts, _ in runs]
+ if all(v is not None for v in vals):
+ print(f" {name:>22} " + " ".join(f"{v:>10.0f}" for v in vals))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py b/.cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py
new file mode 100644
index 00000000000..d0867999d9c
--- /dev/null
+++ b/.cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py
@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+"""Plot Zebra replay-bench commit + verify timing traces by height."""
+
+from __future__ import annotations
+
+import argparse
+import html
+import json
+import math
+from pathlib import Path
+
+
+def load_jsonl(path: Path) -> list[dict[str, object]]:
+ return [json.loads(line) for line in path.open() if line.strip()]
+
+
+def number(row: dict[str, object], key: str) -> float:
+ try:
+ return float(row.get(key, 0) or 0)
+ except (TypeError, ValueError):
+ return 0.0
+
+
+def height(row: dict[str, object]) -> int:
+ return int(row["height"])
+
+
+def percentile(values: list[float], percent: float) -> float:
+ values = sorted(value for value in values if math.isfinite(value))
+ if not values:
+ return 1.0
+
+ index = (len(values) - 1) * percent / 100
+ low = math.floor(index)
+ high = math.ceil(index)
+ if low == high:
+ return values[low]
+
+ return values[low] * (high - index) + values[high] * (index - low)
+
+
+def nice_max(values: list[float]) -> float:
+ maximum = max([value for value in values if math.isfinite(value)] or [1.0])
+ if maximum <= 0:
+ return 1.0
+
+ exponent = math.floor(math.log10(maximum))
+ base = 10**exponent
+ for multiplier in (1, 2, 5, 10):
+ if maximum <= multiplier * base:
+ return multiplier * base
+
+ return 10 * base
+
+
+def rolling_blocks_per_second(
+ rows: list[dict[str, object]],
+ window: int,
+) -> tuple[list[int], list[float]]:
+ xs: list[int] = []
+ ys: list[float] = []
+
+ for index in range(0, len(rows) - window):
+ start = rows[index]
+ end = rows[index + window]
+ height_delta = height(end) - height(start)
+ time_delta = (number(end, "ts_us") - number(start, "ts_us")) / 1_000_000
+
+ if height_delta > 0 and time_delta > 0:
+ xs.append(height(end))
+ ys.append(height_delta / time_delta)
+
+ return xs, ys
+
+
+def polyline(
+ xs: list[int],
+ ys: list[float],
+ *,
+ min_height: int,
+ max_height: int,
+ max_value: float,
+ left: int,
+ width: int,
+ right: int,
+ panel_top: int,
+ panel_height: int,
+) -> str:
+ plot_height = panel_height - 45
+ points = []
+
+ for x_value, y_value in zip(xs, ys):
+ clamped = min(max(y_value, 0.0), max_value)
+ x = left + (x_value - min_height) / (max_height - min_height) * (width - left - right)
+ y = panel_top + 25 + plot_height - (clamped / max_value) * plot_height
+ points.append(f"{x:.1f},{y:.1f}")
+
+ return " ".join(points)
+
+
+def write_svg(
+ commit_rows: list[dict[str, object]],
+ verify_rows: list[dict[str, object]],
+ out_path: Path,
+ *,
+ title: str,
+ window: int,
+) -> None:
+ commit_heights = [height(row) for row in commit_rows]
+ verify_heights = [height(row) for row in verify_rows]
+ min_height = min(commit_heights + verify_heights)
+ max_height = max(commit_heights + verify_heights)
+ if min_height == max_height:
+ max_height += 1
+
+ commit_phase_heights = commit_heights
+ verify_phase_heights = verify_heights
+ commit_stream_heights, commit_bps = rolling_blocks_per_second(commit_rows, window)
+ verify_stream_heights, verify_bps = rolling_blocks_per_second(verify_rows, window)
+
+ panels = [
+ (
+ "Commit phases (ms, y capped at p99.5)",
+ 99.5,
+ [
+ ("spent_utxo_reads", commit_phase_heights, [number(row, "reads_ms") for row in commit_rows], "#1f77b4"),
+ ("address_reads", commit_phase_heights, [number(row, "address_reads_ms") for row in commit_rows], "#ff7f0e"),
+ ("batch_assembly", commit_phase_heights, [number(row, "batch_assembly_ms") for row in commit_rows], "#2ca02c"),
+ ("batch_commit", commit_phase_heights, [number(row, "batch_commit_ms") for row in commit_rows], "#d62728"),
+ ],
+ ),
+ (
+ "Commit total (ms, y capped at p99.5)",
+ 99.5,
+ [
+ ("commit_total", commit_phase_heights, [number(row, "commit_total_ms") for row in commit_rows], "#111111"),
+ ],
+ ),
+ (
+ "Verify phases (ms, y capped at p99.5)",
+ 99.5,
+ [
+ ("pow", verify_phase_heights, [number(row, "pow_ms") for row in verify_rows], "#9467bd"),
+ ("precompute", verify_phase_heights, [number(row, "precompute_ms") for row in verify_rows], "#8c564b"),
+ ("merkle", verify_phase_heights, [number(row, "merkle_ms") for row in verify_rows], "#17becf"),
+ ],
+ ),
+ (
+ "Rolling throughput from timing timestamps (blk/s)",
+ 100.0,
+ [
+ ("commit stream", commit_stream_heights, commit_bps, "#111111"),
+ ("verify stream", verify_stream_heights, verify_bps, "#2ca02c"),
+ ],
+ ),
+ ]
+
+ width = 1500
+ panel_height = 230
+ left = 80
+ right = 30
+ top = 58
+ gap = 52
+ bottom = 45
+ svg_height = top + len(panels) * panel_height + (len(panels) - 1) * gap + bottom
+
+ svg: list[str] = [
+ f'")
+
+ out_path.write_text("\n".join(svg))
+
+
+def write_summary(
+ commit_rows: list[dict[str, object]],
+ verify_rows: list[dict[str, object]],
+ out_path: Path,
+ plot_path: Path,
+) -> None:
+ out_path.write_text(
+ f"commit_samples: {len(commit_rows)}\n"
+ f"verify_samples: {len(verify_rows)}\n"
+ f"commit_height_range: {height(commit_rows[0])}-{height(commit_rows[-1])}\n"
+ f"verify_height_range: {height(verify_rows[0])}-{height(verify_rows[-1])}\n"
+ f"plot: {plot_path}\n"
+ )
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("commit_timing", type=Path, help="commit timing JSONL path")
+ parser.add_argument("verify_timing", type=Path, help="verify timing JSONL path")
+ parser.add_argument("--out-dir", type=Path, default=Path("perf-artifacts"))
+ parser.add_argument("--label", help="output file label; defaults to commit timing stem")
+ parser.add_argument("--window", type=int, default=50, help="sample window for rolling throughput")
+ args = parser.parse_args()
+
+ commit_rows = load_jsonl(args.commit_timing)
+ verify_rows = load_jsonl(args.verify_timing)
+ if not commit_rows:
+ raise SystemExit(f"no rows in {args.commit_timing}")
+ if not verify_rows:
+ raise SystemExit(f"no rows in {args.verify_timing}")
+
+ args.out_dir.mkdir(parents=True, exist_ok=True)
+ label = args.label or args.commit_timing.stem.removesuffix("-commit-timing")
+ plot_path = args.out_dir / f"{label}-commit-verify-by-height.svg"
+ summary_path = args.out_dir / f"{label}-commit-verify-summary.txt"
+
+ write_svg(
+ commit_rows,
+ verify_rows,
+ plot_path,
+ title=f"{label} commit/verify timing by height",
+ window=max(args.window, 1),
+ )
+ write_summary(commit_rows, verify_rows, summary_path, plot_path)
+
+ print(plot_path)
+ print(summary_path)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.cursor/skills/zakura-trace-plots/scripts/plot_latency_throughput_by_height.py b/.cursor/skills/zakura-trace-plots/scripts/plot_latency_throughput_by_height.py
new file mode 100644
index 00000000000..b3c5c338ab8
--- /dev/null
+++ b/.cursor/skills/zakura-trace-plots/scripts/plot_latency_throughput_by_height.py
@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""Plot per-block metric percentiles, throughput, and key composition by height.
+
+Companion to plot_commit_verify_timing.py: a percentile-by-height view from a
+commit-timing.jsonl stream, optionally with a stage-timing.jsonl stream for the
+per-category key breakdown.
+
+Panels (top -> bottom), all sharing the block-height x-axis:
+ 1. p50/p90 of a chosen metric (default block size, KB)
+ 2. throughput (blk/s) derived from ts_us deltas
+ 3. (if --stage-file given) commit batch keys/block, stacked by category
+ (shielded nullifiers vs transparent UTXO+index vs other)
+
+Usage:
+ python3 plot_latency_throughput_by_height.py COMMIT_TIMING.jsonl \
+ --stage-file STAGE.jsonl --out-dir perf-artifacts [--bins 150]
+"""
+import argparse
+import json
+import math
+from pathlib import Path
+
+BLUE, RED, GREEN = "#1f77b4", "#d62728", "#2ca02c"
+# stacked key categories: (label, color, stage names summed into it)
+KEY_CATS = [
+ ("shielded (nullifiers)", "#9467bd", ["keys_shielded"]),
+ ("transparent (UTXO+index)", "#ff7f0e", ["keys_transparent"]),
+ ("other (header/trees/pool)", "#9aa0a6",
+ ["keys_header_tx", "keys_trees", "keys_valuepool", "keys_vct_evict"]),
+]
+
+
+def load(path):
+ rows = []
+ for line in open(path):
+ line = line.strip()
+ if line:
+ try:
+ rows.append(json.loads(line))
+ except json.JSONDecodeError:
+ pass
+ return rows
+
+
+def percentile(values, pct):
+ if not values:
+ return 0.0
+ s = sorted(values)
+ k = (len(s) - 1) * pct / 100.0
+ lo, hi = math.floor(k), math.ceil(k)
+ return s[int(k)] if lo == hi else s[lo] * (hi - k) + s[hi] * (k - lo)
+
+
+def nice_max(v):
+ if v <= 0:
+ return 1.0
+ mag = 10 ** math.floor(math.log10(v))
+ for m in (1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10):
+ if m * mag >= v:
+ return m * mag
+ return 10 * mag
+
+
+def x_map(h, h0, h1, left, w):
+ return left + (h - h0) / (h1 - h0) * w
+
+
+def lin_y(v, vmax, top, h):
+ return top + h - min(max(v, 0.0), vmax) / vmax * h
+
+
+def polyline(pts, color, wid=2.0):
+ if not pts:
+ return ""
+ d = " ".join(f"{x:.1f},{y:.1f}" for x, y in pts)
+ return f''
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("commit_timing")
+ ap.add_argument("--stage-file", default=None)
+ ap.add_argument("--out-dir", default="perf-artifacts")
+ ap.add_argument("--bins", type=int, default=150)
+ ap.add_argument("--metric", default="block_bytes")
+ ap.add_argument("--scale", default="kb", choices=["kb", "mb", "raw", "ms"])
+ ap.add_argument("--name", default=None)
+ args = ap.parse_args()
+
+ sf = {"kb": 1 / 1024, "mb": 1 / 1048576, "raw": 1.0, "ms": 1.0}[args.scale]
+ unit = {"kb": "KB", "mb": "MB", "raw": "", "ms": "ms"}[args.scale]
+
+ rows = sorted(load(args.commit_timing), key=lambda r: r["height"])
+ if not rows:
+ raise SystemExit("no rows")
+ h0, h1 = rows[0]["height"], rows[-1]["height"]
+ width = max(1, math.ceil((h1 - h0 + 1) / args.bins))
+
+ def bin_by(items, key):
+ b = {}
+ for it in items:
+ b.setdefault((key(it) - h0) // width, []).append(it)
+ return b
+
+ # commit-timing bins: metric p50/p90
+ cb = bin_by(rows, lambda r: r["height"])
+ mbins = []
+ for k in sorted(cb):
+ rs = cb[k]
+ vals = [float(r.get(args.metric, 0.0)) * sf for r in rs]
+ mbins.append({"h": h0 + k * width + width // 2,
+ "p50": percentile(vals, 50), "p90": percentile(vals, 90)})
+
+ # stage-timing: per-category mean keys/block + per-height commit timestamps.
+ # Throughput is taken from the stage stream when available (every block, so far
+ # finer than the sampled commit-timing, and unskewed by the commit-trace cost).
+ kbins = None
+ ts_pairs = [(r["height"], r["ts_us"]) for r in rows] # fallback: sampled commit-timing
+ if args.stage_file and Path(args.stage_file).exists():
+ per_height = {}
+ per_height_ts = {}
+ for r in load(args.stage_file):
+ s = r.get("stage", "")
+ h = r["height"]
+ if s.startswith("keys_"):
+ per_height.setdefault(h, {})[s] = r.get("val", 0)
+ ts = r.get("ts_us")
+ if ts is not None:
+ per_height_ts[h] = max(per_height_ts.get(h, 0), ts)
+ if per_height_ts:
+ ts_pairs = sorted(per_height_ts.items())
+ items = [{"h": h, **v} for h, v in per_height.items()]
+ sb = bin_by(items, lambda it: it["h"])
+ kbins = []
+ for k in sorted(sb):
+ rs = sb[k]
+ row = {"h": h0 + k * width + width // 2}
+ for label, _c, stages in KEY_CATS:
+ row[label] = sum(sum(it.get(s, 0) for s in stages) for it in rs) / len(rs)
+ kbins.append(row)
+
+ # throughput bins from the chosen timestamp stream
+ tb = bin_by([{"h": h, "ts": ts} for h, ts in ts_pairs], lambda it: it["h"])
+ tbins = []
+ for k in sorted(tb):
+ rs = sorted(tb[k], key=lambda it: it["h"])
+ bps = None
+ if len(rs) >= 2:
+ dt = (rs[-1]["ts"] - rs[0]["ts"]) / 1e6
+ dh = rs[-1]["h"] - rs[0]["h"]
+ if dt > 0 and dh > 0:
+ bps = dh / dt
+ tbins.append({"h": h0 + k * width + width // 2, "bps": bps})
+
+ # ---- layout ----
+ W, left, right = 1240, 96, 36
+ plot_w = W - left - right
+ panel_h, gap = 280, 86
+ n_panels = 3 if kbins else 2
+ tops = [104 + i * (panel_h + gap) for i in range(n_panels)]
+ H = tops[-1] + panel_h + 64
+
+ m_max = nice_max(max([b["p90"] for b in mbins] + [1.0]))
+ bps_max = nice_max(max([b["bps"] for b in tbins if b["bps"] is not None] + [1.0]) * 1.05)
+ keys_max = nice_max(max([sum(b[l] for l, _c, _s in KEY_CATS) for b in kbins] + [1.0])) if kbins else 1.0
+
+ svg = [
+ f'")
+
+ out_dir = Path(args.out_dir)
+ out_dir.mkdir(parents=True, exist_ok=True)
+ out = out_dir / f"{title}-commit-profile-by-height.svg"
+ out.write_text("\n".join(svg))
+ print(f"wrote {out}")
+ # time-weighted regional throughput (blocks / wall-time), NOT mean of bin rates
+ def region_rate(lo, hi):
+ sel = [(h, t) for h, t in ts_pairs if lo <= h < hi]
+ if len(sel) < 2 or sel[-1][1] <= sel[0][1]:
+ return None
+ return (sel[-1][0] - sel[0][0]) / ((sel[-1][1] - sel[0][1]) / 1e6)
+ print(" throughput (time-weighted, blk/s):")
+ for name, lo, hi in [("overall", h0, h1 + 1), ("light <1862900", h0, 1862900),
+ ("sandblast 1862.9-1872k", 1862900, 1872000),
+ ("recovery 1872k-end", 1872000, h1 + 1)]:
+ r = region_rate(lo, hi)
+ print(f" {name:26} {r:7.1f}" if r else f" {name:26} n/a")
+ if kbins:
+ def avg(rs, l): return sum(b[l] for b in rs) / len(rs) if rs else 0
+ light = [b for b in kbins if b["h"] < 1862900]
+ sand = [b for b in kbins if 1862900 <= b["h"] <= 1872000]
+ print(" keys/block (mean):")
+ for l, _c, _s in KEY_CATS:
+ print(f" {l:28} light~{avg(light,l):7.0f} sandblast~{avg(sand,l):7.0f}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cb5e1a77b0f..eb457ee9693 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,24 @@ and this project adheres to [Semantic Versioning](https://semver.org).
### Performance
+- Fix work-queue scans that stalled the Zakura block-sync commit pipeline for
+ tens of seconds during checkpoint sync. When headers race far ahead of the body
+ tip, the sequencer's `WorkQueue` holds the entire lag (100k+ pending heights) in
+ mutex-guarded `BTreeMap`s, and two O(n) operations ran under that lock on the
+ hot path: `reserved_bytes()` (a full scan re-summing reserved request bytes on
+ every `publish_view`, i.e. every body/control event) and `advance_floor()` (a
+ full-map `retain` garbage-collecting committed heights on every floor advance).
+ As the backlog grew these became quadratic, saturating the single sequencer task
+ and serializing the work-queue lock so both commit *and* download stalled
+ (observed as ~12–30s body-commit freezes with peers blocked on the lock). Both
+ are now bounded: `reserved_bytes` is an O(1) incrementally-maintained counter
+ (cross-checked against the independent byte budget by the existing audit), and
+ `advance_floor`/`reset_above` pop only the committed prefix/suffix
+ (O(removed · log n)) instead of scanning the whole map. The `publish_view`
+ scans over the `applying` map (`applying_buffered_bytes`,
+ `submitted_applying_count`/`_bytes`, and the derived `unsubmitted_applying_count`)
+ are likewise now O(1) incrementally-maintained counters instead of a scan of the
+ apply backlog on every event.
- Compute the v5 ZIP-244 txid and authorizing-data digest natively. Both
previously routed through `Transaction::to_librustzcash`, which re-serializes
and reparses the whole transaction — decompressing every Jubjub and Pallas
@@ -20,6 +38,26 @@ and this project adheres to [Semantic Versioning](https://semver.org).
`librustzcash`. The output is byte-identical: a differential property test
(`native_zip244_matches_librustzcash`) asserts the native txid and auth digest
match the `librustzcash` conversion across thousands of random v5 transactions.
+- Add a run-ahead finalized-commit pipeline that overlaps the next block's batch
+ assembly with the current block's disk write, raising checkpoint-sync commit
+ throughput from `assemble + flush` per block toward `max(assemble, flush)`. The
+ committer is split into an assemble half (`assemble_finalized_direct` /
+ `assemble_block_batch` — treestate compute + batch build, no writes) and a flush
+ half (`flush_finalized_direct` / `flush_block_batch` — the rocksdb write and
+ post-commit bookkeeping). When enabled, the committer assembles each block's
+ batch on its thread and hands it to a dedicated disk-writer thread over a bounded
+ channel (the channel depth is the backpressure). A new `FinalizedPipeline` (in
+ `zebra-state/src/service/finalized_state/pipeline.rs`) carries the in-memory tip
+ state — history tree, note-commitment trees, value pool, `vct_upgrade_height`
+ marker, tip cursor — and a read-through overlay for not-yet-flushed spent UTXOs
+ and address balances, so a block assembled ahead of the durable write reads its
+ parent's effects from memory. The externally-visible finalized tip, the per-block
+ commit response, and the download budget all trail the durable flush
+ (ack-after-flush), so crash recovery is unchanged. Only the checkpoint
+ (reorg-free) region uses the pipeline; near the tip the committer stays
+ synchronous. The new `Config::finalized_block_pipeline_depth` defaults to `0`
+ (synchronous), so behavior is byte-identical to the previous committer until the
+ depth is raised. See `CHANGELOG_PARAMS.md`.
- Parallelize per-block serialization in the finalized block writer. On heavy
shielded blocks, serializing the raw transaction bytes (`tx_by_loc`) and
computing the block size for `BlockInfo` dominate the per-block write cost. Both
diff --git a/CHANGELOG_PARAMS.md b/CHANGELOG_PARAMS.md
index 6ccbcca61f7..c7695aa028a 100644
--- a/CHANGELOG_PARAMS.md
+++ b/CHANGELOG_PARAMS.md
@@ -28,4 +28,5 @@ Keep entries **newest-first**. Each row records:
| Parameter | Location | Old → New | PR | Why |
| --- | --- | --- | --- | --- |
+| `Config::finalized_block_pipeline_depth` | `zebra-state/src/config.rs` | _(new)_ → `0` | _(this PR)_ | Run-ahead finalized-commit pipeline depth (blocks the assembler may build ahead of the durable disk write). Defaults to `0` = synchronous (original behavior); `> 0` overlaps the next block's assembly with the current block's flush, bounded so the in-memory overlay stays small. |
| `OUTBOUND_WINDOW_FLOOR_TIMEOUTS_BEFORE_DISCONNECT` | `zebra-network/src/zakura/block_sync/state.rs` | `3` → `2 * OUTBOUND_WINDOW_REDUCTION_EPOCH_TIMEOUTS` (`32`) | [#303](https://github.com/valargroup/zebra/pull/303) | Tolerate two full reduction epochs (~256s at the 8s request timeout) of floor-pinned timeouts before disconnecting a block-sync peer, instead of ~24s, so briefly-congested peers are not churned. Any successful response resets the streak. |
diff --git a/Cargo.lock b/Cargo.lock
index 0c25e01c9f1..67edea8ecb5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9314,6 +9314,7 @@ version = "0.1.0"
dependencies = [
"clap 4.6.1",
"color-eyre",
+ "crossbeam-channel",
"futures",
"metrics-exporter-prometheus",
"tempfile",
@@ -9324,7 +9325,9 @@ dependencies = [
"zebra-chain",
"zebra-consensus",
"zebra-network",
+ "zebra-node-services",
"zebra-state",
+ "zebrad",
]
[[package]]
diff --git a/Makefile b/Makefile
index 3d4fc069d46..d5f10b8d913 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,7 @@ help:
@echo " perf-replay-worker Replay through the real write worker (one rung up)"
@echo " perf-replay-verifier Replay through the checkpoint verifier (adds PoW + Merkle)"
@echo " perf-replay-sequencer Replay through the Zakura block-sync Sequencer (VCT-only,"
- @echo " pruned by default; REPLAY_ARCHIVE=1 for archive, REPLAY_TRACE_DIR= for traces)"
+ @echo " pruned by default; REPLAY_ARCHIVE=1 archive, REPLAY_STOP_HEIGHT= sub-range, REPLAY_TRACE_DIR= traces)"
@echo " perf-analyze Bottleneck attribution over the CSV window (PERF_LABEL/PERF_LO/PERF_HI)"
@echo " perf-dashboard Live metrics dashboard for the running bench node"
@echo " perf-verify-isolation Confirm the bench sees only the two cohort peers"
diff --git a/deploy/runner/replay_run.sh b/deploy/runner/replay_run.sh
index 546ee9d9565..f624aa00100 100755
--- a/deploy/runner/replay_run.sh
+++ b/deploy/runner/replay_run.sh
@@ -28,11 +28,16 @@ RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DBREL="${BENCH_DB_REL:-state/v27/mainnet}"
SRC="${REPLAY_SRC:-/mnt/roman-dev-2-data/zebra-cache}"
-BASE_SRC="${REPLAY_BASE_SRC:-/mnt/roman-dev-2-data/zebra-ckpt-1800000-warm}"
-CACHE="${REPLAY_CACHE:-/mnt/roman-dev-2-data/win-fwd.zrb}"
-SIDECAR="${REPLAY_SIDECAR:-/mnt/roman-dev-2-data/win-fwd.vct}"
-START="${REPLAY_START:-1802001}"
-END="${REPLAY_END:-1832000}"
+# Base = the cleanly-pruned snapshot at tip 1849957 (START-1). It was produced by
+# replaying the older 1800000 pruned base forward past 1849957 so the checkpoint
+# raw-tx archive backlog is already drained: forking it for a `run` does NOT
+# trigger a startup DrainBacklog (the earlier 1800000-warm-pruned base did, which
+# polluted the first ~10K blocks of every run with delete/compaction churn).
+BASE_SRC="${REPLAY_BASE_SRC:-/mnt/roman-dev-2-data/zebra-ckpt-1850000-warm-pruned}"
+CACHE="${REPLAY_CACHE:-/mnt/roman-dev-2-data/win-1850k.zrb}"
+SIDECAR="${REPLAY_SIDECAR:-/mnt/roman-dev-2-data/win-1850k.vct}"
+START="${REPLAY_START:-1849958}"
+END="${REPLAY_END:-1899957}"
FORK_DIR="${BENCH_FORK_DIR:-/mnt/roman-dev-2-data}"
BIN_DEFAULT="${REPLAY_BIN:-/root/wal-bench/zebra-replay-bench}"
@@ -158,6 +163,12 @@ cmd_run_sequencer() {
else
note "storage mode: pruned (default)"
fi
+ # Clamp the committed window to a sub-range of the cache (stops at the last
+ # checkpoint <= REPLAY_STOP_HEIGHT). Lets a smaller window reuse a larger cache.
+ if [ -n "${REPLAY_STOP_HEIGHT:-}" ]; then
+ note "stop height: $REPLAY_STOP_HEIGHT"
+ args+=(--stop-height "$REPLAY_STOP_HEIGHT")
+ fi
# Structured Zakura JSONL traces, like perf-run-mainnet's [network.zakura] trace_dir.
if [ -n "${REPLAY_TRACE_DIR:-}" ]; then
note "Zakura JSONL traces -> $REPLAY_TRACE_DIR"
diff --git a/docs/design/deferred-transparent-reconcile.md b/docs/design/deferred-transparent-reconcile.md
new file mode 100644
index 00000000000..9f98638a65e
--- /dev/null
+++ b/docs/design/deferred-transparent-reconcile.md
@@ -0,0 +1,136 @@
+# Deferred transparent reconcile (prototype)
+
+Status: **prototype, default-off, opt-in.** Byte-match-verified with lifecycle guards
+(height gate, handoff drain barrier, format-check + clean-shutdown). Benchmark-only: safe on
+a **disposable** below-checkpoint node; crash recovery + RPC guards are still missing (see
+"Production-readiness gaps (remaining)").
+
+## Problem
+
+In the checkpoint-trusted fast-sync range, the finalized committer's dominant cost in
+high transparent-churn regions (the 2022 "sandblast" consolidation spam) is **spent-UTXO
+resolution**: for each spent transparent input the committer reads `tx_loc_by_hash`
+(txid → location) and `utxo_by_out_loc` (location → value), then deletes the entry and
+debits the value pool. In the sandblast range this is ~439 cold, random reads per block,
+all on the per-block commit critical path.
+
+These reads are not cacheable: ~97% of spends in that range reference UTXOs created
+>4096 blocks earlier, so there is no temporal locality. A ceiling probe (skip the spend
+work entirely) measured **sandblast throughput 124 → 383 blk/s (+209%)** — i.e. the spend
+resolution *is* the entire sandblast penalty; with it removed a sandblast block runs at
+the normal (light-region) rate.
+
+## Design
+
+The reads can't be removed (consensus needs the spent value + location), but they can be
+**deferred off the per-block path and batched on a separate worker**:
+
+- **Per block** (in the deferred range): the committer does not resolve spent UTXOs. It
+ records the spent outpoints + the block into an in-memory window, writes the UTXO
+ *creates* + headers + nullifiers + the VCT root fold inline, and skips the spent-side of
+ the value pool. (The auxiliary address index is already skipped in pruned mode.)
+- **Every N blocks** (`defer_reconcile_interval`, not per checkpoint — mainnet checkpoints
+ are only ~30–40 blocks apart, too frequent to amortize): a **reconcile** resolves the
+ window's spent outpoints from disk (sorted/deduped), recomputes the value-pool delta, and
+ writes the deletes + value pool (`chain_value_pools` tip + per-height `BlockInfo`) in one
+ atomic batch.
+- **v2** runs the reconcile on a dedicated **worker thread** with a capacity-1 handoff
+ channel, so the (CPU-bound) resolution + value-pool recompute + write overlap with
+ continued block assembly. The worker owns the running value pool exclusively (the
+ assembler defers it), processes windows FIFO (so the value-pool chaining stays ordered),
+ and writes only keys disjoint from the assembler's (deletes + value pool vs. creates), so
+ there are no shared-state races or key conflicts.
+
+### Correctness invariants
+
+1. **Older spends are durable at reconcile time.** A UTXO is deleted only when its
+ spender's window reconciles; each UTXO is spent once, so it is never deleted earlier, and
+ by reconcile time all the window's blocks are flushed.
+2. **Lazy deletes are safe.** Between reconciles `utxo_by_out_loc` is a transient *superset*;
+ the only reader is the next reconcile, which knows the entries are spent. A stale entry
+ would only be re-read by a double-spend, impossible in a checkpoint-valid chain.
+3. **Deferring the value pool is safe in the checkpoint range.** It is consensus state but
+ only *checked* above `max_checkpoint_height` (the semantic verifier). The per-block
+ value-pool change is additive, so the interval delta is the sum of per-block deltas.
+
+## Results
+
+Benchmarked offline via `zebra-replay-bench` on a mainnet 1.85M–1.9M pruned snapshot
+(includes the sandblast region), run-ahead pipeline depth 8, interval 2000.
+
+- **Byte-match: identical.** The deferred run (both the v1 inline path and the v2 worker)
+ produces a **byte-identical value pool, UTXO set (`utxo_by_out_loc`, ~20.0M entries), and
+ per-height `BlockInfo`** as the non-deferred run, verified with the `cf-dump` digests at a
+ deterministic checkpoint stop. The block-hash checkpoint gate also passes.
+- **Throughput (sandblast region):**
+ | variant | sandblast | overall |
+ |---|---|---|
+ | non-deferred baseline | 122 | 213 blk/s |
+ | v1 per-checkpoint (~38-block) | −51% | −43% |
+ | v1 interval=2000 inline | +2% | +3% |
+ | **v2 interval=2000 worker** | **+73%** | **+43%** |
+
+ v2 recovers ~34% of the +209% ceiling. The remaining gap is that the single-threaded
+ worker is now the bottleneck (it re-traverses every transaction of each block for the
+ value-pool recompute), so sandblast stays below the light-region rate.
+
+## Gating
+
+Default off. Opt-in, reachable by a real node for benchmarking:
+
+- `zebra-state` `Config` (settable under `[state]` in `zebrad.toml`):
+ `defer_transparent_reconcile` (off), `defer_reconcile_interval` (0 = per checkpoint).
+ `defer_reconcile_inline` (force the v1 inline path) stays `#[serde(skip)]` (bench A/B only).
+- `Config::defers_transparent_spends_at(network, height)` requires
+ `defer_transparent_reconcile && skip_address_index()` (pruned + checkpoint-sync) **and**
+ `height <= max_checkpoint_height`. `Config::defer_reconcile_configured()` is the
+ height-independent lifecycle predicate (worker spawn / drain triggers).
+- `zebra-replay-bench`: `ZRB_DEFER_TRANSPARENT`, `ZRB_RECONCILE_INTERVAL`,
+ `ZRB_RECONCILE_INLINE`, `ZRB_STOP_AT_HEIGHT`.
+
+## Lifecycle guards (implemented)
+
+The reconcile *logic* is byte-match-verified; these lifecycle guards make it safe to
+benchmark on a real below-checkpoint node:
+
+1. **Reachability.** `defer_transparent_reconcile` + `defer_reconcile_interval` are serde
+ fields on the state config, so a node can opt in from `[state]`. Default off.
+2. **Height-bound gate.** `defers_transparent_spends_at` returns false above
+ `max_checkpoint_height`, so every above-checkpoint block commits inline (non-deferred).
+ Unit-tested (`defers_transparent_spends_only_in_checkpoint_range`).
+3. **Handoff drain barrier (consensus-critical).** Before the first block above
+ `max_checkpoint_height` commits, the finalized committer flushes every in-flight block to
+ disk, drains the pending reconcile window (`flush_and_join` the worker), and refreshes the
+ pipeline's threaded value pool from the now-current disk pool — so the semantic verifier
+ never reads a stale/superset UTXO set or lagging value pool. The common handoff (the
+ checkpoint→non-finalized channel close) drains the same way.
+4. **Format check + clean shutdown.** The background `check_new_blocks` skips the new-blocks
+ validation while deferral is configured and the tip is in the deferred range (the
+ `BlockInfo`/value-pool lag there is expected, and the address index is off). The reconcile
+ worker is `flush_and_join`'d at the handoff and again via a `Drop` safety net, so a clean
+ stop finishes any queued reconcile before teardown.
+
+## Production-readiness gaps (remaining)
+
+1. **No crash recovery.** The reconciled (durable transparent) tip trails the committed tip.
+ A crash mid-window leaves `utxo_by_out_loc` a superset and `BlockInfo`/value pool lagging;
+ restart would need to re-derive the pending window. Not implemented — a node enabling this
+ must be treated as **disposable / re-snapshottable**.
+2. **No RPC guards.** Mid-window the UTXO set is a superset and the value pool lags, so
+ address/utxo/value RPCs would return wrong results in the deferred range. Not guarded.
+
+Until crash recovery + RPC guards are implemented, this remains a benchmarking feature for a
+disposable below-checkpoint node, not a general-purpose production mode.
+
+## Benchmarking on a real below-checkpoint node
+
+1. Snapshot a pruned + checkpoint-sync node whose tip is well below `max_checkpoint_height`
+ (mainnet ~3.36M) — the node must be **disposable** (re-snapshot per run; no crash
+ recovery).
+2. In `zebrad.toml` `[state]`: set `defer_transparent_reconcile = true` and
+ `defer_reconcile_interval = 2000` (pruned storage + checkpoint sync are required for the
+ address index to be off, which the gate needs).
+3. Set `debug_stop_at_height` to a height still below `max_checkpoint_height` so the run
+ stops inside the deferred range (the handoff barrier is exercised by code/tests, not by
+ the bench range).
+4. Avoid address/utxo/value RPCs against the node while it is in the deferred range.
diff --git a/make/perf.mk b/make/perf.mk
index f55226e626c..194a3ff0a34 100644
--- a/make/perf.mk
+++ b/make/perf.mk
@@ -114,6 +114,7 @@ perf-replay-verifier:
# requires REPLAY_VCT_SIDECAR. Commits in Pruned storage mode by default (BASE_SRC must
# be a pruned snapshot), matching the production mainnet config. Optional knobs:
# REPLAY_ARCHIVE=1 commit in Archive storage mode instead (needs an archive base)
+# REPLAY_STOP_HEIGHT= stop at the last checkpoint <= h (bench a sub-range of the cache)
# REPLAY_TRACE_DIR= write structured Zakura JSONL traces (block_sync.jsonl),
# the same tables perf-run-mainnet emits via trace_dir; plot
# with .cursor/skills/zakura-trace-plots
diff --git a/scripts/zebra-metrics-dashboard.py b/scripts/zebra-metrics-dashboard.py
index 2b4fb17a310..963d479bc29 100644
--- a/scripts/zebra-metrics-dashboard.py
+++ b/scripts/zebra-metrics-dashboard.py
@@ -103,6 +103,15 @@ def quantile(m, name, q):
("Zakura", "zk_peers", "Cohort peers (active)", "", "gauge"),
("Zakura", "zk_qdepth", "Zakura queue depth", "", "gauge"),
("Zakura", "zk_block_sync", "block_sync streams accepted", "", "gauge"),
+ # Download supply: received-body rate + whether a peer can serve the floor.
+ ("Download", "recv_s", "Blocks received / s", "blk/s", "rate"),
+ ("Download", "floor_avail_peers", "Floor-gap available peers", "", "gauge"),
+ ("Download", "floor_servable_peers","Floor-gap servable peers", "", "gauge"),
+ # Peer churn / connectivity: high accepted+closed with flat active = churn;
+ # dial failures = discovery/connectivity trouble.
+ ("Peers", "conn_new_s", "Connections accepted / s", "/s", "rate"),
+ ("Peers", "conn_closed_s", "Connections closed / s", "/s", "rate"),
+ ("Peers", "dial_fail_s", "Dial failures / s", "/s", "rate"),
# Apply-queue depth + floor-gap attribution: separates HOL download stalls
# from the sequencer→committer handoff ("glue") from peer-supply starvation.
("Apply queue","applied_s", "Submitted blocks / s", "/s", "rate"),
@@ -360,6 +369,12 @@ def _tick(self):
_reorder_bytes = bare(m, "sync_block_reorder_buffered_bytes")
d["reorder_mb"] = (_reorder_bytes / 1e6) if _reorder_bytes is not None else None
d["outstanding"] = bare(m, "sync_block_outstanding")
+ # Download-supply health at the sync floor: whether any peer can currently
+ # serve the next-needed block. `available` dropping to 0 while `servable`
+ # is non-zero is the head-of-line-on-the-floor signature (peers exist but
+ # none is dispatchable — the stall we saw at ~9 blk/s).
+ d["floor_avail_peers"] = bare(m, "sync_block_floor_gap_available_peers")
+ d["floor_servable_peers"] = bare(m, "sync_block_floor_gap_servable_peers")
cur = {
"h": bare(m, "state_finalized_block_height"),
@@ -371,6 +386,12 @@ def _tick(self):
"txid_s": bare(m, "zebra_state_prepare_txid_auth_digest_duration_seconds_sum"),
"bc_s": bare(m, "zebra_state_rocksdb_batch_commit_duration_seconds_sum"),
"bc_c": bare(m, "zebra_state_rocksdb_batch_commit_duration_seconds_count"),
+ # Download supply + peer-churn counters (rates computed below). `total`
+ # so a labeled series still aggregates.
+ "recv": total(m, "sync_block_body_received"),
+ "conn_acc": total(m, "zakura_p2p_conn_accepted"),
+ "conn_closed": total(m, "zakura_p2p_conn_closed_neutral"),
+ "dial_fail": total(m, "zakura_p2p_discovery_dial_failed"),
}
# Maintain the trailing height window for smoothed throughput.
if cur["h"] is not None:
@@ -413,6 +434,12 @@ def busy_pct(*names):
d["vct_fast_s"] = rate(p["vf"], cur["vf"], dt)
d["vct_prevalid_s"] = rate(p["vp"], cur["vp"], dt)
d["applied_s"] = rate(p["sub"], cur["sub"], dt)
+ # Download supply rate + peer churn (connections opened/closed per sec,
+ # dial failures per sec) — to tell "slow/few peers" from a commit stall.
+ d["recv_s"] = rate(p["recv"], cur["recv"], dt)
+ d["conn_new_s"] = rate(p["conn_acc"], cur["conn_acc"], dt)
+ d["conn_closed_s"] = rate(p["conn_closed"], cur["conn_closed"], dt)
+ d["dial_fail_s"] = rate(p["dial_fail"], cur["dial_fail"], dt)
d["p_auth_root"] = ms_per_block(p["aroot_s"], cur["aroot_s"], p["h"], cur["h"])
d["p_ordered_out"] = ms_per_block(p["ordered_s"], cur["ordered_s"], p["h"], cur["h"])
d["p_txid_digest"] = ms_per_block(p["txid_s"], cur["txid_s"], p["h"], cur["h"])
diff --git a/zebra-chain/src/lib.rs b/zebra-chain/src/lib.rs
index 9721ea7df19..d84eb49704c 100644
--- a/zebra-chain/src/lib.rs
+++ b/zebra-chain/src/lib.rs
@@ -38,6 +38,7 @@ pub mod sapling;
pub mod serialization;
pub mod shutdown;
pub mod sprout;
+pub mod stage_timing;
pub mod subtree;
pub mod transaction;
pub mod transparent;
diff --git a/zebra-chain/src/stage_timing.rs b/zebra-chain/src/stage_timing.rs
new file mode 100644
index 00000000000..23b71601141
--- /dev/null
+++ b/zebra-chain/src/stage_timing.rs
@@ -0,0 +1,79 @@
+//! Optional per-height pipeline-stage timing trace for offline profiling.
+//!
+//! When `ZEBRA_STAGE_TIMING_TRACE=` is set, [`record`] appends one JSON row
+//! per call, pairing a block `height` with a named pipeline `stage` and a wall-clock
+//! timestamp (microseconds since the Unix epoch). Because the timestamp is wall-clock
+//! and the facility is process-global, callers in different crates and threads (the
+//! apply driver, the consensus verifier, the state service, the write worker) all
+//! land on the same timeline, so an offline join by `height` localizes where a block
+//! spends its time between stages.
+//!
+//! This lives in `zebra-chain` only because it is the common crate every stage
+//! depends on; it has no consensus role. The hot path is a single `OnceLock` load
+//! when the env var is unset (the default), so it stays inert in production.
+
+use std::{
+ fs::File,
+ io::{BufWriter, Write},
+ sync::{Mutex, OnceLock},
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+static TRACE: OnceLock>>> = OnceLock::new();
+
+fn writer() -> Option<&'static Mutex>> {
+ TRACE
+ .get_or_init(|| {
+ let path = std::env::var_os("ZEBRA_STAGE_TIMING_TRACE")?;
+ match File::create(&path) {
+ Ok(file) => Some(Mutex::new(BufWriter::new(file))),
+ Err(_) => None,
+ }
+ })
+ .as_ref()
+}
+
+/// Whether stage timing is active (`ZEBRA_STAGE_TIMING_TRACE` is set). Lets callers
+/// skip building trace-only aggregates on the hot path when tracing is off.
+pub fn enabled() -> bool {
+ writer().is_some()
+}
+
+/// Records that block `height` reached pipeline `stage`, with a wall-clock
+/// (epoch-microsecond) timestamp, when `ZEBRA_STAGE_TIMING_TRACE` is set.
+///
+/// A cheap no-op (one `OnceLock` load) when the env var is unset. The row is small
+/// and carries no re-serialized block data, so the per-call cost is a timestamp read
+/// plus a buffered write — negligible relative to the stages it measures.
+pub fn record(height: u32, stage: &str) {
+ write_row(height, stage, None);
+}
+
+/// Like [`record`], but also tags the row with a numeric `value` (e.g. an in-flight
+/// queue depth at this stage), under the JSON key `"val"`.
+pub fn record_val(height: u32, stage: &str, value: u64) {
+ write_row(height, stage, Some(value));
+}
+
+fn write_row(height: u32, stage: &str, value: Option) {
+ let Some(writer) = writer() else { return };
+ let ts_us = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map(|d| d.as_micros())
+ .unwrap_or(0);
+ if let Ok(mut file) = writer.lock() {
+ // Flush each row so a killed/partial run is still inspectable; the write is a
+ // tiny line (no block re-serialization), so this stays off the critical path.
+ let _ = match value {
+ Some(v) => writeln!(
+ file,
+ r#"{{"height":{height},"stage":"{stage}","ts_us":{ts_us},"val":{v}}}"#
+ ),
+ None => writeln!(
+ file,
+ r#"{{"height":{height},"stage":"{stage}","ts_us":{ts_us}}}"#
+ ),
+ };
+ let _ = file.flush();
+ }
+}
diff --git a/zebra-consensus/src/block/request.rs b/zebra-consensus/src/block/request.rs
index d03abc2de26..284f15da1b1 100644
--- a/zebra-consensus/src/block/request.rs
+++ b/zebra-consensus/src/block/request.rs
@@ -6,7 +6,7 @@ use zebra_chain::{
block::{self, Block},
parameters::Network,
};
-use zebra_state::CheckpointVerifiedBlock;
+use zebra_state::{AuthenticatedCheckpointHash, CheckpointVerifiedBlock};
use crate::checkpoint::VerifyCheckpointError;
@@ -26,6 +26,25 @@ pub enum Request {
/// which can build these blocks concurrently across many download tasks.
CommitCheckpointPrecomputed(CheckpointVerifiedBlock),
+ /// Commits a checkpoint-range block whose expected hash has already been
+ /// authenticated against the hardcoded checkpoint list by Zakura header sync.
+ ///
+ /// The verifier validates the block in isolation (proof of work, Merkle root,
+ /// height) and asserts `block.hash() == expected_hash`, then releases it to the
+ /// state commit pipeline immediately — it does **not** accumulate or walk the
+ /// checkpoint range. `expected_hash` is an [`AuthenticatedCheckpointHash`], a
+ /// provenance token the caller can only obtain from the state's authenticated
+ /// header frontier, so this request cannot be forged from a raw block hash.
+ ///
+ /// Only valid at or below the checkpoint height. There is no fallback: a height
+ /// above the checkpoint, or a hash mismatch, is a hard invariant violation.
+ CommitCheckpointAuthenticated {
+ /// The downloaded block body.
+ block: Arc,
+ /// The checkpoint-authenticated expected hash for this height.
+ expected_hash: AuthenticatedCheckpointHash,
+ },
+
/// Performs semantic validation but skips checking proof of work,
/// then asks the state to perform contextual validation.
/// Does not commit the block to the state.
@@ -81,6 +100,7 @@ impl Request {
match self {
Request::Commit(block) => Arc::clone(block),
Request::CommitCheckpointPrecomputed(block) => Arc::clone(&block.block),
+ Request::CommitCheckpointAuthenticated { block, .. } => Arc::clone(block),
Request::CheckProposal(block) => Arc::clone(block),
}
}
@@ -88,7 +108,9 @@ impl Request {
/// Returns `true` if the request is a proposal
pub fn is_proposal(&self) -> bool {
match self {
- Request::Commit(_) | Request::CommitCheckpointPrecomputed(_) => false,
+ Request::Commit(_)
+ | Request::CommitCheckpointPrecomputed(_)
+ | Request::CommitCheckpointAuthenticated { .. } => false,
Request::CheckProposal(_) => true,
}
}
diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs
index 31de33bfbf3..ac6c7700a83 100644
--- a/zebra-consensus/src/checkpoint.rs
+++ b/zebra-consensus/src/checkpoint.rs
@@ -37,7 +37,7 @@ use zebra_chain::{
},
work::equihash,
};
-use zebra_state::{self as zs, CheckpointVerifiedBlock};
+use zebra_state::{self as zs, AuthenticatedCheckpointHash, CheckpointVerifiedBlock};
use crate::{
block::VerifyBlockError,
@@ -603,14 +603,24 @@ where
// Cheap proof-of-work checks run *before* the expensive precomputation,
// so a flood of invalid-PoW blocks can't make us do per-transaction work.
+ let pow_start = std::time::Instant::now();
self.check_proof_of_work(&block.header, height, hash)?;
+ let pow = pow_start.elapsed();
// Precompute the per-transaction hashes and auth data root, which scale
// with block weight. (The precomputed path does this concurrently in the
// caller and skips it here.)
+ let precompute_start = std::time::Instant::now();
let block = CheckpointVerifiedBlock::with_hash(block, hash);
+ let precompute = precompute_start.elapsed();
+ metrics::histogram!("zebra.consensus.checkpoint.precompute.duration_seconds")
+ .record(precompute.as_secs_f64());
- self.finish_validation(block)
+ let merkle_start = std::time::Instant::now();
+ let block = self.finish_validation(block)?;
+ crate::verify_timing::record(height.0, pow, precompute, merkle_start.elapsed());
+
+ Ok(block)
}
/// Check a [`CheckpointVerifiedBlock`] whose precomputation (txids, auth data
@@ -636,6 +646,7 @@ where
height: block::Height,
hash: block::Hash,
) -> Result<(), VerifyCheckpointError> {
+ let pow_start = std::time::Instant::now();
if self.network.disable_pow() {
crate::block::check::difficulty_threshold_is_valid(
header,
@@ -647,6 +658,8 @@ where
crate::block::check::difficulty_is_valid(header, &self.network, &height, &hash)?;
crate::block::check::equihash_solution_is_valid(header)?;
}
+ metrics::histogram!("zebra.consensus.checkpoint.pow.duration_seconds")
+ .record(pow_start.elapsed().as_secs_f64());
Ok(())
}
@@ -671,11 +684,14 @@ where
.map(DeferredPoolBalanceChange::new),
);
+ let merkle_start = std::time::Instant::now();
crate::block::check::merkle_root_validity(
&self.network,
&block.block,
&block.transaction_hashes,
)?;
+ metrics::histogram!("zebra.consensus.checkpoint.merkle.duration_seconds")
+ .record(merkle_start.elapsed().as_secs_f64());
Ok(block)
}
@@ -900,6 +916,108 @@ where
.boxed()
}
+ /// Verify and release a Zakura header-authenticated checkpoint block.
+ ///
+ /// This is the narrow fast path: it validates the block in isolation (height, proof of work,
+ /// Merkle root) and asserts the body matches the checkpoint-authenticated header hash, then
+ /// releases it straight to the state commit pipeline. It does **not** enqueue the block, walk
+ /// the checkpoint range, or touch `verifier_progress`.
+ ///
+ /// Zakura checkpoint heights always take this path (there is no fallback into range
+ /// accumulation), so the verifier's range/progress state is never consulted for them; the
+ /// `check_height` bound stays correct because progress only ever lags the committed tip, which
+ /// is lenient (no false `AlreadyVerified`). The router enforces that `block` is at or below the
+ /// checkpoint height before calling this.
+ pub(crate) fn call_authenticated(
+ &mut self,
+ block: Arc,
+ expected_hash: AuthenticatedCheckpointHash,
+ ) -> Pin> + Send + 'static>>
+ {
+ zebra_chain::stage_timing::record(
+ block.coinbase_height().map_or(0, |h| h.0),
+ "verifier_call_start",
+ );
+ // Per-block validity checks (height, proof of work, Merkle root), the same checks the
+ // accumulation path runs per block — defense in depth on top of the authenticated header.
+ let block = match self.check_block(block) {
+ Ok(block) => block,
+ Err(e) => return async move { Err(e) }.boxed(),
+ };
+ zebra_chain::stage_timing::record(block.height.0, "verifier_checked");
+
+ // Bind the body to the authenticated header. A mismatch means the downloaded body and the
+ // checkpoint-authenticated header disagree: a hard invariant violation, never recoverable.
+ let expected = expected_hash.hash();
+ if block.hash != expected {
+ let (height, found) = (block.height, block.hash);
+ return async move {
+ Err(VerifyCheckpointError::AuthenticatedHashMismatch {
+ height,
+ expected,
+ found,
+ })
+ }
+ .boxed();
+ }
+
+ self.commit_authenticated_block(block)
+ }
+
+ /// Release an already-validated, authenticated checkpoint block straight to the state commit
+ /// pipeline.
+ ///
+ /// Mirrors the commit/reset behavior of [`Self::verify_and_commit`] but without the range
+ /// release channel: the block is committed directly (DB mutation stays parent-ordered in the
+ /// state write task), and on a commit error the verifier is reset to the state tip.
+ fn commit_authenticated_block(
+ &mut self,
+ block: CheckpointVerifiedBlock,
+ ) -> Pin> + Send + 'static>>
+ {
+ let hash = block.hash;
+
+ let state_service = self.state_service.clone();
+ let commit_checkpoint_verified = tokio::spawn(async move {
+ match state_service
+ .oneshot(zs::Request::CommitCheckpointVerifiedBlock(block))
+ .map_err(VerifyCheckpointError::CommitCheckpointVerified)
+ .await?
+ {
+ zs::Response::Committed(committed_hash) => {
+ assert_eq!(committed_hash, hash, "state must commit correct hash");
+ Ok(hash)
+ }
+ _ => unreachable!("wrong response for CommitCheckpointVerifiedBlock"),
+ }
+ });
+
+ let state_service = self.state_service.clone();
+ let reset_sender = self.reset_sender.clone();
+ async move {
+ let result = commit_checkpoint_verified.await;
+ let result = if zebra_chain::shutdown::is_shutting_down() {
+ Err(VerifyCheckpointError::ShuttingDown)
+ } else {
+ result.expect("commit_checkpoint_verified should not panic")
+ };
+ if result.is_err() {
+ // Keep the verifier consistent with the state for any later legacy caller.
+ let tip = match state_service
+ .oneshot(zs::Request::Tip)
+ .await
+ .map_err(VerifyCheckpointError::Tip)?
+ {
+ zs::Response::Tip(tip) => tip,
+ _ => unreachable!("wrong response for Tip"),
+ };
+ let _ = reset_sender.send(tip);
+ }
+ result
+ }
+ .boxed()
+ }
+
/// During checkpoint range processing, process all the blocks at `height`.
///
/// Returns the first valid block. If there is no valid block, returns None.
@@ -1203,6 +1321,15 @@ pub enum VerifyCheckpointError {
},
#[error("zebra is shutting down")]
ShuttingDown,
+ #[error(
+ "authenticated checkpoint body at {height:?} has hash {found:?}, \
+ but the authenticated header hash is {expected:?}"
+ )]
+ AuthenticatedHashMismatch {
+ height: block::Height,
+ expected: block::Hash,
+ found: block::Hash,
+ },
}
impl From for VerifyCheckpointError {
diff --git a/zebra-consensus/src/checkpoint/tests.rs b/zebra-consensus/src/checkpoint/tests.rs
index 098e7a61426..4e6b06ede68 100644
--- a/zebra-consensus/src/checkpoint/tests.rs
+++ b/zebra-consensus/src/checkpoint/tests.rs
@@ -94,6 +94,87 @@ async fn single_item_checkpoint_list() -> Result<(), Report> {
Ok(())
}
+#[tokio::test(flavor = "multi_thread")]
+async fn authenticated_checkpoint_release_test() -> Result<(), Report> {
+ let _init_guard = zebra_test::init();
+
+ let block0 =
+ Arc::::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..])?;
+ let hash0 = block0.hash();
+
+ let genesis_checkpoint_list: BTreeMap =
+ [(block0.coinbase_height().unwrap(), hash0)]
+ .iter()
+ .cloned()
+ .collect();
+
+ let state_service = zebra_state::init_test(&Mainnet).await;
+ let mut checkpoint_verifier =
+ CheckpointVerifier::from_list(genesis_checkpoint_list, &Mainnet, None, state_service)
+ .map_err(|e| eyre!(e))?;
+
+ // The authenticated fast path commits the block against its authenticated hash, with no other
+ // height queued, and without touching the range/progress state.
+ let token = zebra_state::AuthenticatedCheckpointHash::new_for_test(hash0);
+ let response = timeout(
+ Duration::from_secs(VERIFY_TIMEOUT_SECONDS),
+ checkpoint_verifier.call_authenticated(block0.clone(), token),
+ )
+ .await
+ .expect("timeout should not happen")
+ .expect("authenticated block should commit");
+
+ assert_eq!(response, hash0);
+ // The fast path must not enqueue anything or walk/advance the checkpoint range state.
+ assert!(checkpoint_verifier.queued.is_empty());
+ assert_eq!(
+ checkpoint_verifier.previous_checkpoint_height(),
+ BeforeGenesis
+ );
+
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn authenticated_checkpoint_hash_mismatch_test() -> Result<(), Report> {
+ let _init_guard = zebra_test::init();
+
+ let block0 =
+ Arc::::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..])?;
+ let hash0 = block0.hash();
+ let block1 = Arc::::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_1_BYTES[..])?;
+ let wrong_hash = block1.hash();
+
+ let genesis_checkpoint_list: BTreeMap =
+ [(block0.coinbase_height().unwrap(), hash0)]
+ .iter()
+ .cloned()
+ .collect();
+
+ let state_service = zebra_state::init_test(&Mainnet).await;
+ let mut checkpoint_verifier =
+ CheckpointVerifier::from_list(genesis_checkpoint_list, &Mainnet, None, state_service)
+ .map_err(|e| eyre!(e))?;
+
+ // A body whose hash disagrees with the authenticated header is a hard invariant violation:
+ // there is no fallback, and nothing is released or queued.
+ let token = zebra_state::AuthenticatedCheckpointHash::new_for_test(wrong_hash);
+ let result = timeout(
+ Duration::from_secs(VERIFY_TIMEOUT_SECONDS),
+ checkpoint_verifier.call_authenticated(block0.clone(), token),
+ )
+ .await
+ .expect("timeout should not happen");
+
+ assert!(matches!(
+ result,
+ Err(VerifyCheckpointError::AuthenticatedHashMismatch { .. })
+ ));
+ assert!(checkpoint_verifier.queued.is_empty());
+
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn multi_item_checkpoint_list_test() -> Result<(), Report> {
multi_item_checkpoint_list().await
diff --git a/zebra-consensus/src/lib.rs b/zebra-consensus/src/lib.rs
index 30a81414975..858e288a3c5 100644
--- a/zebra-consensus/src/lib.rs
+++ b/zebra-consensus/src/lib.rs
@@ -38,6 +38,7 @@ mod block;
mod checkpoint;
mod primitives;
mod script;
+mod verify_timing;
pub mod config;
pub mod error;
diff --git a/zebra-consensus/src/router.rs b/zebra-consensus/src/router.rs
index 905702de21e..498ef67f228 100644
--- a/zebra-consensus/src/router.rs
+++ b/zebra-consensus/src/router.rs
@@ -211,6 +211,36 @@ where
.boxed();
}
+ // A Zakura header-authenticated checkpoint body: validate in isolation and release
+ // immediately, no range accumulation. Policy lives here — this is checkpoint-range only.
+ if let Request::CommitCheckpointAuthenticated {
+ block,
+ expected_hash,
+ } = request
+ {
+ let max_checkpoint_height = self.max_checkpoint_height;
+ return match block.coinbase_height() {
+ Some(height) if height <= max_checkpoint_height => self
+ .checkpoint
+ .call_authenticated(block, expected_hash)
+ .map_err(Into::into)
+ .boxed(),
+ Some(height) => async move {
+ Err(VerifyCheckpointError::TooHigh {
+ height,
+ max_height: max_checkpoint_height,
+ })?
+ }
+ .boxed(),
+ None => async move {
+ Err(VerifyCheckpointError::CoinbaseHeight {
+ hash: expected_hash.hash(),
+ })?
+ }
+ .boxed(),
+ };
+ }
+
let block = request.block();
match block.coinbase_height() {
diff --git a/zebra-consensus/src/verify_timing.rs b/zebra-consensus/src/verify_timing.rs
new file mode 100644
index 00000000000..92a63f1c655
--- /dev/null
+++ b/zebra-consensus/src/verify_timing.rs
@@ -0,0 +1,79 @@
+//! Optional per-block checkpoint-verify timing trace (off by default).
+//!
+//! When `ZEBRA_VERIFY_TIMING_TRACE=` is set, the checkpoint verifier appends a
+//! JSON row per block (or every `ZEBRA_VERIFY_TIMING_EVERY` blocks) with the verify
+//! sub-phase timings — proof-of-work (difficulty + equihash), precompute (per-tx txids
+//! + auth digest), and Merkle root — keyed by height, so an offline analysis can build
+//! a by-height verify breakdown and join it with the committer's per-block trace.
+//!
+//! The hot path is a single `OnceLock` load when the env var is unset (the production
+//! default), so this stays inert unless explicitly enabled (e.g. by the replay bench).
+
+use std::{
+ fs::File,
+ io::{BufWriter, Write},
+ sync::{
+ atomic::{AtomicU64, Ordering},
+ Mutex, OnceLock,
+ },
+ time::{Duration, Instant},
+};
+
+struct VerifyTimingTrace {
+ file: Mutex>,
+ every: u64,
+ count: AtomicU64,
+ start: Instant,
+}
+
+static TRACE: OnceLock> = OnceLock::new();
+
+fn trace() -> Option<&'static VerifyTimingTrace> {
+ TRACE
+ .get_or_init(|| {
+ let path = std::env::var_os("ZEBRA_VERIFY_TIMING_TRACE")?;
+ let every = std::env::var("ZEBRA_VERIFY_TIMING_EVERY")
+ .ok()
+ .and_then(|s| s.parse::().ok())
+ .unwrap_or(1)
+ .max(1);
+ match File::create(&path) {
+ Ok(file) => {
+ tracing::info!(?path, every, "verify-timing trace enabled");
+ Some(VerifyTimingTrace {
+ file: Mutex::new(BufWriter::new(file)),
+ every,
+ count: AtomicU64::new(0),
+ start: Instant::now(),
+ })
+ }
+ Err(error) => {
+ tracing::warn!(?path, %error, "could not open verify-timing trace");
+ None
+ }
+ }
+ })
+ .as_ref()
+}
+
+/// Records one verify-timing row (every `every` blocks). Cheap no-op (one `OnceLock`
+/// load) when `ZEBRA_VERIFY_TIMING_TRACE` is unset.
+pub(crate) fn record(height: u32, pow: Duration, precompute: Duration, merkle: Duration) {
+ let Some(trace) = trace() else { return };
+ let n = trace.count.fetch_add(1, Ordering::Relaxed);
+ if n % trace.every != 0 {
+ return;
+ }
+ let line = format!(
+ r#"{{"ts_us":{},"height":{},"pow_ms":{:.3},"precompute_ms":{:.3},"merkle_ms":{:.3}}}"#,
+ trace.start.elapsed().as_micros(),
+ height,
+ pow.as_secs_f64() * 1000.0,
+ precompute.as_secs_f64() * 1000.0,
+ merkle.as_secs_f64() * 1000.0,
+ );
+ if let Ok(mut file) = trace.file.lock() {
+ let _ = writeln!(file, "{line}");
+ let _ = file.flush();
+ }
+}
diff --git a/zebra-network/src/zakura/block_sync/bench.rs b/zebra-network/src/zakura/block_sync/bench.rs
index ff191e41f23..e4044e49dd5 100644
--- a/zebra-network/src/zakura/block_sync/bench.rs
+++ b/zebra-network/src/zakura/block_sync/bench.rs
@@ -37,18 +37,19 @@ use zebra_jsonl_trace::{JsonlTraceGuard, JsonlTracer};
use crate::zakura::{
trace::{block_sync_trace as bs_trace, BLOCK_SYNC_TABLE},
transport::ByteBudget,
- ZakuraPeerId, ZakuraTrace,
+ ServicePeerSnapshot, ZakuraBlockSyncCandidateState, ZakuraPeerId, ZakuraTrace,
};
use super::{
- events::{BlockApplyResult, BlockApplyToken, BlockSyncAction},
+ config::ZakuraBlockSyncConfig,
+ events::{BlockApplyResult, BlockApplyToken, BlockSyncAction, BlockSyncEvent},
reactor::{bs_insert_height, bs_insert_u64},
reorder::BufferedBlockBody,
sequencer::Sequencer,
sequencer_task::{
initial_view, SequencedBody, SequencerControlInput, SequencerTask, SequencerView,
},
- state::{BlockSyncFrontiers, ThroughputMeter},
+ state::{BlockSyncFrontiers, BlockSyncHandle, ThroughputMeter},
work_queue::WorkQueue,
};
@@ -71,6 +72,8 @@ pub struct BenchSubmit {
pub struct SequencerProgress {
/// Verified block tip (last committed height the sequencer knows about).
pub verified_tip: block::Height,
+ /// Hash of the verified block tip (the committed hash reported for `verified_tip`).
+ pub verified_hash: block::Hash,
/// Bodies buffered out-of-order in the reorder queue.
pub reorder_len: u64,
/// Bodies drained into the contiguous `applying` set.
@@ -207,6 +210,100 @@ impl BenchSequencerHandle {
pub fn into_parts(self) -> (BenchBodyFeeder, BenchSubmissions, BenchCommitter) {
(self.feeder, self.submissions, self.committer)
}
+
+ /// Production-driver split: returns the raw `BlockSyncAction` stream and an inert
+ /// [`BlockSyncHandle`] so the bench can drive the **real** block-sync apply driver
+ /// (`zebrad`'s `drive_block_sync_actions`) directly, instead of a hand-rolled
+ /// verify/commit loop. The driver reports completions through
+ /// [`BlockSyncHandle::send_control`]; with no reactor present, a spawned translator
+ /// forwards each `BlockApplyFinished` into the sequencer's `ApplyFinished` control
+ /// input — the exact hop `reactor::handle_block_apply_finished` performs in
+ /// production. Every other handle channel is inert: the bench feeds bodies directly,
+ /// so the driver never emits peer queries or reads peer/status/candidate state.
+ pub fn into_driver_parts(self) -> BenchDriverParts {
+ let BenchSequencerHandle {
+ feeder,
+ submissions,
+ committer,
+ } = self;
+
+ // Highest committed height the driver has reported, so the feed can backpressure
+ // on the *committed* tip (the submit window caps submission, but the sequencer's
+ // `applying` set drains all contiguous fed bodies unbounded — a fast feed would
+ // otherwise pile up the whole window in memory and OOM).
+ let committed_tip = Arc::new(AtomicU64::new(0));
+ let committed_tip_shim = committed_tip.clone();
+ let control = committer.control.clone();
+ let (lifecycle_tx, mut lifecycle_rx) = mpsc::unbounded_channel::();
+ tokio::spawn(async move {
+ while let Some(event) = lifecycle_rx.recv().await {
+ if let BlockSyncEvent::BlockApplyFinished {
+ token,
+ height,
+ hash,
+ result,
+ local_frontier,
+ } = event
+ {
+ committed_tip_shim.fetch_max(u64::from(height.0), Ordering::Relaxed);
+ if control
+ .send(SequencerControlInput::ApplyFinished {
+ token,
+ height,
+ hash,
+ result,
+ local_frontier,
+ })
+ .is_err()
+ {
+ break;
+ }
+ }
+ }
+ });
+
+ let config = ZakuraBlockSyncConfig::default();
+ let (events, _events_rx) = mpsc::channel(1);
+ let (_peers_tx, peers) = watch::channel(ServicePeerSnapshot::default());
+ let (_status_tx, status) = watch::channel(config.initial_status());
+ let (_candidates_tx, candidates) = watch::channel(ZakuraBlockSyncCandidateState::default());
+ let block_sync = BlockSyncHandle {
+ events,
+ lifecycle: lifecycle_tx,
+ peers,
+ status,
+ candidates,
+ routine_wiring: None,
+ };
+
+ BenchDriverParts {
+ feeder,
+ actions: submissions.actions,
+ block_sync,
+ committer,
+ committed_tip,
+ }
+ }
+}
+
+/// The driver-shaped split returned by [`BenchSequencerHandle::into_driver_parts`].
+///
+/// Hand `actions` + `block_sync` to the production `drive_block_sync_actions`, feed
+/// cached bodies through `feeder`, and use `committer` for progress/trace snapshots
+/// (it also keeps the sequencer task alive).
+pub struct BenchDriverParts {
+ /// Feeds cached bodies into the sequencer's reorder queue.
+ pub feeder: BenchBodyFeeder,
+ /// The ordered `SubmitBlock` (and other) actions the sequencer emits.
+ pub actions: mpsc::Receiver,
+ /// Inert handle the production driver reports completions through; its
+ /// `BlockApplyFinished` events are forwarded to the sequencer.
+ pub block_sync: BlockSyncHandle,
+ /// Progress/trace snapshots; retained to keep the sequencer task alive.
+ pub committer: BenchCommitter,
+ /// Highest committed height reported by the driver. The feed reads this to cap how
+ /// far ahead of the committed tip it runs, bounding the in-flight body backlog.
+ pub committed_tip: Arc,
}
impl BenchBodyFeeder {
@@ -315,11 +412,26 @@ impl BenchCommitter {
}
}
+ /// Awaits sequencer progress until the verified tip reaches `target` (or the
+ /// sequencer task ends). Used to detect completion when the production driver owns
+ /// the apply loop and the bench no longer sees individual commits.
+ pub async fn wait_for_verified_tip(&mut self, target: block::Height) {
+ loop {
+ if self.view.borrow().verified_tip >= target {
+ return;
+ }
+ if self.view.changed().await.is_err() {
+ return;
+ }
+ }
+ }
+
/// Latest progress snapshot from the sequencer view.
pub fn progress(&self) -> SequencerProgress {
let view = *self.view.borrow();
SequencerProgress {
verified_tip: view.verified_tip,
+ verified_hash: view.verified_hash,
reorder_len: view.reorder_len,
applying_len: view.applying_len,
committed_blocks_per_sec: view.committed_blocks_per_sec,
diff --git a/zebra-network/src/zakura/block_sync/mod.rs b/zebra-network/src/zakura/block_sync/mod.rs
index 30dc4eecf53..4f4241e61e3 100644
--- a/zebra-network/src/zakura/block_sync/mod.rs
+++ b/zebra-network/src/zakura/block_sync/mod.rs
@@ -54,8 +54,8 @@ mod work_queue;
#[cfg(feature = "internal-bench")]
pub use bench::{
- spawn_bench_sequencer, BenchBodyFeeder, BenchCommitter, BenchSequencerHandle, BenchSubmissions,
- BenchSubmit, SequencerProgress,
+ spawn_bench_sequencer, BenchBodyFeeder, BenchCommitter, BenchDriverParts, BenchSequencerHandle,
+ BenchSubmissions, BenchSubmit, SequencerProgress,
};
pub use config::{
BlockSyncStatus, CwndUnit, ZakuraBlockSyncConfig, DEFAULT_BS_MAX_SUBMITTED_BLOCK_APPLIES,
diff --git a/zebra-network/src/zakura/block_sync/reactor.rs b/zebra-network/src/zakura/block_sync/reactor.rs
index 3a9ac9bed38..14f1ea61c6e 100644
--- a/zebra-network/src/zakura/block_sync/reactor.rs
+++ b/zebra-network/src/zakura/block_sync/reactor.rs
@@ -1888,18 +1888,46 @@ impl BlockSyncReactor {
fn publish_metrics(&self) {
// These lossy casts are metrics-only gauges; consensus and scheduling
// continue to use the original integer values.
+ let view = self.last_view;
+
metrics::gauge!("sync.block.best_header_tip.height")
.set(self.state.best_header_tip.0 as f64);
- metrics::gauge!("sync.block.verified_tip.height").set(self.verified_block_tip.0 as f64);
+ metrics::gauge!("sync.block.verified_tip.height").set(view.verified_tip.0 as f64);
+ metrics::gauge!("sync.block.download_floor.height").set(view.download_floor.0 as f64);
+ metrics::gauge!("sync.block.commit_gap.height")
+ .set(view.download_floor.0.saturating_sub(view.verified_tip.0) as f64);
metrics::gauge!("sync.block.missing_bodies").set(self.state.needed_heights.len() as f64);
metrics::gauge!("sync.block.budget.reserved_bytes")
.set(self.state.budget.reserved() as f64);
+ metrics::gauge!("sync.block.reorder").set(view.reorder_len as f64);
+ metrics::gauge!("sync.block.reorder.buffered_blocks").set(view.reorder_len as f64);
metrics::gauge!("sync.block.reorder.buffered_bytes")
- .set(self.last_view.reorder_buffered_bytes as f64);
- metrics::gauge!("sync.block.applying").set(self.last_view.applying_len as f64);
+ .set(view.reorder_buffered_bytes as f64);
+ metrics::gauge!("sync.block.applying").set(view.applying_len as f64);
+ metrics::gauge!("sync.block.unsubmitted_applying")
+ .set(view.unsubmitted_applying_count as f64);
+ metrics::gauge!("sync.block.submitted_applying").set(view.submitted_applying_count as f64);
// Outstanding (unreceived in-flight) heights summed across peers from the
// registry (the routines own the per-peer outstanding now).
metrics::gauge!("sync.block.outstanding").set(self.registry.total_unreceived() as f64);
+
+ if let Some(floor_gap) = self.floor_gap_diagnostics(Instant::now()) {
+ metrics::gauge!("sync.block.floor_gap.height").set(floor_gap.height.0 as f64);
+ metrics::gauge!("sync.block.floor_gap.servable_peers")
+ .set(floor_gap.servable_peers as f64);
+ metrics::gauge!("sync.block.floor_gap.available_peers")
+ .set(floor_gap.available_peers as f64);
+ metrics::gauge!("sync.block.floor_gap.outstanding_peers")
+ .set(floor_gap.outstanding_peers as f64);
+ metrics::gauge!("sync.block.commit_frontier_stall.seconds").set(
+ floor_gap
+ .oldest_outstanding_ms
+ .map(|age| age as f64 / 1000.0)
+ .unwrap_or(0.0),
+ );
+ metrics::counter!("sync.block.floor_gap.state_ticks", "state" => floor_gap.state)
+ .increment(1);
+ }
}
fn clamp_served_block_count(&self, start_height: block::Height, count: u32) -> u32 {
diff --git a/zebra-network/src/zakura/block_sync/sequencer.rs b/zebra-network/src/zakura/block_sync/sequencer.rs
index cf2f1e22ada..e510ff25a0d 100644
--- a/zebra-network/src/zakura/block_sync/sequencer.rs
+++ b/zebra-network/src/zakura/block_sync/sequencer.rs
@@ -72,6 +72,16 @@ pub(super) struct Sequencer {
submitted_applies: BTreeMap>,
next_apply_token: BlockApplyToken,
+ /// Running totals over `applying`, maintained incrementally at every
+ /// insert/remove/submit-flag change so `publish_view` reads them in O(1)
+ /// instead of scanning `applying` (which holds the whole apply backlog) on
+ /// every body/control event. `unsubmitted` is derived as
+ /// `applying.len() - submitted_applying_count`. Tests assert these never drift
+ /// from a full scan.
+ applying_buffered_bytes: u64,
+ submitted_applying_count: usize,
+ submitted_applying_bytes: u64,
+
// The highest block height whose body has already been accepted into the contiguous
// download-apply pipeline.
body_download_floor: block::Height,
@@ -86,6 +96,9 @@ impl Sequencer {
applying: BTreeMap::new(),
submitted_applies: BTreeMap::new(),
next_apply_token: 1,
+ applying_buffered_bytes: 0,
+ submitted_applying_count: 0,
+ submitted_applying_bytes: 0,
body_download_floor: verified_block_tip,
verified_block_tip,
submitted_apply_limit,
@@ -126,6 +139,13 @@ impl Sequencer {
}
pub(super) fn applying_buffered_bytes(&self) -> u64 {
+ self.applying_buffered_bytes
+ }
+
+ /// Ground-truth recomputation of [`applying_buffered_bytes`], used by tests to
+ /// assert the maintained counter never drifts.
+ #[cfg(test)]
+ pub(super) fn applying_buffered_bytes_scanned(&self) -> u64 {
self.applying
.values()
.map(|applying| applying.bytes)
@@ -142,27 +162,38 @@ impl Sequencer {
}
pub(super) fn unsubmitted_applying_count(&self) -> usize {
+ // Derived: every applying body is either submitted or not.
self.applying
- .values()
- .filter(|applying| !applying.submitted)
- .count()
+ .len()
+ .saturating_sub(self.submitted_applying_count)
}
pub(super) fn submitted_applying_bytes(&self) -> u64 {
- self.applying
- .values()
- .filter_map(|applying| applying.submitted.then_some(applying.bytes))
- .fold(0u64, u64::saturating_add)
+ self.submitted_applying_bytes
}
/// Number of `applying` bodies already submitted to the verifier.
pub(super) fn submitted_applying_count(&self) -> usize {
+ self.submitted_applying_count
+ }
+
+ /// Ground-truth recomputations of the submitted-apply counters, for tests.
+ #[cfg(test)]
+ pub(super) fn submitted_applying_count_scanned(&self) -> usize {
self.applying
.values()
.filter(|applying| applying.submitted)
.count()
}
+ #[cfg(test)]
+ pub(super) fn submitted_applying_bytes_scanned(&self) -> u64 {
+ self.applying
+ .values()
+ .filter_map(|applying| applying.submitted.then_some(applying.bytes))
+ .fold(0u64, u64::saturating_add)
+ }
+
pub(super) fn has_submitted_apply(&self, height: block::Height, hash: block::Hash) -> bool {
self.submitted_applies
.get(&height)
@@ -296,6 +327,8 @@ impl Sequencer {
source_peer,
},
);
+ // New bodies enter `applying` unsubmitted, so only the total grows.
+ self.applying_buffered_bytes = self.applying_buffered_bytes.saturating_add(bytes);
}
covered
}
@@ -327,12 +360,20 @@ impl Sequencer {
.get(&height)
.map(|applying| applying.block.clone())?;
let token = self.next_apply_token();
- let applying = self.applying.get_mut(&height)?;
- applying.token = token;
- applying.submitted = true;
+ let (hash, bytes, was_submitted) = {
+ let applying = self.applying.get_mut(&height)?;
+ applying.token = token;
+ let was_submitted = applying.submitted;
+ applying.submitted = true;
+ (applying.hash, applying.bytes, was_submitted)
+ };
+ if !was_submitted {
+ self.submitted_applying_count = self.submitted_applying_count.saturating_add(1);
+ self.submitted_applying_bytes = self.submitted_applying_bytes.saturating_add(bytes);
+ }
Some(SubmitItem {
height,
- hash: applying.hash,
+ hash,
token,
block,
})
@@ -341,11 +382,23 @@ impl Sequencer {
/// Roll back a submit whose dispatch failed (only if the token still matches,
/// so a stale rollback cannot clobber a newer submission).
pub(super) fn unsubmit(&mut self, height: block::Height, token: BlockApplyToken) {
- if let Some(applying) = self.applying.get_mut(&height) {
- if applying.token == token {
- applying.token = 0;
- applying.submitted = false;
+ let unsubmitted_bytes = {
+ let Some(applying) = self.applying.get_mut(&height) else {
+ return;
+ };
+ if applying.token != token {
+ return;
}
+ // Only a currently-submitted body affects the submitted counters; if the
+ // matched token was already rolled back, just clear it.
+ let was_submitted = applying.submitted;
+ applying.token = 0;
+ applying.submitted = false;
+ was_submitted.then_some(applying.bytes)
+ };
+ if let Some(bytes) = unsubmitted_bytes {
+ self.submitted_applying_count = self.submitted_applying_count.saturating_sub(1);
+ self.submitted_applying_bytes = self.submitted_applying_bytes.saturating_sub(bytes);
}
}
@@ -422,7 +475,14 @@ impl Sequencer {
}
pub(super) fn remove_applying(&mut self, height: block::Height) -> Option {
- self.applying.remove(&height)
+ let removed = self.applying.remove(&height)?;
+ self.applying_buffered_bytes = self.applying_buffered_bytes.saturating_sub(removed.bytes);
+ if removed.submitted {
+ self.submitted_applying_count = self.submitted_applying_count.saturating_sub(1);
+ self.submitted_applying_bytes =
+ self.submitted_applying_bytes.saturating_sub(removed.bytes);
+ }
+ Some(removed)
}
/// After a rejected/timed-out apply at `height`, roll the download floor back
@@ -448,7 +508,7 @@ impl Sequencer {
.collect();
let mut released = 0u64;
for height in heights {
- if let Some(applying) = self.applying.remove(&height) {
+ if let Some(applying) = self.remove_applying(height) {
released = released.saturating_add(applying.bytes);
}
}
@@ -466,7 +526,7 @@ impl Sequencer {
self.clear_submitted_applies_through(tip);
let mut released = 0u64;
for height in applied {
- if let Some(applying) = self.applying.remove(&height) {
+ if let Some(applying) = self.remove_applying(height) {
released = released.saturating_add(applying.bytes);
}
}
@@ -521,6 +581,9 @@ impl Sequencer {
self.submitted_applies.clear();
}
self.applying.clear();
+ self.applying_buffered_bytes = 0;
+ self.submitted_applying_count = 0;
+ self.submitted_applying_bytes = 0;
released
}
}
diff --git a/zebra-network/src/zakura/block_sync/sequencer_task.rs b/zebra-network/src/zakura/block_sync/sequencer_task.rs
index 6c1963706ef..9bb71d2cdf1 100644
--- a/zebra-network/src/zakura/block_sync/sequencer_task.rs
+++ b/zebra-network/src/zakura/block_sync/sequencer_task.rs
@@ -738,6 +738,9 @@ impl SequencerTask {
let body_input_bytes = self
.body_input_bytes
.load(std::sync::atomic::Ordering::Relaxed);
+ // Cross-layer drift check: the independently-maintained `ByteBudget` total
+ // must equal the sum of the component counters. `work.reserved_bytes()` is
+ // now an O(1) counter, so this runs on every event without a work-queue scan.
let expected_budget = self
.work
.reserved_bytes()
diff --git a/zebra-network/src/zakura/block_sync/tests.rs b/zebra-network/src/zakura/block_sync/tests.rs
index d0213c1f849..e3da6e222b6 100644
--- a/zebra-network/src/zakura/block_sync/tests.rs
+++ b/zebra-network/src/zakura/block_sync/tests.rs
@@ -1447,6 +1447,102 @@ fn work_queue_force_cancel_and_owner_timeout_release_once() {
assert!(queue.pending_contains(block::Height(1)));
}
+#[test]
+fn work_queue_reserved_bytes_counter_matches_scan_across_transitions() {
+ // Exercise every ledger transition and assert the O(1) `reserved_bytes` counter
+ // never drifts from the O(n) ground-truth scan.
+ let queue = work_queue_with(
+ 0,
+ (1..=6).map(|h| needed(h, BlockSizeEstimate::Advertised(100))),
+ );
+ let check = |label: &str| {
+ assert_eq!(
+ queue.reserved_bytes(),
+ queue.reserved_bytes_scanned(),
+ "reserved_bytes counter drifted from scan after {label}"
+ );
+ };
+ // Nothing reserved yet: all pending items are `Released`.
+ assert_eq!(queue.reserved_bytes(), 0);
+ check("seed");
+
+ // take + mark_reserved: Released -> Reserved.
+ let taken = queue.take_in_range(block::Height(1), block::Height(6), 6);
+ assert_eq!(taken.len(), 6);
+ assert_eq!(queue.mark_reserved((1..=6).map(block::Height)), 600);
+ assert_eq!(queue.reserved_bytes(), 600);
+ check("mark_reserved");
+
+ // settle: Reserved -> Held drops the reservation for height 1.
+ queue
+ .settle_active_reserved_height(block::Height(1), 80)
+ .expect("height 1 is reserved");
+ assert_eq!(queue.reserved_bytes(), 500);
+ check("settle");
+
+ // mark_held_direct: Reserved -> Held drops height 2's reservation.
+ queue.mark_held_direct(block::Height(2), 90);
+ assert_eq!(queue.reserved_bytes(), 400);
+ check("mark_held_direct");
+
+ // release_heights: Reserved -> Released for height 3.
+ queue.release_heights([block::Height(3)]);
+ assert_eq!(queue.reserved_bytes(), 300);
+ check("release_heights");
+
+ // release_reserved_and_return_items: only the still-reserved height 4 releases.
+ let released = queue.release_reserved_and_return_items([block::Height(4)]);
+ assert_eq!(released, 100);
+ assert_eq!(queue.reserved_bytes(), 200);
+ check("release_reserved_and_return_items");
+
+ // advance_floor drops the committed `<= floor` prefix. Heights 1 (Held) and 2
+ // (Held) contribute no reservation; height 3 is already Released. Only heights
+ // 5 and 6 remain reserved above the new floor.
+ let released = queue.advance_floor(block::Height(4));
+ assert_eq!(released, 0, "heights <= 4 owned no live reservation");
+ assert!(!queue.pending_contains(block::Height(1)));
+ assert!(!queue.in_flight_contains(block::Height(2)));
+ assert_eq!(queue.reserved_bytes(), 200);
+ check("advance_floor");
+
+ // reset_above drops the `> floor` suffix (heights 5 and 6), releasing their
+ // reservations and zeroing the counter.
+ let released = queue.reset_above(block::Height(4));
+ assert_eq!(released, 200);
+ assert_eq!(queue.reserved_bytes(), 0);
+ assert!(!queue.in_flight_contains(block::Height(5)));
+ assert!(!queue.in_flight_contains(block::Height(6)));
+ check("reset_above");
+}
+
+#[test]
+fn work_queue_advance_floor_drops_only_committed_prefix() {
+ let queue = work_queue_with(
+ 0,
+ (1..=10).map(|h| needed(h, BlockSizeEstimate::Advertised(100))),
+ );
+ // Reserve a contiguous run so we can see the reservation accounting on GC.
+ let _ = queue.take_in_range(block::Height(1), block::Height(10), 10);
+ assert_eq!(queue.mark_reserved((1..=10).map(block::Height)), 1000);
+
+ // advance_floor to 4: drops heights 1..=4 (still reserved -> released), keeps 5..=10.
+ let released = queue.advance_floor(block::Height(4));
+ assert_eq!(released, 400, "reserved bytes for the dropped 1..=4 prefix");
+ for h in 1..=4 {
+ assert!(!queue.in_flight_contains(block::Height(h)));
+ }
+ for h in 5..=10 {
+ assert!(queue.in_flight_contains(block::Height(h)));
+ }
+ assert_eq!(queue.reserved_bytes(), 600);
+ assert_eq!(queue.reserved_bytes(), queue.reserved_bytes_scanned());
+
+ // A lower floor is a no-op (floor only advances).
+ assert_eq!(queue.advance_floor(block::Height(2)), 0);
+ assert_eq!(queue.reserved_bytes(), 600);
+}
+
#[test]
fn watchdog_after_held_settle_releases_once() {
let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]);
@@ -2755,6 +2851,93 @@ fn sequencer_retains_raw_bytes_for_non_contiguous_backlog() {
);
}
+#[test]
+fn sequencer_applying_counters_match_scan_across_transitions() {
+ // Assert the O(1) applying counters (buffered bytes, submitted count/bytes)
+ // never drift from a full scan across insert / submit / unsubmit / remove /
+ // commit-release / reset.
+ let mut seq = test_sequencer(0, 8);
+ let blocks = mainnet_blocks_1_to_3();
+ let check = |seq: &Sequencer, label: &str| {
+ assert_eq!(
+ seq.applying_buffered_bytes(),
+ seq.applying_buffered_bytes_scanned(),
+ "applying_buffered_bytes drifted after {label}"
+ );
+ assert_eq!(
+ seq.submitted_applying_count(),
+ seq.submitted_applying_count_scanned(),
+ "submitted_applying_count drifted after {label}"
+ );
+ assert_eq!(
+ seq.submitted_applying_bytes(),
+ seq.submitted_applying_bytes_scanned(),
+ "submitted_applying_bytes drifted after {label}"
+ );
+ assert_eq!(
+ seq.unsubmitted_applying_count(),
+ seq.applying_len() - seq.submitted_applying_count(),
+ "unsubmitted derivation wrong after {label}"
+ );
+ };
+
+ // Buffer heights 1..=3 with distinct sizes and drain them into `applying`.
+ for (i, block) in blocks.iter().enumerate() {
+ let height = (i + 1) as u32;
+ seq.accept_body(
+ block::Height(height),
+ block.hash(),
+ block.clone(),
+ 100 * u64::from(height),
+ peer(0),
+ );
+ }
+ assert_eq!(
+ seq.drain_ready_into_applying(),
+ vec![block::Height(1), block::Height(2), block::Height(3)]
+ );
+ assert_eq!(seq.applying_buffered_bytes(), 600);
+ assert_eq!(seq.submitted_applying_count(), 0);
+ check(&seq, "drain");
+
+ // Submit heights 1 and 2.
+ let item1 = seq
+ .prepare_submit(block::Height(1))
+ .expect("height 1 applying");
+ let _ = seq
+ .prepare_submit(block::Height(2))
+ .expect("height 2 applying");
+ assert_eq!(seq.submitted_applying_count(), 2);
+ assert_eq!(seq.submitted_applying_bytes(), 300);
+ assert_eq!(seq.unsubmitted_applying_count(), 1);
+ check(&seq, "submit 1,2");
+
+ // Roll back the submit for height 1.
+ seq.unsubmit(block::Height(1), item1.token);
+ assert_eq!(seq.submitted_applying_count(), 1);
+ assert_eq!(seq.submitted_applying_bytes(), 200);
+ check(&seq, "unsubmit 1");
+
+ // Remove the still-submitted height 2 directly.
+ seq.remove_applying(block::Height(2));
+ assert_eq!(seq.submitted_applying_count(), 0);
+ assert_eq!(seq.submitted_applying_bytes(), 0);
+ assert_eq!(seq.applying_buffered_bytes(), 400);
+ check(&seq, "remove submitted 2");
+
+ // Commit through height 1: releases the applied body (height 1) from `applying`.
+ seq.advance_verified_tip(block::Height(1), true);
+ assert_eq!(seq.applying_buffered_bytes(), 300);
+ check(&seq, "advance_verified_tip");
+
+ // Reset drops all applying state and zeroes the counters.
+ seq.reset_to(block::Height(0), false);
+ assert_eq!(seq.applying_buffered_bytes(), 0);
+ assert_eq!(seq.submitted_applying_count(), 0);
+ assert_eq!(seq.submitted_applying_bytes(), 0);
+ check(&seq, "reset");
+}
+
#[test]
fn sequencer_accept_body_rejects_at_or_below_floor() {
let mut seq = test_sequencer(5, 4);
@@ -2954,6 +3137,175 @@ fn sequencer_reject_drops_successors_and_rolls_floor_back() {
assert_eq!(seq.applying_len(), 0);
}
+#[test]
+fn sequencer_release_applying_blocks_from_keeps_submitted_counters_consistent() {
+ // `release_applying_blocks_from` removes each height through `remove_applying`,
+ // which must decrement the O(1) submitted counters for any *submitted* body it
+ // drops. Existing reject coverage only releases unsubmitted bodies, so this
+ // exercises the submitted-counter branch of that path.
+ let mut seq = test_sequencer(0, 8);
+ let blocks = mainnet_blocks_1_to_3();
+ for (index, block) in blocks.iter().enumerate() {
+ let height = block::Height(index as u32 + 1);
+ seq.accept_body(
+ height,
+ block.hash(),
+ block.clone(),
+ 100 * (index as u64 + 1),
+ peer(0),
+ );
+ }
+ seq.drain_ready_into_applying();
+ // Submit heights 2 and 3 so the released prefix (>= 2) is all submitted work.
+ let _ = seq
+ .prepare_submit(block::Height(2))
+ .expect("height 2 applying");
+ let _ = seq
+ .prepare_submit(block::Height(3))
+ .expect("height 3 applying");
+ assert_eq!(seq.submitted_applying_count(), 2);
+ assert_eq!(seq.submitted_applying_bytes(), 200 + 300);
+
+ let released = seq.release_applying_blocks_from(block::Height(2));
+ assert_eq!(released, 500);
+ assert_eq!(seq.applying_len(), 1);
+ // Only the unsubmitted height 1 (100 bytes) survives; the submitted counters
+ // shed exactly the released bodies' contribution.
+ assert_eq!(seq.applying_buffered_bytes(), 100);
+ assert_eq!(seq.submitted_applying_count(), 0);
+ assert_eq!(seq.submitted_applying_bytes(), 0);
+ // Every maintained counter still agrees with a full scan.
+ assert_eq!(
+ seq.applying_buffered_bytes(),
+ seq.applying_buffered_bytes_scanned()
+ );
+ assert_eq!(
+ seq.submitted_applying_count(),
+ seq.submitted_applying_count_scanned()
+ );
+ assert_eq!(
+ seq.submitted_applying_bytes(),
+ seq.submitted_applying_bytes_scanned()
+ );
+}
+
+#[test]
+fn sequencer_unsubmit_ignores_stale_or_mismatched_token() {
+ // `unsubmit` only rolls back (and decrements the submitted counters) when the
+ // token still matches the live submission, so a stale rollback cannot clobber a
+ // newer one or double-decrement the O(1) counters.
+ let mut seq = test_sequencer(0, 4);
+ let block = mainnet_block(&BLOCK_MAINNET_1_BYTES);
+ seq.accept_body(block::Height(1), block.hash(), block.clone(), 100, peer(0));
+ seq.drain_ready_into_applying();
+ let item = seq
+ .prepare_submit(block::Height(1))
+ .expect("height 1 applying");
+ assert_eq!(seq.submitted_applying_count(), 1);
+ assert_eq!(seq.submitted_applying_bytes(), 100);
+
+ // A rollback carrying a non-matching token is ignored: the submission and the
+ // counters are untouched.
+ seq.unsubmit(block::Height(1), item.token + 1);
+ assert_eq!(seq.submitted_applying_count(), 1);
+ assert_eq!(seq.submitted_applying_bytes(), 100);
+ assert_eq!(
+ seq.submitted_applying_count(),
+ seq.submitted_applying_count_scanned()
+ );
+ assert_eq!(
+ seq.submitted_applying_bytes(),
+ seq.submitted_applying_bytes_scanned()
+ );
+
+ // The matching rollback frees the slot exactly once.
+ seq.unsubmit(block::Height(1), item.token);
+ assert_eq!(seq.submitted_applying_count(), 0);
+ assert_eq!(seq.submitted_applying_bytes(), 0);
+
+ // Replaying the now-stale token must not decrement a second time.
+ seq.unsubmit(block::Height(1), item.token);
+ assert_eq!(seq.submitted_applying_count(), 0);
+ assert_eq!(seq.submitted_applying_bytes(), 0);
+ assert_eq!(
+ seq.submitted_applying_count(),
+ seq.submitted_applying_count_scanned()
+ );
+ assert_eq!(
+ seq.submitted_applying_bytes(),
+ seq.submitted_applying_bytes_scanned()
+ );
+}
+
+#[test]
+fn sequencer_reorder_max_height_reports_highest_buffered() {
+ let mut seq = test_sequencer(0, 8);
+ let blocks = mainnet_blocks_1_to_3();
+ // Empty reorder buffer has no top.
+ assert_eq!(seq.reorder_max_height(), None);
+ // Buffer heights 1 and 3, leaving a gap at 2; the top is the highest buffered.
+ seq.accept_body(
+ block::Height(1),
+ blocks[0].hash(),
+ blocks[0].clone(),
+ 100,
+ peer(0),
+ );
+ seq.accept_body(
+ block::Height(3),
+ blocks[2].hash(),
+ blocks[2].clone(),
+ 300,
+ peer(0),
+ );
+ assert_eq!(seq.reorder_max_height(), Some(block::Height(3)));
+ // Draining the contiguous prefix removes height 1 but leaves the top (3).
+ seq.drain_ready_into_applying();
+ assert_eq!(seq.reorder_max_height(), Some(block::Height(3)));
+ // Filling the gap drains 2 and 3 out, emptying the reorder buffer.
+ seq.accept_body(
+ block::Height(2),
+ blocks[1].hash(),
+ blocks[1].clone(),
+ 200,
+ peer(0),
+ );
+ seq.drain_ready_into_applying();
+ assert_eq!(seq.reorder_max_height(), None);
+}
+
+#[test]
+fn sequencer_keeps_whole_body_for_contiguous_height() {
+ // The retain-for-backlog trim only applies to *non-contiguous* bodies. A body
+ // arriving at the next contiguous height above the floor is kept whole, so its
+ // decoded block — not a re-decode of the raw payload — drains into `applying`.
+ // This is the mirror of `sequencer_retains_raw_bytes_for_non_contiguous_backlog`.
+ let mut seq = test_sequencer(0, 4);
+ let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES);
+ // A decoded block distinguishable from its own raw payload lets us observe which
+ // half of `DecodedWithRawFramePayload` survived acceptance.
+ let distinguishable_decoded_block1 = forked_block(&block1, 99);
+ assert_ne!(distinguishable_decoded_block1.hash(), block1.hash());
+
+ let body = BufferedBlockBody::from_decoded_block(
+ distinguishable_decoded_block1.clone(),
+ Some(raw_block_payload(&block1)),
+ );
+ // Height 1 is the next contiguous height above the floor (0).
+ assert_eq!(
+ seq.accept_buffered_body(block::Height(1), block1.hash(), body, 100, peer(0)),
+ AcceptOutcome::Buffered {
+ covered: block::Height(1)
+ }
+ );
+ assert_eq!(seq.drain_ready_into_applying(), vec![block::Height(1)]);
+ // The kept decoded block drained in, not a re-decode of the raw payload.
+ assert_eq!(
+ seq.applying_hash(block::Height(1)),
+ Some(distinguishable_decoded_block1.hash())
+ );
+}
+
#[test]
fn reorder_fuzzes_arrival_order_as_parent_first() {
let orders = [
diff --git a/zebra-network/src/zakura/block_sync/work_queue.rs b/zebra-network/src/zakura/block_sync/work_queue.rs
index e4058d1f824..b79616d5f7a 100644
--- a/zebra-network/src/zakura/block_sync/work_queue.rs
+++ b/zebra-network/src/zakura/block_sync/work_queue.rs
@@ -56,6 +56,12 @@ struct WorkQueueInner {
floor: block::Height,
/// Floor clamp for size estimates (overridable for tests).
floor_estimate_bytes: u64,
+ /// Running sum of `reserved_charge()` across every `pending` + `in_flight`
+ /// item, maintained incrementally at each ledger transition so
+ /// [`WorkQueue::reserved_bytes`] is O(1) instead of an O(pending + in_flight)
+ /// scan on the sequencer's hot path. `debug_assert`s and the throttled budget
+ /// audit cross-check it against the map contents.
+ reserved_bytes: u64,
}
impl WorkQueueInner {
@@ -91,6 +97,7 @@ impl WorkQueue {
in_flight: std::collections::BTreeMap::new(),
floor,
floor_estimate_bytes: DEFAULT_BS_SIZE_FLOOR_BYTES,
+ reserved_bytes: 0,
}),
available: Notify::new(),
}
@@ -300,6 +307,9 @@ impl WorkQueue {
item.budget = BlockBudgetLedger::reserved(item.estimated_bytes);
marked = marked.saturating_add(item.estimated_bytes);
}
+ // Released (0) -> Reserved(estimate): the reserved total grows by exactly
+ // the bytes just marked.
+ inner.reserved_bytes = inner.reserved_bytes.saturating_add(marked);
marked
}
@@ -314,10 +324,16 @@ impl WorkQueue {
actual: u64,
) -> Option {
let mut inner = self.lock();
- let item = inner.in_flight.get_mut(&height)?;
- item.budget
- .is_reserved()
- .then(|| item.budget.settle(actual))
+ let (reserved_before, delta) = {
+ let item = inner.in_flight.get_mut(&height)?;
+ if !item.budget.is_reserved() {
+ return None;
+ }
+ // Reserved(reserved) -> Held(actual): the reserved charge drops to 0.
+ (item.budget.reserved_charge(), item.budget.settle(actual))
+ };
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_before);
+ Some(delta)
}
/// Mark a height as directly held after the caller admitted `actual` bytes.
@@ -327,14 +343,19 @@ impl WorkQueue {
pub(super) fn mark_held_direct(&self, height: block::Height, actual: u64) -> u64 {
let mut inner = self.lock();
if let Some(item) = inner.in_flight.get_mut(&height) {
+ // X -> Held(actual): any reserved charge this item still owned is gone.
+ let reserved_before = item.budget.reserved_charge();
let previous_charge = item.budget.release();
item.budget = BlockBudgetLedger::Held(actual);
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_before);
return previous_charge;
}
if let Some(mut item) = inner.pending.remove(&height) {
+ let reserved_before = item.budget.reserved_charge();
let previous_charge = item.budget.release();
item.budget = BlockBudgetLedger::Held(actual);
inner.in_flight.insert(height, item);
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_before);
return previous_charge;
}
0
@@ -343,14 +364,18 @@ impl WorkQueue {
/// Release any live charges for `heights`, exactly once.
pub(super) fn release_heights(&self, heights: impl IntoIterator) -> u64 {
let mut released = 0u64;
+ let mut reserved_removed = 0u64;
let mut inner = self.lock();
for height in heights {
if let Some(item) = inner.in_flight.get_mut(&height) {
+ reserved_removed = reserved_removed.saturating_add(item.budget.reserved_charge());
released = released.saturating_add(item.budget.release());
} else if let Some(item) = inner.pending.get_mut(&height) {
+ reserved_removed = reserved_removed.saturating_add(item.budget.reserved_charge());
released = released.saturating_add(item.budget.release());
}
}
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_removed);
released
}
@@ -361,15 +386,19 @@ impl WorkQueue {
) -> u64 {
let mut moved = false;
let mut released = 0u64;
+ let mut reserved_removed = 0u64;
{
let mut inner = self.lock();
for height in heights {
if let Some(mut item) = inner.in_flight.remove(&height) {
+ reserved_removed =
+ reserved_removed.saturating_add(item.budget.reserved_charge());
released = released.saturating_add(item.budget.release());
inner.pending.insert(height, item);
moved = true;
}
}
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_removed);
}
if moved {
self.available.notify_waiters();
@@ -401,10 +430,13 @@ impl WorkQueue {
.in_flight
.remove(&height)
.expect("reserved item exists because it was just checked");
+ // Only reserved items reach here, so the released bytes are exactly
+ // the reserved charge leaving the queue.
released = released.saturating_add(item.budget.release());
inner.pending.insert(height, item);
moved = true;
}
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(released);
}
if moved {
self.available.notify_waiters();
@@ -423,15 +455,32 @@ impl WorkQueue {
let mut inner = self.lock();
inner.floor = inner.floor.max(floor);
let floor = inner.floor;
+ // Pop only the committed `<= floor` prefix from each map. `pending` can hold
+ // the entire header-ahead lag (100k+ heights), so a `retain` over the whole
+ // map on every floor advance is O(total) and serializes the work-queue lock;
+ // popping the prefix is O(removed · log n).
let mut released = 0u64;
- for item in inner.pending.range_mut(..=floor).map(|(_, item)| item) {
+ while let Some((&height, _)) = inner.pending.first_key_value() {
+ if height > floor {
+ break;
+ }
+ let (_, mut item) = inner
+ .pending
+ .pop_first()
+ .expect("first_key_value returned Some");
released = released.saturating_add(item.budget.release_reserved());
}
- for item in inner.in_flight.range_mut(..=floor).map(|(_, item)| item) {
+ while let Some((&height, _)) = inner.in_flight.first_key_value() {
+ if height > floor {
+ break;
+ }
+ let (_, mut item) = inner
+ .in_flight
+ .pop_first()
+ .expect("first_key_value returned Some");
released = released.saturating_add(item.budget.release_reserved());
}
- inner.pending.retain(|height, _| *height > floor);
- inner.in_flight.retain(|height, _| *height > floor);
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(released);
released
}
@@ -444,23 +493,30 @@ impl WorkQueue {
pub(super) fn reset_above(&self, floor: block::Height) -> u64 {
let mut inner = self.lock();
inner.floor = floor;
+ // Pop only the `> floor` suffix from each map (O(removed · log n)); see the
+ // note in `advance_floor` on why a full-map `retain` is too expensive here.
let mut released = 0u64;
- for item in inner
- .pending
- .range_mut((std::ops::Bound::Excluded(floor), std::ops::Bound::Unbounded))
- .map(|(_, item)| item)
- {
+ while let Some((&height, _)) = inner.pending.last_key_value() {
+ if height <= floor {
+ break;
+ }
+ let (_, mut item) = inner
+ .pending
+ .pop_last()
+ .expect("last_key_value returned Some");
released = released.saturating_add(item.budget.release_reserved());
}
- for item in inner
- .in_flight
- .range_mut((std::ops::Bound::Excluded(floor), std::ops::Bound::Unbounded))
- .map(|(_, item)| item)
- {
+ while let Some((&height, _)) = inner.in_flight.last_key_value() {
+ if height <= floor {
+ break;
+ }
+ let (_, mut item) = inner
+ .in_flight
+ .pop_last()
+ .expect("last_key_value returned Some");
released = released.saturating_add(item.budget.release_reserved());
}
- inner.pending.retain(|height, _| *height <= floor);
- inner.in_flight.retain(|height, _| *height <= floor);
+ inner.reserved_bytes = inner.reserved_bytes.saturating_sub(released);
released
}
@@ -495,7 +551,20 @@ impl WorkQueue {
})
}
+ /// Sum of reserved request-estimate bytes across `pending` + `in_flight`.
+ ///
+ /// O(1): returns the incrementally-maintained counter (see
+ /// [`WorkQueueInner::reserved_bytes`]). This is on the sequencer's hot path via
+ /// `publish_view`, so it must not scan the maps. [`reserved_bytes_scanned`] is
+ /// the O(n) ground-truth recomputation used by the audit / tests to catch drift.
pub(super) fn reserved_bytes(&self) -> u64 {
+ self.lock().reserved_bytes
+ }
+
+ /// Ground-truth O(pending + in_flight) recomputation of [`reserved_bytes`],
+ /// used by tests to assert the maintained counter never drifts.
+ #[cfg(test)]
+ pub(super) fn reserved_bytes_scanned(&self) -> u64 {
let inner = self.lock();
inner
.pending
diff --git a/zebra-replay-bench/Cargo.toml b/zebra-replay-bench/Cargo.toml
index 8e59ea5ca14..abd900adb76 100644
--- a/zebra-replay-bench/Cargo.toml
+++ b/zebra-replay-bench/Cargo.toml
@@ -24,9 +24,14 @@ zebra-chain = { path = "../zebra-chain", version = "9.0.0" }
zebra-state = { path = "../zebra-state", version = "8.0.0" }
zebra-consensus = { path = "../zebra-consensus", version = "8.0.0" }
zebra-network = { path = "../zebra-network", version = "8.0.0", features = ["internal-bench"] }
+zebra-node-services = { path = "../zebra-node-services", version = "7.0.0" }
+# The production Zakura block-sync apply driver, exposed via `zebrad::bench_api`
+# behind `internal-bench`, so `apply-sequencer` can drive the real apply pipeline.
+zebrad = { path = "../zebrad", version = "5.0.0-rc.3", default-features = false, features = ["internal-bench"] }
clap = { workspace = true, features = ["derive"] }
color-eyre.workspace = true
+crossbeam-channel = { workspace = true }
futures.workspace = true
tower = { workspace = true, features = ["buffer", "util"] }
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "macros", "time"] }
diff --git a/zebra-replay-bench/README.md b/zebra-replay-bench/README.md
index b41efd2fb96..f94ab5f5d15 100644
--- a/zebra-replay-bench/README.md
+++ b/zebra-replay-bench/README.md
@@ -26,22 +26,25 @@ window warm in page cache first.
# What height is a snapshot at?
zebra-replay-bench info --src /path/to/snapshot
-# Phase 1: extract a 30k window into a cache (source opened read-only)
+# Phase 1: extract a 50k window into a cache (source opened read-only)
zebra-replay-bench index \
--src /path/to/high-snapshot \
- --cache /tmp/win.zrb --start 1800001 --end 1830000
+ --cache /tmp/win.zrb --start 1849958 --end 1899957
-# Phase 2: replay onto a fork whose tip == 1800000
-zebra-replay-bench apply --base /path/to/fork-at-1800000 --cache /tmp/win.zrb
+# Phase 2: replay onto a fork whose tip == 1849957
+# The base must be a cleanly-pruned snapshot at START-1 (1849957) — one produced by
+# replaying forward past the checkpoint raw-tx archive backlog, so a run does not
+# re-trigger a DrainBacklog delete storm at startup.
+zebra-replay-bench apply --base /path/to/fork-at-1849957 --cache /tmp/win.zrb
# One altitude up: replay through the real zebra-state write worker
-zebra-replay-bench apply-worker --base /path/to/fork-at-1800000 --cache /tmp/win.zrb
+zebra-replay-bench apply-worker --base /path/to/fork-at-1849957 --cache /tmp/win.zrb
# Two altitudes up: replay through the real zebra-consensus checkpoint verifier
-zebra-replay-bench apply-verifier --base /path/to/fork-at-1800000 --cache /tmp/win.zrb
+zebra-replay-bench apply-verifier --base /path/to/fork-at-1849957 --cache /tmp/win.zrb
# Three altitudes up: replay through the real Zakura block-sync Sequencer (VCT-only)
-zebra-replay-bench apply-sequencer --base /path/to/fork-at-1800000 --cache /tmp/win.zrb --vct-sidecar /tmp/win.vct
+zebra-replay-bench apply-sequencer --base /path/to/fork-at-1849957 --cache /tmp/win.zrb --vct-sidecar /tmp/win.vct
```
## Third altitude: `apply-verifier`
@@ -96,6 +99,9 @@ Two optional flags:
writer is flushed at end-of-run. Without it the tracer is `noop()` (zero overhead).
Via the harness: `REPLAY_TRACE_DIR= make perf-replay-sequencer` (add
`REPLAY_ARCHIVE=1` for archive mode).
+- `--stop-height ` clamps the committed window to a sub-range of the cache (stops at
+ the last checkpoint `<= h`), so a larger cache can be reused for a shorter run.
+ Harness: `REPLAY_STOP_HEIGHT=`.
## Two altitudes: `apply` vs `apply-worker`
diff --git a/zebra-replay-bench/RESULTS.md b/zebra-replay-bench/RESULTS.md
index 578eac88c14..f2c6a51ed22 100644
--- a/zebra-replay-bench/RESULTS.md
+++ b/zebra-replay-bench/RESULTS.md
@@ -24,11 +24,14 @@ and find the next optimization lever.
## Environment
- 8 cores (DO Premium Intel), 31 GiB RAM, Linux.
-- Snapshots under `/mnt/roman-dev-2-data`: base = `zebra-ckpt-1800000-warm`
- (archive 27.3.0, tip 1,802,000); block + root source = `zebra-cache`
- (tip 3,376,789).
-- Window: heights **1,802,001–1,832,000** (30,000 blocks, ~19.1 GiB of bodies);
- VCT roots sidecar derived from the same source over the same window.
+- Snapshots under `/mnt/roman-dev-2-data`: base = `zebra-ckpt-1850000-warm-pruned`
+ (pruned 27.3.0, tip 1,849,957). It is a *cleanly-pruned* base — produced by
+ replaying the older `zebra-ckpt-1800000-warm-pruned` forward past the checkpoint
+ raw-tx archive backlog, so forking it does not trigger a startup `DrainBacklog`
+ delete storm (the older base did, polluting the first ~10K blocks of every run).
+ Block + root source = `zebra-cache` (tip 3,376,789).
+- Window: heights **1,849,958–1,899,957** (50,000 blocks); cache `win-1850k.zrb`,
+ VCT roots sidecar `win-1850k.vct` derived from the same source over the same window.
- Forward replay, no rollback. Each run executes on a fresh hard-link fork of the
base; the final tip-hash gate (byte-identical to the source snapshot tip
1,832,000) passes on every run.
diff --git a/zebra-replay-bench/src/apply_sequencer.rs b/zebra-replay-bench/src/apply_sequencer.rs
index 9b60ea63045..c90122ff37c 100644
--- a/zebra-replay-bench/src/apply_sequencer.rs
+++ b/zebra-replay-bench/src/apply_sequencer.rs
@@ -4,9 +4,16 @@
//! The sequencer is the body reorder + ordered-submit pipeline: bodies are fed into
//! its reorder queue (here in height order, sequentially — random/out-of-order arrival
//! is a future knob), it drains the contiguous prefix into `applying` and emits
-//! `SubmitBlock`s, and a thin driver here commits each through the same real
-//! `CheckpointVerifier` → `StateService` the verifier rung uses, reporting the commit
-//! back so the sequencer frontier advances and releases the next blocks.
+//! `SubmitBlock`s.
+//!
+//! Those actions are driven by the **production** Zakura block-sync apply driver
+//! (`zebrad::bench_api::drive_block_sync_actions`, unmodified): it applies each body
+//! through the real consensus router into the real `StateService` and reports the
+//! commit back to the sequencer so the frontier advances. Checkpoint-class blocks take
+//! the header-authenticated fast path (`Request::CommitCheckpointAuthenticated`), which
+//! requires the base's Zakura header store to be seeded first
+//! (`zebra-replay-bench seed-headers`). The bench only feeds bodies and watches the
+//! sequencer view for the gate checkpoint — there is no hand-rolled apply loop.
//!
//! VCT mode only. Checkpoint batching is the same as `apply_verifier`: feed up to the
//! last checkpoint `<= end` so the last range delivers the successors the worker's VCT
@@ -18,18 +25,18 @@ use std::{
atomic::{AtomicU64, Ordering},
Arc,
},
- time::Instant,
+ time::{Duration, Instant},
};
use color_eyre::eyre::{bail, eyre, Result};
-use futures::stream::{FuturesOrdered, StreamExt};
-use tower::{buffer::Buffer, Service, ServiceExt};
-use zebra_chain::{
- block::{self, Height},
- parameters::Network,
+use tokio::sync::oneshot;
+use tower::{buffer::Buffer, util::BoxService};
+use zebra_chain::{block::Height, chain_tip::NoChainTip, parameters::Network};
+use zebra_consensus::{router, BoxError, Config as ConsensusConfig};
+use zebra_network::zakura::{
+ spawn_bench_sequencer, BenchDriverParts, ZakuraSupervisorHandle, ZakuraTrace,
};
-use zebra_consensus::CheckpointVerifier;
-use zebra_network::zakura::{spawn_bench_sequencer, BenchSubmit};
+use zebra_node_services::mempool;
use zebra_state::{FinalizedState, PruningConfig, StorageMode};
use crate::{
@@ -57,6 +64,7 @@ pub fn run(
vct_sidecar: Option<&Path>,
network: Network,
archive: bool,
+ stop_height: Option,
trace_dir: Option<&Path>,
) -> Result {
let reader = CacheReader::open(cache_path)?;
@@ -72,7 +80,18 @@ pub fn run(
);
}
let start = header.start_height;
- let end = start + header.count - 1;
+ let cache_end = start + header.count - 1;
+ // Optionally bench a sub-range of a larger cache: clamp the window's effective end
+ // to `--stop-height` (the checkpoint logic below still feeds to the last checkpoint
+ // <= end and gates to the second-to-last, so the prefetch stops early).
+ let end = match stop_height {
+ Some(h) if h < start => bail!("--stop-height {h} is below the window start {start}"),
+ Some(h) if h > cache_end => {
+ bail!("--stop-height {h} exceeds the cache end {cache_end}")
+ }
+ Some(h) => h,
+ None => cache_end,
+ };
let expected_parent = start.checked_sub(1).ok_or_else(|| {
eyre!("cache starts at genesis (height 0); apply needs a base at start-1")
})?;
@@ -180,9 +199,16 @@ pub fn run(
.build()
.map_err(|e| eyre!("building tokio runtime: {e}"))?;
+ // When a deterministic stop height is configured, the state's committer thread
+ // exits the process itself (after flushing the deferred-reconcile worker). The
+ // bench's main task must not return first and exit out from under that flush, so
+ // it parks after reaching the gate and lets the committer be the authoritative
+ // terminator.
+ let debug_stop_at_height = config.debug_stop_at_height;
+
let stats = runtime.block_on(async move {
// Real buffered StateService + checkpoint verifier on the base fork.
- let (state, _read, _latest, _change) = zebra_state::init(
+ let (state, read_state, _latest, _change) = zebra_state::init(
config,
&network,
max_checkpoint_height,
@@ -190,21 +216,39 @@ pub fn run(
)
.await;
let state = Buffer::new(state, STATE_BUFFER_BOUND);
- let mut verifier = CheckpointVerifier::new(
+ // Drive through the production block-verifier router (`zebra_consensus::router::init`),
+ // not the bare `CheckpointVerifier`, so the bench exercises the real Zakura apply
+ // entry point (`Request::Commit` → checkpoint verifier → state). The router is
+ // already buffered, so its synchronous verify work runs on the router's worker,
+ // off this single driver loop. It seeds its checkpoint verifier from the state
+ // tip (the forked base at `expected_parent`), so no explicit initial tip is
+ // needed. The mempool input is never sent (block verification does not use it).
+ let consensus_config = ConsensusConfig {
+ checkpoint_sync: true,
+ vct_fast_sync: true,
+ };
+ let (verifier, _tx_verifier, _bg_handles, _max_ckpt) = router::init(
+ consensus_config,
&network,
- Some((Height(expected_parent), parent_hash)),
state.clone(),
- );
-
- // Per-height serialized byte length, written by the feed task and read by the
- // driver (the SubmitBlock action does not carry the size). One slot per fed
- // block; small (8 bytes each).
- let lens: Arc> = Arc::new(
- (0..feed_target).map(|_| AtomicU64::new(0)).collect(),
- );
+ oneshot::channel::<
+ Buffer, mempool::Request>,
+ >()
+ .1,
+ )
+ .await;
- // Spawn the real block-sync Sequencer starting from the base tip.
- let (feeder, mut submissions, mut committer) = spawn_bench_sequencer(
+ // Spawn the real block-sync Sequencer from the base tip, split into
+ // production-driver parts: the raw `BlockSyncAction` stream + an inert
+ // `BlockSyncHandle` whose `BlockApplyFinished` feedback is forwarded into the
+ // sequencer's `ApplyFinished` input (the hop the production reactor performs).
+ let BenchDriverParts {
+ feeder,
+ actions,
+ block_sync,
+ mut committer,
+ committed_tip,
+ } = spawn_bench_sequencer(
Height(expected_parent),
Height(expected_parent),
parent_hash,
@@ -212,32 +256,98 @@ pub fn run(
SEQUENCER_MAX_INFLIGHT_BYTES,
trace_dir.clone(),
)
- .into_parts();
+ .into_driver_parts();
- let (_producer, rx) = prefetch::spawn(reader, in_flight);
+ // Spawn the *production* Zakura block-sync apply driver against the bench's
+ // action stream, state, and consensus router. It owns verify+commit+report:
+ // checkpoint-class blocks take the header-authenticated fast path
+ // (`Request::CommitCheckpointAuthenticated`), so the base must be header-seeded
+ // (`zebra-replay-bench seed-headers`). Applies drain through `FuturesUnordered`
+ // with the production checkpoint/full/combined limits — no bench apply loop.
+ let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
+ let driver = tokio::spawn(zebrad::bench_api::drive_block_sync_actions(
+ actions,
+ ZakuraSupervisorHandle::new(1),
+ None,
+ block_sync,
+ NoChainTip,
+ read_state.clone(),
+ verifier,
+ max_checkpoint_height,
+ in_flight,
+ in_flight,
+ in_flight,
+ ZakuraTrace::noop(),
+ None,
+ async move {
+ let _ = shutdown_rx.await;
+ },
+ ));
+
+ // Block deserialization is the single-threaded feed's dominant cost, which
+ // starves the apply pipeline. Deserialize on `ZRB_PREFETCH_WORKERS` threads
+ // (default 4) so the feed runs deep ahead and the real commit/verify ceiling
+ // shows. The sequencer reorders, so out-of-order producer output is fine.
+ let prefetch_workers = std::env::var("ZRB_PREFETCH_WORKERS")
+ .ok()
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(4);
+ let (_producers, rx) = prefetch::spawn_raw_parallel(reader, in_flight, prefetch_workers);
- // Feed task: stream prepared blocks into the reorder queue, in height order,
- // up to the last checkpoint (`feed_target`). Records each block's size in `lens`.
- let feed_lens = lens.clone();
+ // Feed backpressure: cap how far ahead of the committed tip the feed runs, so a
+ // fast (parallel) feed can't pile the whole window into the sequencer's unbounded
+ // `applying` set and OOM. Count-based (not per-height) so it's deadlock-safe with
+ // the out-of-order parallel producer. Seed the committed tip to the base.
+ committed_tip.store(u64::from(expected_parent), Ordering::Relaxed);
+ let max_feed_ahead: u64 = std::env::var("ZRB_MAX_FEED_AHEAD")
+ .ok()
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(2000);
+
+ // Feed task: stream raw bodies into the sequencer's reorder queue, up to the last
+ // checkpoint (`feed_target`). Accumulates fed bytes for the throughput report.
+ let fed_bytes = Arc::new(AtomicU64::new(0));
+ let feed_bytes = fed_bytes.clone();
let feed = tokio::spawn(async move {
let mut fed = 0u32;
while fed < feed_target {
+ // Backpressure: hold the feed once it is `max_feed_ahead` blocks past the
+ // committed tip, so the in-flight body backlog stays bounded. Count-based
+ // (`start + fed` vs committed), so a high out-of-order block never blocks a
+ // lower one the commit needs. Leaving blocks in the prefetch ring stalls
+ // the producers, propagating the bound upstream.
+ while (u64::from(start) + u64::from(fed))
+ .saturating_sub(committed_tip.load(Ordering::Relaxed))
+ > max_feed_ahead
+ {
+ tokio::time::sleep(Duration::from_millis(1)).await;
+ }
// The prefetch producer is a std thread; do the blocking recv off the
- // async executor.
+ // async executor. Time it: a long recv wait = the bench's deserialize
+ // supply is the gate; a long feed_body = the sequencer is backpressuring
+ // (the real apply pipeline downstream is the gate).
+ let recv_start = Instant::now();
let item = tokio::task::block_in_place(|| rx.recv());
- let prepared = match item {
+ let recv_us = recv_start.elapsed().as_micros() as u64;
+ let raw = match item {
Ok(Ok(p)) => p,
Ok(Err(e)) => return Err(e),
Err(_) => break, // producer exhausted before feed_target
};
- let height = prepared.height;
- let hash = prepared.block.hash();
- let len = prepared.len as u64;
- feed_lens[(height - start) as usize].store(len, Ordering::Relaxed);
- if !feeder
- .feed_body(Height(height), hash, prepared.block, len)
- .await
- {
+ let height = raw.height;
+ let hash = raw.block.hash();
+ let len = raw.len as u64;
+ feed_bytes.fetch_add(len, Ordering::Relaxed);
+ zebra_chain::stage_timing::record_val(height, "prefetch_recv_us", recv_us);
+ zebra_chain::stage_timing::record(height, "body_fed");
+ let feed_start = Instant::now();
+ let accepted = feeder.feed_body(Height(height), hash, raw.block, len).await;
+ zebra_chain::stage_timing::record_val(
+ height,
+ "feed_body_us",
+ feed_start.elapsed().as_micros() as u64,
+ );
+ if !accepted {
break; // sequencer gone
}
fed += 1;
@@ -245,107 +355,116 @@ pub fn run(
Ok::<(), color_eyre::Report>(())
});
- let mut stats = Stats::default();
- let mut verifies = FuturesOrdered::new();
- let mut submitted = 0u32;
- let mut done = 0u32;
-
- let wall_start = Instant::now();
- let mut last = wall_start;
-
- // Cadence for the `block_sync_state` trace snapshots (only when tracing is on).
- // Dense enough for a smooth throughput/applying curve; the plotter downsamples.
let tracing_on = trace_dir.is_some();
if tracing_on {
committer.emit_state_snapshot();
}
- // Drive: pull ordered submissions and verify+commit them concurrently; report
- // each commit back so the sequencer frontier advances. The sequencer's submit
- // limit bounds the in-flight set.
- while done < target {
- tokio::select! {
- biased;
- // A verify+commit completed: report it back and record it.
- Some(item) = verifies.next(), if !verifies.is_empty() => {
- let (token, height, committed_hash, len): (_, Height, block::Hash, u64) = item?;
- if height == last_checkpoint && committed_hash != expected_tip_hash {
- bail!(
- "committed hash at the second-to-last checkpoint {} does not match the embedded checkpoint hash",
- last_checkpoint.0
- );
- }
- committer.apply_committed(token, height, committed_hash);
- done += 1;
- let now = Instant::now();
- stats.record(len as usize, now - last);
- last = now;
- if tracing_on && (done.is_multiple_of(20) || done == target) {
- committer.emit_state_snapshot();
- }
- if done.is_multiple_of(5000) || done == target {
- let p = committer.progress();
- tracing::info!(
- done, submitted,
- verified_tip = p.verified_tip.0,
- reorder = p.reorder_len,
- applying = p.applying_len,
- "sequencer-progress"
- );
- }
- }
- // The next ordered submission from the sequencer: start its verify.
- // The next ordered submission from the sequencer: start its verify.
- // `None` (action channel closed) just leaves the drain arm to finish.
- maybe = submissions.next_submit(), if submitted < feed_target => {
- if let Some(BenchSubmit { token, block }) = maybe {
- let height = block
- .coinbase_height()
- .expect("submitted checkpoint block has a coinbase height");
- let len = lens[(height.0 - start) as usize].load(Ordering::Relaxed);
- let fut = verifier
- .ready()
- .await
- .map_err(|e| eyre!("verifier not ready: {e}"))?
- .call(block);
- verifies.push_back(async move {
- let committed = fut.await.map_err(|e| {
- eyre!("verify/commit failed at height {}: {e}", height.0)
- })?;
- Ok::<(_, Height, block::Hash, u64), color_eyre::Report>((
- token, height, committed, len,
- ))
- });
- submitted += 1;
- }
- }
- else => break,
+ // The production driver owns verify+commit+report; the bench just feeds bodies
+ // and waits for the verified tip to reach the gate checkpoint, emitting periodic
+ // progress + `block_sync_state` snapshots from the sequencer view as it advances.
+ let target_height = last_checkpoint;
+ let wall_start = Instant::now();
+ loop {
+ let reached = tokio::time::timeout(
+ Duration::from_secs(5),
+ committer.wait_for_verified_tip(target_height),
+ )
+ .await
+ .is_ok();
+ let p = committer.progress();
+ if tracing_on {
+ committer.emit_state_snapshot();
+ }
+ tracing::info!(
+ verified_tip = p.verified_tip.0,
+ reorder = p.reorder_len,
+ applying = p.applying_len,
+ bps = p.committed_blocks_per_sec,
+ "sequencer-progress (prod driver)"
+ );
+ if p.verified_tip >= target_height {
+ break;
+ }
+ // `wait_*` returned without reaching the target => the sequencer task ended
+ // (driver gone); or the feed drained with nothing left in flight. Either way
+ // stop and let the gate below surface the failure.
+ if reached || (feed.is_finished() && p.reorder_len == 0 && p.applying_len == 0) {
+ break;
}
}
let wall = wall_start.elapsed();
- if done < target {
- // Drive couldn't reach the target: surface a feed error if there was one.
- match feed.await {
- Ok(Ok(())) => bail!("sequencer stalled at {done}/{target} committed (no feed error)"),
- Ok(Err(e)) => return Err(e),
- Err(e) => bail!("feed task panicked: {e}"),
+ // Gate via the sequencer view (the driver's `ApplyFinished` feedback), not a
+ // read-state query: reaching the gate is itself the correctness proof. The
+ // header-authenticated fast path refuses any block whose hash doesn't match the
+ // hardcoded checkpoint, and a refused block never advances `verified_tip` — so
+ // `verified_tip >= target` means every block up to it committed against the
+ // authenticated checkpoint chain. When the tip lands exactly on the gate (the
+ // common case), additionally assert the verified hash matches.
+ let gate = committer.progress();
+ if gate.verified_tip < target_height {
+ let _ = shutdown_tx.send(());
+ if let Ok(Err(e)) = feed.await {
+ return Err(e);
}
+ bail!(
+ "driver stalled at verified tip {} before reaching the gate checkpoint {} (a non-authenticated block would refuse to commit)",
+ gate.verified_tip.0,
+ target_height.0
+ );
+ }
+ if gate.verified_tip == target_height && gate.verified_hash != expected_tip_hash {
+ let _ = shutdown_tx.send(());
+ bail!(
+ "verified hash at the second-to-last checkpoint {} ({}) does not match the embedded checkpoint hash {}",
+ target_height.0,
+ gate.verified_hash,
+ expected_tip_hash
+ );
+ }
+
+ // Stop the driver and drain the feed.
+ let _ = shutdown_tx.send(());
+ let _ = driver.await;
+ match feed.await {
+ Ok(Ok(())) => {}
+ Ok(Err(e)) => return Err(e),
+ Err(e) => bail!("feed task panicked: {e}"),
}
- // Drain + flush the JSONL trace writer so the tables are complete on disk.
committer.flush_trace().await;
+ let committed = (target_height.0 - start + 1) as usize;
+ let total_bytes = fed_bytes.load(Ordering::Relaxed);
+ let mut stats = Stats::default();
+ stats.record(total_bytes as usize, wall);
+
tracing::info!(
- committed = done,
+ committed,
last_checkpoint = last_checkpoint.0,
- "replay verified (sequencer, vct): committed through the second-to-last checkpoint; hash matches"
+ "replay verified (sequencer, vct, prod driver): committed through the second-to-last checkpoint; hash matches"
);
- println!("mode=sequencer (vct, {storage_label})");
+ println!("mode=sequencer-prod (vct, {storage_label})");
if let Some(dir) = trace_dir.as_deref() {
println!("zakura-traces={}", dir.display());
}
- println!("{}", stats.report(wall));
+ let secs = wall.as_secs_f64().max(f64::MIN_POSITIVE);
+ println!(
+ "throughput: {:.1} blk/s {:.2} MiB/s",
+ committed as f64 / secs,
+ total_bytes as f64 / secs / (1024.0 * 1024.0)
+ );
+
+ // With a configured stop height, the state committer thread flushes the
+ // deferred-reconcile worker and exits the process once it commits the stop
+ // block. That flush can outlast this task's gate wait, so park here and let
+ // the committer's `process::exit` terminate the run, ensuring every reconcile
+ // is durable first. The bounded timeout is a safety net against a missed exit.
+ if debug_stop_at_height.is_some() {
+ tokio::time::sleep(Duration::from_secs(900)).await;
+ }
+
Ok::(stats)
})?;
diff --git a/zebra-replay-bench/src/config.rs b/zebra-replay-bench/src/config.rs
index 3becdb2c817..b7dcdb41f75 100644
--- a/zebra-replay-bench/src/config.rs
+++ b/zebra-replay-bench/src/config.rs
@@ -2,25 +2,58 @@
use std::path::PathBuf;
-use zebra_state::{Config, StorageMode};
+use zebra_state::{Config, PruningConfig, StorageMode};
/// Builds a [`zebra_state::Config`] pointing at `cache_dir`.
///
/// `cache_dir` is the snapshot/fork root that contains `state/vN/`. The
-/// benchmark only ever opens archive snapshots (full block bodies + per-height
-/// trees), so storage mode is fixed to [`StorageMode::Archive`].
+/// benchmark runs the production fast-sync configuration: [`StorageMode::Pruned`]
+/// with checkpoint sync, so it measures exactly what a real pruned validator does
+/// (including skipping the transparent address index). The per-run fork is a
+/// throwaway copy, so online pruning never touches the base snapshot.
///
/// `force_legacy` clears `vct_fast_sync`, which forces the committer onto the full
/// per-block note-commitment recompute path — the write-assembler + disk-writer
/// work this benchmark exists to measure. Left on (the default), Mainnet would
/// select the VCT peer-source fast path and skip the recompute.
pub fn state_config(cache_dir: PathBuf, force_legacy: bool) -> Config {
- Config {
+ let mut config = Config {
cache_dir,
ephemeral: false,
checkpoint_sync: true,
vct_fast_sync: !force_legacy,
- storage_mode: StorageMode::Archive,
+ storage_mode: StorageMode::Pruned(PruningConfig::default()),
..Config::default()
- }
+ };
+ // Run-ahead finalized-commit pipeline depth (PR #309). 0 = synchronous (default).
+ config.finalized_block_pipeline_depth = std::env::var("ZRB_PIPELINE_DEPTH")
+ .ok()
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(0);
+ // Per-checkpoint transparent reconcile prototype: defer the per-block spent-UTXO
+ // resolution off the commit path and reconcile it in a batch at each checkpoint.
+ // Requires the run-ahead pipeline (ZRB_PIPELINE_DEPTH > 0) for correctness.
+ config.defer_transparent_reconcile = std::env::var("ZRB_DEFER_TRANSPARENT")
+ .map(|v| v == "1")
+ .unwrap_or(false);
+ // Blocks between deferred reconciles (0 = per checkpoint). Mainnet checkpoints are
+ // ~30-40 blocks apart here, too frequent to amortize the reconcile; a larger fixed
+ // interval batches a bigger window.
+ config.defer_reconcile_interval = std::env::var("ZRB_RECONCILE_INTERVAL")
+ .ok()
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(0);
+ // Force the v1 inline reconcile (on the assembler thread) instead of the default
+ // v2 worker thread; mainly to A/B the off-critical-path parallelism win.
+ config.defer_reconcile_inline = std::env::var("ZRB_RECONCILE_INLINE")
+ .map(|v| v == "1")
+ .unwrap_or(false);
+ // Deterministic stop for byte-match verification: when set, the committer flushes
+ // and exits the process at exactly this height (and the per-checkpoint reconcile
+ // fires for the final window via the stop-height hook), so two runs land on an
+ // identical tip. Leave unset for throughput runs.
+ config.debug_stop_at_height = std::env::var("ZRB_STOP_AT_HEIGHT")
+ .ok()
+ .and_then(|s| s.parse().ok());
+ config
}
diff --git a/zebra-replay-bench/src/main.rs b/zebra-replay-bench/src/main.rs
index 7e63d77ff00..dedd2134baa 100644
--- a/zebra-replay-bench/src/main.rs
+++ b/zebra-replay-bench/src/main.rs
@@ -22,6 +22,7 @@ mod index;
mod prefetch;
mod rollback;
mod roots_cache;
+mod seed_headers;
mod stats;
use std::path::PathBuf;
@@ -51,6 +52,13 @@ enum Cmd {
#[arg(long)]
src: PathBuf,
},
+ /// Print a snapshot's finalized value pool and a stable digest of the
+ /// `utxo_by_out_loc` column family (read-only), for byte-match verification.
+ CfDump {
+ /// Snapshot root containing `state/vN/`.
+ #[arg(long)]
+ src: PathBuf,
+ },
/// Read blocks `start..=end` from a snapshot into a flat cache file.
Index {
/// Snapshot root to read from (opened read-only).
@@ -136,11 +144,29 @@ enum Cmd {
/// Pruned (the base must already be a pruned snapshot; pruning is one-way).
#[arg(long)]
archive: bool,
+ /// Stop after committing up to this height, clamping the cache window (to bench
+ /// a sub-range of a larger cache). Defaults to the full cache end.
+ #[arg(long)]
+ stop_height: Option,
/// Write structured Zakura JSONL trace tables to this directory (the same
/// tables `perf-run-mainnet` produces via `[network.zakura] trace_dir`).
#[arg(long)]
trace_dir: Option,
},
+ /// Seed the Zakura header store on a base snapshot for every cached block height,
+ /// so the production block-sync driver's header-authenticated checkpoint fast path
+ /// can run offline. Run once on the base (tip must be `start-1`); forks inherit it.
+ SeedHeaders {
+ /// Base fork root (opened writable, mutated in place; must be at start-1).
+ #[arg(long)]
+ base: PathBuf,
+ /// Cache file produced by `index`.
+ #[arg(long)]
+ cache: PathBuf,
+ /// Seed against an Archive-mode base. Default mirrors the run (Pruned).
+ #[arg(long)]
+ archive: bool,
+ },
/// Replay a cache through the real `zebra-consensus` checkpoint verifier, which
/// commits to a real `StateService` (tip must be `start-1`). One altitude above
/// `apply-worker`; adds PoW/equihash + Merkle verification and checkpoint
@@ -189,6 +215,41 @@ fn main() -> Result<()> {
has_body
);
}
+ Cmd::CfDump { src } => {
+ // Open with the VCT fast path enabled (force_legacy = false): the
+ // benchmark forks are interrupted VCT snapshots below the handoff, which
+ // refuse to open read-only with vct_fast_sync off.
+ let config = state_config(src.clone(), false);
+ let state = FinalizedState::new_read_only(&config, &network);
+ let tip = state
+ .db
+ .tip()
+ .ok_or_else(|| eyre!("snapshot has no finalized tip"))?;
+ let value_pool = state.db.finalized_value_pool();
+ let (count, key_sum, key_xor, value_sum, value_xor) =
+ state.db.utxo_by_out_loc_verification_digest();
+ let lo = zebra_chain::block::Height(tip.0 .0.saturating_sub(14_500));
+ let (bi_count, bi_first_missing, bi_missing) = state.db.block_info_coverage(lo, tip.0);
+ let bi_digest = state.db.block_info_verification_digest();
+ println!(
+ "block_info[{}..={}]: count={} missing={} first_missing={:?}",
+ lo.0, tip.0 .0, bi_count, bi_missing, bi_first_missing
+ );
+ println!("block_info_digest={bi_digest:?}");
+ println!(
+ "src={}\ntip_height={}\ntip_hash={}\nvalue_pool={:?}\n\
+ utxo_count={}\nutxo_key_sum={}\nutxo_key_xor={}\nutxo_value_sum={}\nutxo_value_xor={}",
+ src.display(),
+ tip.0 .0,
+ tip.1,
+ value_pool,
+ count,
+ key_sum,
+ key_xor,
+ value_sum,
+ value_xor,
+ );
+ }
Cmd::Index {
src,
cache,
@@ -208,6 +269,13 @@ fn main() -> Result<()> {
Cmd::Rollback { base, target } => {
rollback::run(&base, target, network)?;
}
+ Cmd::SeedHeaders {
+ base,
+ cache,
+ archive,
+ } => {
+ seed_headers::run(&base, &cache, network, archive)?;
+ }
Cmd::Apply {
base,
cache,
@@ -252,6 +320,7 @@ fn main() -> Result<()> {
cache,
vct_sidecar,
archive,
+ stop_height,
trace_dir,
} => {
#[cfg(feature = "commit-metrics")]
@@ -263,6 +332,7 @@ fn main() -> Result<()> {
vct_sidecar.as_deref(),
network,
archive,
+ stop_height,
trace_dir.as_deref(),
)?;
@@ -289,7 +359,10 @@ fn render_metrics(handle: metrics_exporter_prometheus::PrometheusHandle) {
if line.starts_with('#') {
continue;
}
- if line.starts_with("zebra_state_") || line.starts_with("state_vct_") {
+ if line.starts_with("zebra_state_")
+ || line.starts_with("state_vct_")
+ || line.starts_with("zebra_consensus_")
+ {
println!("{line}");
}
}
diff --git a/zebra-replay-bench/src/prefetch.rs b/zebra-replay-bench/src/prefetch.rs
index 9f0504d582d..76536ab6044 100644
--- a/zebra-replay-bench/src/prefetch.rs
+++ b/zebra-replay-bench/src/prefetch.rs
@@ -17,6 +17,7 @@ use std::{
Arc,
},
thread::{self, JoinHandle},
+ time::Instant,
};
use color_eyre::eyre::{eyre, Result};
@@ -45,6 +46,16 @@ pub fn capacity() -> usize {
.unwrap_or(DEFAULT_PREFETCH_CAPACITY)
}
+/// One raw block ready to feed into an upstream verifier or sequencer.
+///
+/// This path avoids verifier-side precomputation when the consumer will submit the
+/// raw block to the real verifier anyway.
+pub struct RawBlock {
+ pub block: Arc,
+ pub len: usize,
+ pub height: u32,
+}
+
/// One prepared block ready to commit: the verified block plus the per-block prep
/// the committer's `next_checkpoint` needs (the block handle and its auth-data
/// root), with its height and serialized byte length.
@@ -104,3 +115,193 @@ pub fn spawn(reader: CacheReader, capacity: usize) -> (JoinHandle<()>, Receiver<
(handle, rx)
}
+
+/// Spawns a raw-block prefetch producer over `reader`.
+///
+/// Use this for paths that feed raw blocks into the real verifier. Unlike
+/// [`spawn`], this does not build [`CheckpointVerifiedBlock`] values that would
+/// be discarded before the verifier rebuilds them.
+pub fn spawn_raw(
+ reader: CacheReader,
+ capacity: usize,
+) -> (JoinHandle<()>, Receiver>) {
+ let (tx, rx) = sync_channel(capacity);
+ let mut reader = reader;
+ let mut height = reader.header().start_height;
+
+ let handle = thread::spawn(move || loop {
+ // Split the single-threaded producer's per-block cost: disk read vs shielded
+ // deserialize (point decompression) vs send backpressure (consumer slow).
+ let read_start = Instant::now();
+ let bytes = match reader.next_block() {
+ Ok(Some(bytes)) => bytes,
+ Ok(None) => return,
+ Err(e) => {
+ let _ = tx.send(Err(eyre!("reading block {height}: {e}")));
+ return;
+ }
+ };
+ let read_us = read_start.elapsed().as_micros() as u64;
+ let len = bytes.len();
+ let deser_start = Instant::now();
+ let block = match Block::zcash_deserialize(&bytes[..]) {
+ Ok(block) => Arc::new(block),
+ Err(e) => {
+ let _ = tx.send(Err(eyre!("deserializing block {height}: {e}")));
+ return;
+ }
+ };
+ zebra_chain::stage_timing::record_val(height, "prefetch_read_us", read_us);
+ zebra_chain::stage_timing::record_val(
+ height,
+ "prefetch_deser_us",
+ deser_start.elapsed().as_micros() as u64,
+ );
+
+ let send_start = Instant::now();
+ let sent = tx.send(Ok(RawBlock { block, len, height }));
+ zebra_chain::stage_timing::record_val(
+ height,
+ "prefetch_send_us",
+ send_start.elapsed().as_micros() as u64,
+ );
+ if sent.is_err() {
+ // Consumer dropped the receiver: stop producing.
+ return;
+ }
+ height += 1;
+ });
+
+ (handle, rx)
+}
+
+/// Like [`spawn_raw`], but deserializes on `workers` threads in parallel.
+///
+/// Block deserialization (shielded point decompression) is the single-threaded
+/// producer's dominant cost, so this splits it: one reader thread streams raw bytes
+/// off the cache (cheap, mostly page-cached) into an MPMC work queue, and `workers`
+/// threads deserialize concurrently. Output arrives in *roughly* — not strictly —
+/// height order (workers finish out of order); the Zakura sequencer reorders, so the
+/// caller may feed bodies as received. `workers <= 1` falls back to [`spawn_raw`].
+pub fn spawn_raw_parallel(
+ reader: CacheReader,
+ capacity: usize,
+ workers: usize,
+) -> (Vec>, Receiver>) {
+ let workers = workers.max(1);
+ if workers == 1 {
+ let (handle, rx) = spawn_raw(reader, capacity);
+ return (vec![handle], rx);
+ }
+
+ let start_height = reader.header().start_height;
+ let (out_tx, out_rx) = sync_channel(capacity);
+ let (work_tx, work_rx) = crossbeam_channel::bounded::<(u32, Vec)>(capacity);
+ // Workers emit (height, result) out of order; the reorder thread serializes them.
+ // The worker->reorder channel is intentionally unbounded so the reorder thread can
+ // always accept the (out-of-order) block it is waiting for. Read-ahead is instead
+ // bounded by `permit`: at most `capacity` blocks may be read but not yet emitted.
+ // Without this bound, a backpressured consumer (e.g. a slow sandblast region) lets
+ // the reader race ahead and deserialize the whole window into RAM (each block's
+ // Orchard Halo2 proof is large), exhausting memory.
+ let (result_tx, result_rx) = std::sync::mpsc::channel::<(u32, Result)>();
+ let (permit_tx, permit_rx) = crossbeam_channel::bounded::<()>(capacity);
+ for _ in 0..capacity {
+ let _ = permit_tx.send(());
+ }
+ let mut handles = Vec::with_capacity(workers + 2);
+
+ // Reader thread: sequential disk read -> work queue (errors go to the reorder).
+ let mut reader = reader;
+ let mut height = start_height;
+ let reader_result = result_tx.clone();
+ handles.push(thread::spawn(move || loop {
+ // Acquire a read-ahead permit (released by the reorder thread once a block is
+ // emitted downstream). Closed channel => consumer is gone, so stop reading.
+ if permit_rx.recv().is_err() {
+ return;
+ }
+ let read_start = Instant::now();
+ let bytes = match reader.next_block() {
+ Ok(Some(bytes)) => bytes,
+ Ok(None) => return,
+ Err(e) => {
+ let _ = reader_result.send((height, Err(eyre!("reading block {height}: {e}"))));
+ return;
+ }
+ };
+ zebra_chain::stage_timing::record_val(
+ height,
+ "prefetch_read_us",
+ read_start.elapsed().as_micros() as u64,
+ );
+ if work_tx.send((height, bytes)).is_err() {
+ return;
+ }
+ height += 1;
+ }));
+
+ // Deserialize workers: pull bytes, deserialize (the expensive part), emit (height, result).
+ for _ in 0..workers {
+ let work_rx = work_rx.clone();
+ let result_tx = result_tx.clone();
+ handles.push(thread::spawn(move || {
+ while let Ok((height, bytes)) = work_rx.recv() {
+ let len = bytes.len();
+ let deser_start = Instant::now();
+ let item = match Block::zcash_deserialize(&bytes[..]) {
+ Ok(block) => Ok(RawBlock {
+ block: Arc::new(block),
+ len,
+ height,
+ }),
+ Err(e) => Err(eyre!("deserializing block {height}: {e}")),
+ };
+ let is_err = item.is_err();
+ zebra_chain::stage_timing::record_val(
+ height,
+ "prefetch_deser_us",
+ deser_start.elapsed().as_micros() as u64,
+ );
+ if result_tx.send((height, item)).is_err() || is_err {
+ return;
+ }
+ }
+ }));
+ }
+ drop(result_tx); // result_rx closes once the reader + all workers finish
+
+ // Reorder thread: emit blocks in strict contiguous height order, so the consumer's
+ // committed-tip backpressure is deadlock-safe (the next block the committer needs is
+ // always produced before any block beyond it). Pending is bounded by the read-ahead
+ // permit capacity, so it can't grow unbounded.
+ handles.push(thread::spawn(move || {
+ let mut pending: std::collections::BTreeMap> =
+ std::collections::BTreeMap::new();
+ let mut next = start_height;
+ while let Ok((height, item)) = result_rx.recv() {
+ pending.insert(height, item);
+ while let Some(item) = pending.remove(&next) {
+ let stop = item.is_err();
+ if out_tx.send(item).is_err() {
+ return;
+ }
+ // Release one read-ahead permit now that this block has left the buffer.
+ let _ = permit_tx.send(());
+ next += 1;
+ if stop {
+ return;
+ }
+ }
+ }
+ // Producers done (normal completion drains fully above; this only runs if a gap
+ // remains, e.g. after an error elsewhere — emit what's left so the consumer ends).
+ while let Some((_, item)) = pending.pop_first() {
+ if out_tx.send(item).is_err() {
+ return;
+ }
+ }
+ }));
+
+ (handles, out_rx)
+}
diff --git a/zebra-replay-bench/src/seed_headers.rs b/zebra-replay-bench/src/seed_headers.rs
new file mode 100644
index 00000000000..51a10ad0534
--- /dev/null
+++ b/zebra-replay-bench/src/seed_headers.rs
@@ -0,0 +1,100 @@
+//! One-time seeding of the Zakura header store on a base snapshot.
+//!
+//! The production block-sync apply driver commits checkpoint-class blocks through the
+//! **header-authenticated** fast path (`Request::CommitCheckpointAuthenticated`), and
+//! refuses any block whose height is not header-authenticated. Authentication reads
+//! strictly from the Zakura header store (`zakura_header_hash`), which live header sync
+//! populates. An offline base snapshot has no header sync, so this command writes the
+//! frontier header rows (hash + header) for every cached block height — exactly what
+//! header sync would persist — directly into the base.
+//!
+//! Run it **once** on the base: forks inherit the rows via the hard-link clone, and the
+//! driver releases each row when it commits that height, so a fresh fork always presents
+//! the full frontier above the body tip. The rows are independent of `tx_by_loc`
+//! pruning, so seeding does not disturb the base's cleanly-pruned state.
+
+use std::{path::Path, sync::Arc};
+
+use color_eyre::eyre::{bail, eyre, Result};
+use zebra_chain::{
+ block::{Block, Height},
+ parameters::Network,
+ serialization::ZcashDeserialize,
+};
+use zebra_state::{FinalizedState, PruningConfig, StorageMode};
+
+use crate::{cache::CacheReader, config::state_config};
+
+/// Seeds the Zakura header store on `base` for every block in `cache_path`.
+///
+/// `base` must be at `cache.start_height - 1` (the frontier sits strictly above the
+/// body tip). `archive` mirrors the run's storage mode (default Pruned).
+pub fn run(base: &Path, cache_path: &Path, network: Network, archive: bool) -> Result<()> {
+ let mut reader = CacheReader::open(cache_path)?;
+ let header = reader.header();
+ let start = header.start_height;
+ let expected_parent = start.checked_sub(1).ok_or_else(|| {
+ eyre!("cache starts at genesis (height 0); seeding needs a base at start-1")
+ })?;
+
+ let mut config = state_config(base.to_path_buf(), /* force_legacy */ false);
+ if !archive {
+ config.storage_mode = StorageMode::Pruned(PruningConfig::default());
+ }
+
+ let state = FinalizedState::new_writable(&config, &network);
+ match state.db.finalized_tip_height() {
+ Some(tip) if tip.0 == expected_parent => {}
+ Some(tip) => bail!(
+ "base tip is {} but cache window starts at {start}; base must be at height {expected_parent}",
+ tip.0
+ ),
+ None => bail!("base has no finalized tip; expected height {expected_parent}"),
+ }
+
+ let mut height = start;
+ let mut chunk: Vec<(Height, Arc)> = Vec::with_capacity(2000);
+ let mut seeded = 0u64;
+
+ let flush = |state: &FinalizedState, chunk: &mut Vec<(Height, Arc)>| -> Result {
+ let n = chunk.len() as u64;
+ state
+ .db
+ .seed_zakura_headers_from_blocks(chunk.drain(..))
+ .map_err(|e| eyre!("seeding zakura header rows: {e}"))?;
+ Ok(n)
+ };
+
+ loop {
+ let bytes = match reader
+ .next_block()
+ .map_err(|e| eyre!("reading cache block {height}: {e}"))?
+ {
+ Some(b) => b,
+ None => break,
+ };
+ let block = Arc::new(
+ Block::zcash_deserialize(&bytes[..])
+ .map_err(|e| eyre!("deserializing block {height}: {e}"))?,
+ );
+ chunk.push((Height(height), block));
+ height += 1;
+ if chunk.len() >= 2000 {
+ seeded += flush(&state, &mut chunk)?;
+ if seeded.is_multiple_of(20000) {
+ tracing::info!(seeded, "seeding zakura header store");
+ }
+ }
+ }
+ if !chunk.is_empty() {
+ seeded += flush(&state, &mut chunk)?;
+ }
+
+ let end = height.saturating_sub(1);
+ tracing::info!(seeded, start, end, "zakura header store seeded");
+ println!(
+ "seeded {seeded} zakura header rows ({start}..={end}) into {}",
+ base.display()
+ );
+ Ok(())
+}
diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs
index 9cf5a4a0a80..7fd580b7df1 100644
--- a/zebra-state/src/config.rs
+++ b/zebra-state/src/config.rs
@@ -16,7 +16,7 @@ use serde::{
use tokio::task::{spawn_blocking, JoinHandle};
use tracing::Span;
-use zebra_chain::{common::default_cache_dir, parameters::Network};
+use zebra_chain::{block, common::default_cache_dir, parameters::Network};
use crate::{
constants::{
@@ -136,6 +136,61 @@ pub struct Config {
#[serde(skip)]
pub vct_fast_sync: bool,
+ /// Maximum number of checkpoint-verified blocks the finalized committer may
+ /// assemble ahead of the durable disk write (the run-ahead pipeline depth).
+ ///
+ /// When `0` (the default), the committer is fully synchronous: each block is
+ /// assembled and flushed to disk before the next is assembled — the original
+ /// behavior. When greater than `0`, the committer assembles up to this many
+ /// blocks ahead on the assembler thread while a dedicated disk-writer thread
+ /// flushes earlier blocks, so per-block wall time approaches `max(assemble,
+ /// flush)` instead of their sum. The externally-visible finalized tip and the
+ /// download budget still trail the durable flush (ack-after-flush), so crash
+ /// recovery is unchanged.
+ ///
+ /// Only applies below the last checkpoint (the reorg-free region); near the
+ /// tip the committer is always synchronous. The bound caps the in-memory
+ /// overlay of not-yet-flushed blocks.
+ pub finalized_block_pipeline_depth: usize,
+
+ /// **Experimental.** Defer the transparent spend resolution (UTXO deletes +
+ /// value-pool debit) off the per-block commit path and reconcile it in a batched
+ /// pass on a dedicated worker thread, in the checkpoint-trusted range.
+ ///
+ /// The per-block committer records spent outpoints into an in-memory window
+ /// instead of resolving them; every [`defer_reconcile_interval`] blocks the window
+ /// is batch-resolved (sorted/deduped disk reads), the value pool is recomputed, and
+ /// the deletes + pool are written in one atomic batch. Only takes effect when the
+ /// auxiliary address index is off ([`skip_address_index`](Config::skip_address_index),
+ /// i.e. pruned + checkpoint sync) and only below the last checkpoint; at the
+ /// checkpoint→semantic handoff the window is drained so the value pool / UTXO set
+ /// are current before semantic verification reads them.
+ ///
+ /// Default `false`. Opt-in for performance benchmarking on a disposable
+ /// below-checkpoint node; crash recovery and RPC mid-reconcile guards are not yet
+ /// implemented, so a node enabling this should be treated as re-snapshottable.
+ pub defer_transparent_reconcile: bool,
+
+ /// **Experimental.** How many blocks between deferred-transparent reconciles, when
+ /// [`defer_transparent_reconcile`](Config::defer_transparent_reconcile) is on.
+ ///
+ /// `0` (the default) reconciles at every checkpoint boundary — but mainnet
+ /// checkpoints are only ~30-40 blocks apart in dense regions, so the reconcile
+ /// fires far too often to amortize its fixed cost. A larger fixed interval
+ /// (e.g. ~2000) reconciles a bigger batched window. Correctness only requires a
+ /// bounded window with the window's spent UTXOs durable by reconcile time, not
+ /// checkpoint alignment.
+ pub defer_reconcile_interval: usize,
+
+ /// Prototype: run the deferred-transparent reconcile inline on the assembler
+ /// thread (the v1 path) instead of on the dedicated reconcile worker thread (v2).
+ ///
+ /// The default (`false`) hands each window to a worker thread so the reconcile's
+ /// disk reads and value-pool recompute overlap continued block assembly. Set to
+ /// `true` to force the inline path, mainly to A/B the parallelism win.
+ #[serde(skip)]
+ pub defer_reconcile_inline: bool,
+
/// Whether to delete the old database directories when present.
///
/// Set to `true` by default. If this is set to `false`,
@@ -274,6 +329,46 @@ impl Config {
}
}
+ /// Whether to omit the auxiliary transparent **address index** (balances,
+ /// address→utxo, address→tx).
+ ///
+ /// The address index is RPC-only state, not consensus. It is skipped only for the
+ /// minimal fast-validator configuration: a node that is both [`StorageMode::Pruned`]
+ /// **and** checkpoint-syncing ([`checkpoint_sync`](Config::checkpoint_sync)). That
+ /// drops the per-block address-balance reads and index writes, just as pruned mode
+ /// drops raw-transaction storage.
+ ///
+ /// An archive node keeps the index; so does a node with checkpoint sync disabled
+ /// (full semantic verification), even when pruned. Address-lookup RPCs
+ /// (`getaddressbalance`/`getaddressutxos`/`getaddresstxids`) return an error when
+ /// the index is skipped, rather than wrong (empty) results.
+ pub fn skip_address_index(&self) -> bool {
+ matches!(self.storage_mode, StorageMode::Pruned(_)) && self.checkpoint_sync
+ }
+
+ /// Whether the deferred-transparent reconcile is configured for this node: the
+ /// experimental [`defer_transparent_reconcile`](Config::defer_transparent_reconcile)
+ /// opt-in is on **and** the auxiliary address index is off (the deferred path skips
+ /// the per-block spent-UTXO reads the address index depends on).
+ ///
+ /// This is the lifecycle predicate (worker spawn, drain triggers); the per-block
+ /// decision is the height-bound [`defers_transparent_spends_at`](Config::defers_transparent_spends_at).
+ pub fn defer_reconcile_configured(&self) -> bool {
+ self.defer_transparent_reconcile && self.skip_address_index()
+ }
+
+ /// Whether the per-block committer defers transparent spend resolution for a block
+ /// at `height` on `network`.
+ ///
+ /// Deferral is only valid in the checkpoint-trusted range: above the last
+ /// checkpoint the semantic verifier validates each block's spends against the UTXO
+ /// set and value pool, so those must be current (non-deferred). The handoff drain
+ /// barrier flushes any pending window before the first above-checkpoint block, and
+ /// this gate keeps every above-checkpoint block on the inline (non-deferred) path.
+ pub fn defers_transparent_spends_at(&self, network: &Network, height: block::Height) -> bool {
+ self.defer_reconcile_configured() && height <= network.checkpoint_list().max_height()
+ }
+
/// Validates the configured [`StorageMode`].
///
/// This must be called before opening the database, so that a misconfigured
@@ -430,6 +525,13 @@ impl Default for Config {
enable_zakura_header_seed_from_committed_blocks: false,
checkpoint_sync: true,
vct_fast_sync: true,
+ // Off by default: the committer is synchronous (assemble then flush)
+ // until run-ahead is explicitly enabled and validated.
+ finalized_block_pipeline_depth: 0,
+ // Off by default: prototype, benchmark-set only.
+ defer_transparent_reconcile: false,
+ defer_reconcile_interval: 0,
+ defer_reconcile_inline: false,
delete_old_database: true,
storage_mode: StorageMode::default(),
debug_stop_at_height: None,
@@ -449,6 +551,85 @@ impl Default for Config {
mod tests {
use super::*;
+ #[test]
+ fn skip_address_index_only_when_pruned_and_checkpoint_syncing() {
+ let pruned_checkpoint = Config {
+ storage_mode: StorageMode::Pruned(PruningConfig::default()),
+ checkpoint_sync: true,
+ ..Default::default()
+ };
+ assert!(
+ pruned_checkpoint.skip_address_index(),
+ "pruned + checkpoint-sync skips the transparent address index"
+ );
+
+ let archive = Config {
+ storage_mode: StorageMode::Archive,
+ checkpoint_sync: true,
+ ..Default::default()
+ };
+ assert!(
+ !archive.skip_address_index(),
+ "archive mode keeps the address index"
+ );
+
+ let pruned_legacy = Config {
+ storage_mode: StorageMode::Pruned(PruningConfig::default()),
+ checkpoint_sync: false,
+ ..Default::default()
+ };
+ assert!(
+ !pruned_legacy.skip_address_index(),
+ "pruned with checkpoint sync disabled keeps the address index"
+ );
+ }
+
+ #[test]
+ fn defers_transparent_spends_only_in_checkpoint_range() {
+ let network = Network::Mainnet;
+ let max_checkpoint = network.checkpoint_list().max_height();
+ let below = block::Height(max_checkpoint.0 - 1);
+ let above = block::Height(max_checkpoint.0 + 1);
+
+ // Configured: pruned + checkpoint sync (so the address index is off) + opt-in.
+ let deferring = Config {
+ storage_mode: StorageMode::Pruned(PruningConfig::default()),
+ checkpoint_sync: true,
+ defer_transparent_reconcile: true,
+ ..Default::default()
+ };
+ assert!(
+ deferring.defers_transparent_spends_at(&network, below),
+ "defers below the last checkpoint"
+ );
+ assert!(
+ deferring.defers_transparent_spends_at(&network, max_checkpoint),
+ "defers at the last checkpoint height"
+ );
+ assert!(
+ !deferring.defers_transparent_spends_at(&network, above),
+ "must NOT defer above the last checkpoint (the semantic verifier reads the live UTXO set / value pool there)"
+ );
+
+ // Opt-in off: never defers, at any height.
+ let off = Config {
+ storage_mode: StorageMode::Pruned(PruningConfig::default()),
+ checkpoint_sync: true,
+ defer_transparent_reconcile: false,
+ ..Default::default()
+ };
+ assert!(!off.defers_transparent_spends_at(&network, below));
+
+ // Address index on (archive): the deferred path is unsafe, never defers.
+ let archive = Config {
+ storage_mode: StorageMode::Archive,
+ checkpoint_sync: true,
+ defer_transparent_reconcile: true,
+ ..Default::default()
+ };
+ assert!(!archive.defers_transparent_spends_at(&network, below));
+ }
+
#[test]
fn storage_mode_deserializes_from_documented_toml() {
assert!(
diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs
index 6c555a22b98..397f6b7e9b5 100644
--- a/zebra-state/src/lib.rs
+++ b/zebra-state/src/lib.rs
@@ -46,8 +46,8 @@ pub use error::{
CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError,
};
pub use request::{
- CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest,
- ReadRequest, Request, SemanticallyVerifiedBlock,
+ AuthenticatedCheckpointHash, CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest,
+ HashOrHeight, MappedRequest, ReadRequest, Request, SemanticallyVerifiedBlock,
};
#[cfg(feature = "indexer")]
diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs
index 66415aa6857..0a60f246776 100644
--- a/zebra-state/src/request.rs
+++ b/zebra-state/src/request.rs
@@ -1264,6 +1264,41 @@ impl Request {
}
}
+/// A block hash that has been positively authenticated against the hardcoded checkpoint list
+/// via the Zakura header chain.
+///
+/// The inner hash is private and can only be constructed by the state (after re-establishing the
+/// checkpoint anchor for the height — see
+/// [`ReadRequest::AuthenticatedCheckpointHash`]). This makes it impossible to fabricate one from a
+/// raw `block.hash()`: it is the provenance marker the Zakura checkpoint fast-commit path relies on
+/// to prove a body's expected hash came from authenticated headers, not from the body itself.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct AuthenticatedCheckpointHash(block::Hash);
+
+impl AuthenticatedCheckpointHash {
+ /// Construct an authenticated checkpoint hash.
+ ///
+ /// Crate-internal: only the state may call this, and only after positively re-establishing the
+ /// checkpoint anchor that authenticates `hash`.
+ pub(crate) fn new(hash: block::Hash) -> Self {
+ Self(hash)
+ }
+
+ /// The authenticated block hash.
+ pub fn hash(&self) -> block::Hash {
+ self.0
+ }
+
+ /// Construct an authenticated checkpoint hash directly, bypassing the state's authentication.
+ ///
+ /// Test-only: this exists so other crates' tests can exercise the checkpoint fast-commit path.
+ /// It is never compiled into production builds, preserving the provenance guarantee.
+ #[cfg(any(test, feature = "proptest-impl"))]
+ pub fn new_for_test(hash: block::Hash) -> Self {
+ Self(hash)
+ }
+}
+
#[derive(Clone, Debug, PartialEq, Eq)]
/// A read-only query about the chain state, via the
/// [`ReadStateService`](crate::service::ReadStateService).
@@ -1486,6 +1521,19 @@ pub enum ReadRequest {
/// Returns the highest header held on disk.
BestHeaderTip,
+ /// Returns the checkpoint-authenticated hash at `height`, if `height` is at or below the
+ /// checkpoint-authenticated header frontier; otherwise `None`.
+ ///
+ /// The hash is positively re-established against the hardcoded checkpoint list (the next
+ /// checkpoint `C >= height` is re-checked, and the persisted header store is a contiguous
+ /// continuity-verified frontier, so continuity pins `height`'s hash to `C`). The returned
+ /// [`AuthenticatedCheckpointHash`] is the provenance token the Zakura checkpoint fast-commit
+ /// path requires.
+ AuthenticatedCheckpointHash {
+ /// The height to authenticate.
+ height: block::Height,
+ },
+
/// Returns header-known, body-missing heights in `(verified_block_tip, best_header_tip]`.
MissingBlockBodies {
/// First height to consider.
@@ -1683,6 +1731,7 @@ impl ReadRequest {
ReadRequest::FindBlockHeaders { .. } => "find_block_headers",
ReadRequest::HeadersByHeightRange { .. } => "headers_by_height_range",
ReadRequest::BestHeaderTip => "best_header_tip",
+ ReadRequest::AuthenticatedCheckpointHash { .. } => "authenticated_checkpoint_hash",
ReadRequest::MissingBlockBodies { .. } => "missing_block_bodies",
ReadRequest::BlockSizeHints { .. } => "block_size_hints",
ReadRequest::BlocksByHeightRange { .. } => "blocks_by_height_range",
diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs
index 5b8316c445a..6085617037d 100644
--- a/zebra-state/src/response.rs
+++ b/zebra-state/src/response.rs
@@ -395,6 +395,9 @@ pub enum ReadResponse {
/// Response to [`ReadRequest::BestHeaderTip`].
BestHeaderTip(Option<(block::Height, block::Hash)>),
+ /// Response to [`ReadRequest::AuthenticatedCheckpointHash`].
+ AuthenticatedCheckpointHash(Option),
+
/// Response to [`ReadRequest::MissingBlockBodies`].
MissingBlockBodies(Vec),
@@ -580,6 +583,7 @@ impl TryFrom for Response {
| ReadResponse::ChainInfo(_)
| ReadResponse::Headers(_)
| ReadResponse::BestHeaderTip(_)
+ | ReadResponse::AuthenticatedCheckpointHash(_)
| ReadResponse::MissingBlockBodies(_)
| ReadResponse::BlockSizeHints(_)
| ReadResponse::Blocks(_)
diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs
index d86c6d2376d..ddd2374408f 100644
--- a/zebra-state/src/service.rs
+++ b/zebra-state/src/service.rs
@@ -96,6 +96,12 @@ const FAST_SYNCED_TREE_UNAVAILABLE_ERROR: &str =
"note commitment treestate is unavailable below the checkpoint on a fast-synced node; \
historical treestate queries require an archive node";
+/// Returned for transparent address-lookup requests when the address index was
+/// not built (pruned storage mode). The index is RPC-only, not consensus.
+const ADDRESS_INDEX_DISABLED: &str =
+ "the transparent address index is disabled in pruned storage mode; \
+ getaddressbalance, getaddressutxos, and getaddresstxids require an archive node";
+
/// A read-write service for Zebra's cached blockchain state.
///
/// This service modifies and provides access to:
@@ -617,6 +623,7 @@ impl StateService {
// If we've finished sending finalized blocks, ignore any repeated blocks.
// (Blocks can be repeated after a syncer reset.)
if let Some(finalized_block_write_sender) = &self.block_write_sender.finalized {
+ zebra_chain::stage_timing::record(queued_block.0.height.0, "state_to_worker");
let send_result = finalized_block_write_sender.send(queued_block);
// If the receiver is closed, we can't send any more blocks.
@@ -1883,6 +1890,12 @@ impl Service for ReadStateService {
)))
}
+ ReadRequest::AuthenticatedCheckpointHash { height } => {
+ Ok(ReadResponse::AuthenticatedCheckpointHash(
+ state.db.authenticated_checkpoint_hash(height),
+ ))
+ }
+
ReadRequest::MissingBlockBodies { from, limit } => {
let verified_block_tip = read::tip_height(state.latest_best_chain(), &state.db);
let best_header_tip = state
@@ -1987,31 +2000,47 @@ impl Service for ReadStateService {
// For the get_address_balance RPC.
ReadRequest::AddressBalance(addresses) => {
- let (balance, received) =
- read::transparent_balance(state.latest_best_chain(), &state.db, addresses)?;
- Ok(ReadResponse::AddressBalance { balance, received })
+ if state.db.config().skip_address_index() {
+ Err(ADDRESS_INDEX_DISABLED.into())
+ } else {
+ let (balance, received) =
+ read::transparent_balance(state.latest_best_chain(), &state.db, addresses)?;
+ Ok(ReadResponse::AddressBalance { balance, received })
+ }
}
// For the get_address_tx_ids RPC.
ReadRequest::TransactionIdsByAddresses {
addresses,
height_range,
- } => read::transparent_tx_ids(
- state.latest_best_chain(),
- &state.db,
- addresses,
- height_range,
- )
- .map(ReadResponse::AddressesTransactionIds),
+ } => {
+ if state.db.config().skip_address_index() {
+ Err(ADDRESS_INDEX_DISABLED.into())
+ } else {
+ read::transparent_tx_ids(
+ state.latest_best_chain(),
+ &state.db,
+ addresses,
+ height_range,
+ )
+ .map(ReadResponse::AddressesTransactionIds)
+ }
+ }
// For the get_address_utxos RPC.
- ReadRequest::UtxosByAddresses(addresses) => read::address_utxos(
- &state.network,
- state.latest_best_chain(),
- &state.db,
- addresses,
- )
- .map(ReadResponse::AddressUtxos),
+ ReadRequest::UtxosByAddresses(addresses) => {
+ if state.db.config().skip_address_index() {
+ Err(ADDRESS_INDEX_DISABLED.into())
+ } else {
+ read::address_utxos(
+ &state.network,
+ state.latest_best_chain(),
+ &state.db,
+ addresses,
+ )
+ .map(ReadResponse::AddressUtxos)
+ }
+ }
ReadRequest::CheckBestChainTipNullifiersAndAnchors(unmined_tx) => {
let latest_non_finalized_best_chain = state.latest_best_chain();
diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
index 8c8da85088e..4ab4e4e32de 100644
--- a/zebra-state/src/service/finalized_state.rs
+++ b/zebra-state/src/service/finalized_state.rs
@@ -76,6 +76,27 @@ static COMMIT_COMPUTE_POOL: LazyLock = LazyLock::new(|| {
.expect("rayon thread pool configuration is valid")
});
+/// A dedicated rayon thread pool for the deferred-transparent reconcile worker's
+/// per-interval parallel passes (spent-UTXO resolution + per-block value-pool
+/// deltas).
+///
+/// Isolating the worker's large `par_iter`s (a heavy interval resolves ~900k
+/// outpoints) onto their own pool keeps them out of the global pool's shared queue,
+/// so the concurrently-running assembler's smaller `par_iter`s (raw-tx
+/// serialization) are not stuck behind them. The worker has spare time per interval,
+/// so a separate queue trades a little worker parallelism for steadier assembler
+/// progress.
+pub(crate) static RECONCILE_POOL: LazyLock = LazyLock::new(|| {
+ let threads = std::thread::available_parallelism()
+ .map(|n| n.get())
+ .unwrap_or(4);
+ rayon::ThreadPoolBuilder::new()
+ .num_threads(threads)
+ .thread_name(|i| format!("reconcile-{i}"))
+ .build()
+ .expect("rayon thread pool configuration is valid")
+});
+
/// Spawns the note-commitment tree per-leaf hashing for `block` onto the
/// commit-compute pool, returning a receiver for the result and a cancellation
/// flag.
@@ -116,17 +137,77 @@ pub(crate) fn spawn_note_precompute(
(rx, cancel)
}
+/// A fully-assembled finalized-block commit that has not yet been flushed to disk.
+///
+/// Produced by [`FinalizedState::assemble_finalized_direct`] (the read/compute half
+/// of a block commit) and consumed by [`FinalizedState::flush_finalized_direct`] (the
+/// disk-write half). Carrying the batch and its post-commit bookkeeping inputs lets
+/// the run-ahead committer hand the batch to a separate disk-writer thread while it
+/// assembles the next block.
+pub struct AssembledCommit {
+ /// The fully-prepared write batch, ready to flush.
+ batch: DiskWriteBatch,
+ /// The committed block's hash.
+ hash: block::Hash,
+ /// The committed block's height.
+ height: block::Height,
+ /// The note-commitment trees after this block, threaded to the next block.
+ note_commitment_trees: NoteCommitmentTrees,
+ /// The retention plan, needed for post-flush archive-backlog bookkeeping.
+ retention: RetentionPlan,
+ /// Commit-pressure trace metadata prepared during batch assembly.
+ commit_trace: PreparedCommitTrace,
+ /// This block's run-ahead pipeline contribution, captured only when the
+ /// committer is running ahead (`None` on the synchronous path). The committer
+ /// records it into the [`FinalizedPipeline`](pipeline::FinalizedPipeline)
+ /// before handing the batch to the disk-writer thread.
+ ///
+ /// Read by the disk-writer-thread integration in `write.rs`; until that lands,
+ /// it is only ever the `None` produced by the synchronous path.
+ #[allow(dead_code)]
+ pipeline_contribution: Option,
+ /// The committed block, for elasticsearch indexing after the flush.
+ #[cfg(feature = "elasticsearch")]
+ finalized_block: Arc,
+}
+
+/// The subset of a run-ahead [`AssembledCommit`] the disk-writer thread and the
+/// committer's ack handling need, returned by
+/// [`FinalizedState::commit_finalized_pipelined`].
+///
+/// The overlay-specific fields of `AssembledCommit` have already been folded into
+/// the pipeline by the time this is returned, so this carries only the batch to
+/// flush and the scalars used to advance the chain tip and thread the trees.
+pub(crate) struct PipelineAssembled {
+ /// The fully-prepared write batch, handed to the disk-writer thread.
+ pub batch: DiskWriteBatch,
+ /// The committed block's hash (sent on the commit response after the flush).
+ pub hash: block::Hash,
+ /// The committed block's height (acked by the writer to advance the tip).
+ pub height: block::Height,
+ /// The note-commitment trees after this block, threaded to the next block.
+ pub note_commitment_trees: NoteCommitmentTrees,
+ /// Commit-pressure trace metadata prepared during batch assembly.
+ pub commit_trace: PreparedCommitTrace,
+}
+
pub mod column_family;
+mod commit_pressure;
+pub(crate) use commit_pressure::PreparedCommitTrace;
mod commitment_aux;
mod commitment_aux_verify;
mod disk_db;
mod disk_format;
+mod pipeline;
mod vct;
mod zebra_db;
use vct::VctState;
+use pipeline::BlockPipelineContribution;
+pub(crate) use pipeline::{reconcile_window, FinalizedPipeline, ReconcileBlock};
+
/// The verified-commitment-trees `tree_aux` serving read path (design §9): the per-block
/// commitment roots for a height range, derived from the per-height trees.
pub(crate) use commitment_aux::serve_block_roots;
@@ -824,6 +905,70 @@ impl FinalizedState {
}
}
+ /// Assemble a checkpoint-verified block for the run-ahead pipeline: build its
+ /// write batch against the in-memory `pipeline` overlay, record the block's
+ /// contribution into the overlay, run the committer-owned post-assemble
+ /// bookkeeping, and return the batch (plus the scalars the disk-writer thread
+ /// needs) without writing anything to disk.
+ ///
+ /// The caller hands the returned batch to the disk-writer thread, and advances
+ /// the chain tip and retires the overlay once that thread acknowledges the
+ /// durable flush (ack-after-flush).
+ ///
+ /// # Errors
+ ///
+ /// Propagates the same assembly errors as
+ /// [`assemble_finalized_direct`](Self::assemble_finalized_direct) (including the
+ /// retryable verified-commitment-trees stalls), without mutating the overlay.
+ pub(crate) fn commit_finalized_pipelined(
+ &mut self,
+ finalizable_block: FinalizableBlock,
+ prev_note_commitment_trees: Option,
+ note_precompute: Option,
+ next_checkpoint: Option<(Arc, Option)>,
+ pipeline: &mut pipeline::FinalizedPipeline,
+ ) -> Result {
+ pipeline.seed(&self.db);
+
+ let assembled = self.assemble_finalized_direct(
+ finalizable_block,
+ prev_note_commitment_trees,
+ note_precompute,
+ next_checkpoint,
+ Some(pipeline),
+ "commit checkpoint-verified request",
+ )?;
+
+ let AssembledCommit {
+ batch,
+ hash,
+ height,
+ note_commitment_trees,
+ retention,
+ commit_trace,
+ pipeline_contribution,
+ // Elasticsearch indexing is skipped on the run-ahead path.
+ ..
+ } = assembled;
+
+ let contribution =
+ pipeline_contribution.expect("overlay was supplied, so a contribution is captured");
+ pipeline.record_block(contribution);
+
+ // Evict committed roots and clear the archive backlog now: a block that has
+ // been assembled into a pipeline batch is guaranteed to flush in order (the
+ // checkpoint region is reorg-free), so this need not wait for the flush.
+ self.pipeline_post_assemble(height, retention);
+
+ Ok(PipelineAssembled {
+ batch,
+ hash,
+ height,
+ note_commitment_trees,
+ commit_trace,
+ })
+ }
+
/// Immediately commit a `finalized` block to the finalized state.
///
/// This can be called either by the non-finalized state (when finalizing
@@ -831,6 +976,11 @@ impl FinalizedState {
///
/// Use `source` as the source of the block in log messages.
///
+ /// This is the synchronous (tip-mode) entry point: it
+ /// [assembles](Self::assemble_finalized_direct) the block's batch and
+ /// [flushes](Self::flush_finalized_direct) it to disk in one call. The
+ /// run-ahead committer calls those two halves on separate threads instead.
+ ///
/// # Errors
///
/// - Propagates any errors from writing to the DB
@@ -839,6 +989,42 @@ impl FinalizedState {
/// does not match the expected value
#[allow(clippy::unwrap_in_result)]
pub fn commit_finalized_direct(
+ &mut self,
+ finalizable_block: FinalizableBlock,
+ prev_note_commitment_trees: Option,
+ note_precompute: Option,
+ next_checkpoint: Option<(Arc, Option)>,
+ source: &str,
+ ) -> Result<(block::Hash, NoteCommitmentTrees), CommitCheckpointVerifiedError> {
+ let assembled = self.assemble_finalized_direct(
+ finalizable_block,
+ prev_note_commitment_trees,
+ note_precompute,
+ next_checkpoint,
+ None,
+ source,
+ )?;
+ Ok(self.flush_finalized_direct(assembled, source))
+ }
+
+ /// Assemble (but do not flush) the [`DiskWriteBatch`] for `finalizable_block`.
+ ///
+ /// This is the read/compute half of [`commit_finalized_direct`](Self::commit_finalized_direct):
+ /// it updates the note-commitment and history trees, runs the verified-commitment-trees
+ /// fast path or legacy recompute, checks the ordering asserts, and builds the
+ /// batch — but performs no disk writes. The returned [`AssembledCommit`] is
+ /// handed to [`flush_finalized_direct`](Self::flush_finalized_direct).
+ ///
+ /// Splitting assembly from the flush lets the run-ahead committer build the
+ /// next block's batch while the disk-writer thread flushes this one.
+ ///
+ /// # Errors
+ ///
+ /// - Propagates any errors from updating history and note commitment trees
+ /// - If `hashFinalSaplingRoot` / `hashLightClientRoot` / `hashBlockCommitments`
+ /// does not match the expected value
+ #[allow(clippy::unwrap_in_result)]
+ pub(crate) fn assemble_finalized_direct(
&mut self,
finalizable_block: FinalizableBlock,
prev_note_commitment_trees: Option,
@@ -849,8 +1035,17 @@ impl FinalizedState {
// handoff, where the embedded final frontiers independently authenticate
// this height's roots, or outside the checkpoint commit path.
next_checkpoint: Option<(Arc, Option)>,
+ // The run-ahead pipeline's in-memory tip state and overlay. When `Some`, the
+ // parent history tree and ordering-assert tip cursor are read from it, and
+ // the block's contribution is captured into the returned `AssembledCommit`.
+ overlay: Option<&pipeline::FinalizedPipeline>,
source: &str,
- ) -> Result<(block::Hash, NoteCommitmentTrees), CommitCheckpointVerifiedError> {
+ ) -> Result {
+ // VCT commitment-root verification time (the `verify_commitment_roots` fold),
+ // set inside the checkpoint match arm below and stamped onto the commit trace
+ // after assembly. It runs before `assemble_block_batch`'s `write_start`, so it
+ // is CPU not otherwise counted in `commit_total`. `ZERO` for legacy commits.
+ let mut fold_dur = std::time::Duration::ZERO;
let (
height,
hash,
@@ -874,8 +1069,13 @@ impl FinalizedState {
// so the commitment check below doesn't recompute it here on the
// single-threaded committer. `AuthDataRoot` is `Copy`.
let precomputed_auth_data_root = checkpoint_verified.auth_data_root;
- let mut history_tree = self.db.history_tree();
+ // The parent history tree is read fresh from disk per block, so when
+ // running ahead it must come from the in-memory pipeline tip instead.
+ let mut history_tree = overlay
+ .map(|overlay| overlay.history_tree())
+ .unwrap_or_else(|| self.db.history_tree());
let prev_note_commitment_trees = prev_note_commitment_trees
+ .or_else(|| overlay.map(|overlay| overlay.note_commitment_trees()))
.unwrap_or_else(|| self.db.note_commitment_trees_for_tip());
let mut note_commitment_trees = prev_note_commitment_trees.clone();
@@ -955,6 +1155,7 @@ impl FinalizedState {
// Verifies this block's own header, folds its supplied roots into
// the candidate tree, and when buffered checks the successor header
// against that candidate (the one-block lag).
+ let fold_start = std::time::Instant::now();
let candidate = COMMIT_COMPUTE_POOL
.install(|| {
commitment_aux_verify::verify_commitment_roots(
@@ -967,6 +1168,10 @@ impl FinalizedState {
self.vct_prevalidated_next = None;
self.vct_reject_supplied_root(height, error)
})?;
+ fold_dur = fold_start.elapsed();
+ #[cfg(feature = "commit-metrics")]
+ metrics::histogram!("zebra.state.write.vct_verify.duration_seconds")
+ .record(fold_dur.as_secs_f64());
if let Some((next_block, _next_auth)) = &next_checkpoint {
self.vct_prevalidated_next = Some((
@@ -1120,16 +1325,20 @@ impl FinalizedState {
let history_tree_mut = Arc::make_mut(&mut history_tree);
let sapling_root = note_commitment_trees.sapling.root();
let orchard_root = note_commitment_trees.orchard.root();
- history_tree_mut
- .push(
- &network,
- block.clone(),
- &sapling_root,
- &orchard_root,
- &Default::default(),
- )
- .map_err(Arc::new)
- .map_err(ValidateContextError::from)?;
+ let ironwood_root = Default::default();
+ timed_commit_phase!(
+ "zebra.state.commit.history_push.duration_seconds",
+ history_tree_mut
+ .push(
+ &network,
+ block.clone(),
+ &sapling_root,
+ &orchard_root,
+ &ironwood_root,
+ )
+ .map_err(Arc::new)
+ .map_err(ValidateContextError::from)
+ )?;
#[cfg(feature = "commit-metrics")]
metrics::histogram!("zebra.state.write.checkpoint_compute.duration_seconds")
@@ -1171,11 +1380,21 @@ impl FinalizedState {
}
};
- let committed_tip_hash = self.db.finalized_tip_hash();
- let committed_tip_height = self.db.finalized_tip_height();
+ // The finalized tip is read fresh from disk per block for the ordering
+ // asserts, so when running ahead it must come from the in-memory pipeline
+ // tip (which is seeded from disk and equals it when nothing is in flight).
+ let (committed_tip_hash, committed_tip_height, db_is_empty) =
+ match overlay.and_then(|overlay| overlay.tip()) {
+ Some((tip_height, tip_hash)) => (tip_hash, Some(tip_height), false),
+ None => (
+ self.db.finalized_tip_hash(),
+ self.db.finalized_tip_height(),
+ self.db.is_empty(),
+ ),
+ };
// Assert that callers (including unit tests) get the chain order correct
- if self.db.is_empty() {
+ if db_is_empty {
assert_eq!(
committed_tip_hash, finalized.block.header.previous_block_hash,
"the first block added to an empty state must be a genesis block, source: {source}",
@@ -1201,8 +1420,11 @@ impl FinalizedState {
#[cfg(feature = "elasticsearch")]
let finalized_inner_block = finalized.block.clone();
let note_commitment_trees = finalized.treestate.note_commitment_trees.clone();
+ // Captured for the run-ahead pipeline's in-memory tip state (the history
+ // tree is otherwise read fresh from disk per block); cheap `Arc` clone.
+ let history_tree = finalized.treestate.history_tree.clone();
- // Run `write_block` directly on the committer thread rather than entering the
+ // Assemble the write batch on the committer thread rather than entering the
// dedicated commit-compute pool via `install()`.
//
// The committer is not a member of `COMMIT_COMPUTE_POOL`, so `install()` is a
@@ -1210,59 +1432,150 @@ impl FinalizedState {
// picks up the job, runs it, and signals back. The look-ahead note-commitment
// precompute (`spawn_note_precompute`) keeps those workers busy, so the handoff
// waits on a contended pool, and that wait dominates the isolation it was meant
- // to provide for `write_block`'s internal rayon (`join`/`par_iter`). Running
- // `write_block` here removes the per-block round-trip; its internal rayon uses
+ // to provide for the batch's internal rayon (`join`/`par_iter`). Running
+ // assembly here removes the per-block round-trip; its internal rayon uses
// the global pool instead. Measured net win on the sandblast region (see PR).
let network = self.network();
- let result = self.db.write_block(
+ // `assemble_block_batch` also returns `finalized.hash`, which equals the
+ // `hash` already destructured above; keep the outer binding and ignore it.
+ let (batch, _hash, batch_contribution, mut commit_trace) = self.db.assemble_block_batch(
finalized,
prev_note_commitment_trees,
&network,
- source,
retention,
fast_anchor_roots,
fast_sync_below,
- );
-
- if result.is_ok() {
- if let Some(vct) = &self.vct {
- vct.evict_committed_roots_through(height);
- }
+ overlay,
+ )?;
+ // The fold runs before `assemble_block_batch`'s `write_start`, so record it on
+ // the trace here (it is CPU not otherwise visible in `commit_total`).
+ commit_trace.fold = fold_dur;
+
+ // When running ahead, fold this block's batch contribution together with its
+ // tip cursor and trees into the contribution the committer records into the
+ // pipeline before handing the batch to the disk-writer thread.
+ let pipeline_contribution =
+ batch_contribution.map(|contribution| BlockPipelineContribution {
+ tip: (height, hash),
+ history_tree,
+ note_commitment_trees: note_commitment_trees.clone(),
+ value_pool: contribution.value_pool,
+ wrote_vct_upgrade_marker: contribution.wrote_vct_upgrade_marker,
+ created_outputs: contribution.created_outputs,
+ updated_balances: contribution.updated_balances,
+ deferred_spent: contribution.deferred_spent,
+ deferred_block: contribution.deferred_block,
+ deferred_pool_change: contribution.deferred_pool_change,
+ });
- if retention.clears_archive_backlog() {
- self.checkpoint_raw_tx_archive_backlog
- .store(false, Ordering::Relaxed);
- }
+ Ok(AssembledCommit {
+ batch,
+ hash,
+ height,
+ note_commitment_trees,
+ retention,
+ commit_trace,
+ pipeline_contribution,
+ #[cfg(feature = "elasticsearch")]
+ finalized_block: finalized_inner_block,
+ })
+ }
- // Save blocks to elasticsearch if the feature is enabled.
+ /// Flush a previously [`assemble_finalized_direct`](Self::assemble_finalized_direct)d
+ /// batch to disk and run the post-commit bookkeeping (root eviction, archive
+ /// backlog clearing, elasticsearch, and the configured stop-height check).
+ ///
+ /// This is the write half of [`commit_finalized_direct`](Self::commit_finalized_direct).
+ /// In run-ahead (sync) mode the disk write runs on the disk-writer thread; the
+ /// bookkeeping that mutates committer-owned state runs back on the assembler.
+ /// A rocksdb write failure is fatal, as before.
+ pub fn flush_finalized_direct(
+ &mut self,
+ assembled: AssembledCommit,
+ source: &str,
+ ) -> (block::Hash, NoteCommitmentTrees) {
+ let AssembledCommit {
+ batch,
+ hash,
+ height,
+ note_commitment_trees,
+ retention,
+ commit_trace,
+ // Recorded into the pipeline by the committer before the flush, so the
+ // disk-writer side ignores it here.
+ pipeline_contribution: _,
#[cfg(feature = "elasticsearch")]
- self.elasticsearch(&finalized_inner_block);
+ finalized_block,
+ } = assembled;
- // TODO: move the stop height check to the syncer (#3442)
- if self.is_at_stop_height(height) {
- tracing::info!(
- ?height,
- ?hash,
- block_source = ?source,
- "stopping at configured height, flushing database to disk"
- );
+ self.db.flush_block_batch(batch, commit_trace, source);
- // POC: emit the equivalence digest + fast-path summary before exit.
- self.vct_log_equivalence_digest();
+ self.pipeline_post_assemble(height, retention);
- // We're just about to do a forced exit, so it's ok to do a forced db shutdown
- self.db.shutdown(true);
+ // Save blocks to elasticsearch if the feature is enabled.
+ #[cfg(feature = "elasticsearch")]
+ self.elasticsearch(&finalized_block);
- // Drops tracing log output that's hasn't already been written to stdout
- // since this exits before calling drop on the WorkerGuard for the logger thread.
- // This is okay for now because this is test-only code
- //
- // TODO: Call ZebradApp.shutdown or drop its Tracing component before calling exit_process to flush logs to stdout
- Self::exit_process();
- }
+ self.finalized_stop_at_height_if_configured(height, hash, source);
+
+ (hash, note_commitment_trees)
+ }
+
+ /// Run the committer-owned bookkeeping for a block that has been assembled
+ /// (and, on the synchronous path, flushed): evict its now-committed
+ /// verified-commitment-tree roots and clear the archive backlog flag if the
+ /// retention plan says it is now drained.
+ ///
+ /// In the run-ahead pipeline this runs on the assembler thread right after a
+ /// block is assembled (it only frees memory / clears a flag, so it does not
+ /// need to wait for the durable flush); the synchronous
+ /// [`flush_finalized_direct`](Self::flush_finalized_direct) runs it after the
+ /// flush.
+ fn pipeline_post_assemble(&self, height: block::Height, retention: RetentionPlan) {
+ if let Some(vct) = &self.vct {
+ vct.evict_committed_roots_through(height);
+ }
+
+ if retention.clears_archive_backlog() {
+ self.checkpoint_raw_tx_archive_backlog
+ .store(false, Ordering::Relaxed);
}
+ }
+
+ /// If `height` is the configured `debug_stop_at_height`, flush the database and
+ /// exit the process.
+ ///
+ /// This must only be called once the block at `height` is durable on disk, so
+ /// the run-ahead pipeline calls it from the assembler after the disk-writer
+ /// thread acknowledges the flush of that height.
+ pub(crate) fn finalized_stop_at_height_if_configured(
+ &mut self,
+ height: block::Height,
+ hash: block::Hash,
+ source: &str,
+ ) {
+ // TODO: move the stop height check to the syncer (#3442)
+ if self.is_at_stop_height(height) {
+ tracing::info!(
+ ?height,
+ ?hash,
+ block_source = ?source,
+ "stopping at configured height, flushing database to disk"
+ );
- result.map(|hash| (hash, note_commitment_trees))
+ // POC: emit the equivalence digest + fast-path summary before exit.
+ self.vct_log_equivalence_digest();
+
+ // We're just about to do a forced exit, so it's ok to do a forced db shutdown
+ self.db.shutdown(true);
+
+ // Drops tracing log output that's hasn't already been written to stdout
+ // since this exits before calling drop on the WorkerGuard for the logger thread.
+ // This is okay for now because this is test-only code
+ //
+ // TODO: Call ZebradApp.shutdown or drop its Tracing component before calling exit_process to flush logs to stdout
+ Self::exit_process();
+ }
}
/// POC: `true` when the verified-commitment-trees fast (skip-recompute) path will
@@ -1504,7 +1817,7 @@ impl FinalizedState {
/// Stop the process if `block_height` is greater than or equal to the
/// configured stop height.
- fn is_at_stop_height(&self, block_height: block::Height) -> bool {
+ pub(crate) fn is_at_stop_height(&self, block_height: block::Height) -> bool {
let debug_stop_at_height = match self.debug_stop_at_height {
Some(debug_stop_at_height) => debug_stop_at_height,
None => return false,
diff --git a/zebra-state/src/service/finalized_state/commit_pressure.rs b/zebra-state/src/service/finalized_state/commit_pressure.rs
new file mode 100644
index 00000000000..966450c6584
--- /dev/null
+++ b/zebra-state/src/service/finalized_state/commit_pressure.rs
@@ -0,0 +1,170 @@
+//! Optional per-commit RocksDB pressure trace (off by default).
+//!
+//! When `ZEBRA_COMMIT_PRESSURE_TRACE=` is set, every block commit whose
+//! `batch_commit` write takes at least `ZEBRA_COMMIT_PRESSURE_MS` (default 50 ms)
+//! appends one JSON row to ``, pairing the commit latency with the RocksDB
+//! internal pressure sampled right after the write (L0 files, pending-compaction
+//! bytes, running compactions/flushes, memtable + SST bytes). This lets an offline
+//! analysis line up commit-latency spikes with compaction/flush pressure.
+//!
+//! The hot path is a single `OnceLock` load when the env var is unset (the
+//! production default), so this stays inert unless explicitly enabled — e.g. by
+//! the offline replay bench, which sets the env var for a run.
+
+use std::{
+ fs::File,
+ io::{BufWriter, Write},
+ sync::{
+ atomic::{AtomicU64, Ordering},
+ Arc, Mutex, OnceLock,
+ },
+ time::{Duration, Instant},
+};
+
+use zebra_chain::{block::Block, serialization::ZcashSerialize};
+
+use super::disk_db::DiskDb;
+
+struct CommitPressureTrace {
+ file: Mutex>,
+ threshold: Duration,
+ /// Also emit a baseline row every `every` commits (0 = only slow commits), so the
+ /// trace carries a continuous compaction-pressure curve, not just the spikes.
+ every: u64,
+ count: AtomicU64,
+ start: Instant,
+}
+
+static TRACE: OnceLock> = OnceLock::new();
+
+fn trace() -> Option<&'static CommitPressureTrace> {
+ TRACE
+ .get_or_init(|| {
+ let path = std::env::var_os("ZEBRA_COMMIT_PRESSURE_TRACE")?;
+ let threshold_ms = std::env::var("ZEBRA_COMMIT_PRESSURE_MS")
+ .ok()
+ .and_then(|s| s.parse::().ok())
+ .unwrap_or(50);
+ let every = std::env::var("ZEBRA_COMMIT_PRESSURE_EVERY")
+ .ok()
+ .and_then(|s| s.parse::().ok())
+ .unwrap_or(0);
+ match File::create(&path) {
+ Ok(file) => {
+ tracing::info!(?path, threshold_ms, every, "commit-pressure trace enabled");
+ Some(CommitPressureTrace {
+ file: Mutex::new(BufWriter::new(file)),
+ threshold: Duration::from_millis(threshold_ms),
+ every,
+ count: AtomicU64::new(0),
+ start: Instant::now(),
+ })
+ }
+ Err(error) => {
+ tracing::warn!(?path, %error, "could not open commit-pressure trace");
+ None
+ }
+ }
+ })
+ .as_ref()
+}
+
+/// Per-block timings of the committer's instrumented phases, for the trace row.
+pub(crate) struct PreparedCommitTrace {
+ pub(crate) height: u32,
+ pub(crate) block: Arc,
+ pub(crate) tx_count: usize,
+ pub(crate) output_count: usize,
+ pub(crate) batch_keys: usize,
+ pub(crate) batch_bytes: usize,
+ pub(crate) reads: Duration,
+ pub(crate) address_reads: Duration,
+ pub(crate) batch_assembly: Duration,
+ /// VCT commitment-root verification (`verify_commitment_roots`) — CPU, runs
+ /// before `write_start`, so it is *not* part of `commit_total`. Set by the
+ /// caller after assembly. `ZERO` for legacy (non-VCT-fast) commits.
+ pub(crate) fold: Duration,
+ /// Self-time of `assemble_block_batch` (`write_start` → trace build): the
+ /// read/compute half's CPU, excluding the pipeline queue-wait and disk flush.
+ pub(crate) assemble_self: Duration,
+ pub(crate) write_start: Instant,
+}
+
+/// Records one commit-pressure row, when the trace is enabled and the commit was
+/// at least the configured threshold (or on the periodic baseline tick).
+///
+/// A cheap no-op (one `OnceLock` load) when `ZEBRA_COMMIT_PRESSURE_TRACE` is unset.
+/// Resident set size of this process in MiB, read from `/proc/self/statm`
+/// (field 2 = resident pages × 4 KiB page). Returns 0 if unavailable. Used to
+/// distinguish RocksDB-internal growth from total-process growth in the trace.
+fn process_rss_mb() -> u64 {
+ std::fs::read_to_string("/proc/self/statm")
+ .ok()
+ .and_then(|s| s.split_whitespace().nth(1).map(str::to_string))
+ .and_then(|pages| pages.parse::().ok())
+ .map(|pages| pages * 4096 / (1024 * 1024))
+ .unwrap_or(0)
+}
+
+pub(super) fn record_commit(db: &DiskDb, prepared: PreparedCommitTrace, batch_commit: Duration) {
+ let Some(trace) = trace() else { return };
+ let n = trace.count.fetch_add(1, Ordering::Relaxed);
+ let slow = batch_commit >= trace.threshold;
+ let periodic = trace.every > 0 && n % trace.every == 0;
+ if !slow && !periodic {
+ return;
+ }
+
+ // Sample RocksDB pressure right after the write, and the block size (re-serialized
+ // here; only on the slow-commit / periodic-sample path, so it stays off the hot path).
+ let p = db.pressure_snapshot();
+ let block_bytes = prepared.block.zcash_serialized_size();
+ // Full per-block decomposition. `commit_total` spans `write_start` (assemble
+ // start) → here (after the disk flush), so it = assemble_self + queue_wait +
+ // batch_commit. `fold` is the VCT commitment-root verify, which runs *before*
+ // `write_start` and is therefore additional CPU not counted in `commit_total`.
+ let commit_total = prepared.write_start.elapsed();
+ let assemble_self = prepared.assemble_self;
+ let queue_wait = commit_total
+ .saturating_sub(assemble_self)
+ .saturating_sub(batch_commit);
+ let line = format!(
+ concat!(
+ r#"{{"ts_us":{},"height":{},"slow":{},"fold_ms":{:.3},"reads_ms":{:.3},"address_reads_ms":{:.3},"#,
+ r#""batch_assembly_ms":{:.3},"assemble_self_ms":{:.3},"queue_wait_ms":{:.3},"batch_commit_ms":{:.3},"commit_total_ms":{:.3},"#,
+ r#""block_bytes":{},"tx_count":{},"output_count":{},"batch_keys":{},"batch_bytes":{},"#,
+ r#""l0_files":{},"pending_compaction_bytes":{},"running_compactions":{},"#,
+ r#""running_flushes":{},"memtable_bytes":{},"total_sst_bytes":{},"live_data_bytes":{},"rss_mb":{}}}"#
+ ),
+ trace.start.elapsed().as_micros(),
+ prepared.height,
+ slow,
+ prepared.fold.as_secs_f64() * 1000.0,
+ prepared.reads.as_secs_f64() * 1000.0,
+ prepared.address_reads.as_secs_f64() * 1000.0,
+ prepared.batch_assembly.as_secs_f64() * 1000.0,
+ assemble_self.as_secs_f64() * 1000.0,
+ queue_wait.as_secs_f64() * 1000.0,
+ batch_commit.as_secs_f64() * 1000.0,
+ commit_total.as_secs_f64() * 1000.0,
+ block_bytes,
+ prepared.tx_count,
+ prepared.output_count,
+ prepared.batch_keys,
+ prepared.batch_bytes,
+ p.l0_files,
+ p.pending_compaction_bytes,
+ p.running_compactions,
+ p.running_flushes,
+ p.memtable_bytes,
+ p.total_sst_bytes,
+ p.live_data_bytes,
+ process_rss_mb(),
+ );
+
+ if let Ok(mut file) = trace.file.lock() {
+ // Flush each row so a killed/partial run is still inspectable.
+ let _ = writeln!(file, "{line}");
+ let _ = file.flush();
+ }
+}
diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs
index d425fab2530..0a7513cc719 100644
--- a/zebra-state/src/service/finalized_state/disk_db.rs
+++ b/zebra-state/src/service/finalized_state/disk_db.rs
@@ -124,6 +124,15 @@ pub struct DiskWriteBatch {
batch: rocksdb::WriteBatch,
}
+impl DiskWriteBatch {
+ /// Returns the serialized size of this pending RocksDB write batch.
+ // Always compiled: the commit-pressure trace (`PreparedCommitTrace`) reads it
+ // independently of the `commit-metrics` feature.
+ pub(crate) fn size_in_bytes(&self) -> usize {
+ self.batch.size_in_bytes()
+ }
+}
+
impl Debug for DiskWriteBatch {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DiskWriteBatch")
@@ -537,6 +546,25 @@ impl ReadDisk for DiskDb {
}
}
+/// A point-in-time snapshot of RocksDB compaction/flush pressure, summed across
+/// column families, sampled right after a commit for [`super::commit_pressure`].
+pub(super) struct PressureSnapshot {
+ /// Total SST files sitting at L0 across all CFs (the write-stall trigger).
+ pub(super) l0_files: u64,
+ /// Estimated bytes of pending compaction work across all CFs.
+ pub(super) pending_compaction_bytes: u64,
+ /// Compactions currently running (database-wide).
+ pub(super) running_compactions: u64,
+ /// Memtable flushes currently running (database-wide).
+ pub(super) running_flushes: u64,
+ /// Total memtable (mutable + immutable) bytes across all CFs.
+ pub(super) memtable_bytes: u64,
+ /// Total on-disk SST bytes across all CFs.
+ pub(super) total_sst_bytes: u64,
+ /// Estimated live (post-compaction) data bytes across all CFs.
+ pub(super) live_data_bytes: u64,
+}
+
impl DiskWriteBatch {
/// Creates and returns a new transactional batch write.
///
@@ -550,6 +578,16 @@ impl DiskWriteBatch {
batch: rocksdb::WriteBatch::default(),
}
}
+
+ /// Returns the number of operations (puts + deletes) queued in this batch.
+ pub fn len(&self) -> usize {
+ self.batch.len()
+ }
+
+ /// Returns whether this batch has no queued operations.
+ pub fn is_empty(&self) -> bool {
+ self.batch.is_empty()
+ }
}
impl DiskDb {
@@ -679,6 +717,48 @@ impl DiskDb {
}
}
+ /// Samples RocksDB compaction/flush pressure across all column families, for the
+ /// optional per-commit pressure trace ([`super::commit_pressure`]). Reads only
+ /// in-memory RocksDB properties (no disk I/O); called on the rare slow-commit path.
+ pub(super) fn pressure_snapshot(&self) -> PressureSnapshot {
+ let db: &Arc = &self.db;
+ let db_options = DiskDb::options();
+
+ let mut l0_files = 0;
+ let mut pending_compaction_bytes = 0;
+ let mut memtable_bytes = 0;
+ let mut total_sst_bytes = 0;
+ let mut live_data_bytes = 0;
+
+ for cf_descriptor in DiskDb::construct_column_families(db_options, db.path(), []) {
+ let Some(cf_handle) = db.cf_handle(cf_descriptor.name()) else {
+ continue;
+ };
+ let cf_u64 = |prop: &str| {
+ db.property_int_value_cf(cf_handle, prop)
+ .ok()
+ .flatten()
+ .unwrap_or(0)
+ };
+ l0_files += cf_u64("rocksdb.num-files-at-level0");
+ pending_compaction_bytes += cf_u64("rocksdb.estimate-pending-compaction-bytes");
+ memtable_bytes += cf_u64("rocksdb.size-all-mem-tables");
+ total_sst_bytes += cf_u64("rocksdb.total-sst-files-size");
+ live_data_bytes += cf_u64("rocksdb.estimate-live-data-size");
+ }
+
+ let db_u64 = |prop: &str| db.property_int_value(prop).ok().flatten().unwrap_or(0);
+ PressureSnapshot {
+ l0_files,
+ pending_compaction_bytes,
+ running_compactions: db_u64("rocksdb.num-running-compactions"),
+ running_flushes: db_u64("rocksdb.num-running-flushes"),
+ memtable_bytes,
+ total_sst_bytes,
+ live_data_bytes,
+ }
+ }
+
/// Returns the estimated total disk space usage of the database.
pub fn size(&self) -> u64 {
let db: &Arc = &self.db;
@@ -1283,7 +1363,34 @@ impl DiskDb {
// Tune level-style database file compaction.
//
// This improves Zebra's initial sync speed slightly, as of April 2022.
- opts.optimize_level_style_compaction(Self::MEMTABLE_RAM_CACHE_MEGABYTES * ONE_MEGABYTE);
+ //
+ // The memtable budget is env-tunable for empirical benchmarking; it defaults
+ // to the production constant. `optimize_level_style_compaction` derives
+ // write_buffer_size (budget/4), max_write_buffer_number (6), the L0 trigger,
+ // and the level sizes from this budget.
+ let memtable_mb = std::env::var("ZEBRA_ROCKSDB_MEMTABLE_MB")
+ .ok()
+ .and_then(|v| v.parse::().ok())
+ .unwrap_or(Self::MEMTABLE_RAM_CACHE_MEGABYTES);
+ opts.optimize_level_style_compaction(memtable_mb * ONE_MEGABYTE);
+
+ // Compaction/flush parallelism. RocksDB defaults to `max_background_jobs=2`
+ // (effectively one flush + one compaction) and `max_subcompactions=1`. With
+ // random-keyed nullifier/UTXO inserts a single compaction thread cannot keep
+ // up at high write rates, so L0 backs up. Env-gated for empirical sweeps;
+ // default behavior is unchanged when unset.
+ if let Some(jobs) = std::env::var("ZEBRA_ROCKSDB_BG_JOBS")
+ .ok()
+ .and_then(|v| v.parse::().ok())
+ {
+ opts.set_max_background_jobs(jobs);
+ }
+ if let Some(subc) = std::env::var("ZEBRA_ROCKSDB_SUBCOMPACTIONS")
+ .ok()
+ .and_then(|v| v.parse::().ok())
+ {
+ opts.set_max_subcompactions(subc);
+ }
// Increase the process open file limit if needed,
// then use it to set RocksDB's limit.
diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs
index bf695381b70..0afff70ef4f 100644
--- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs
+++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs
@@ -284,9 +284,11 @@ impl From> for AddressBalanceLocati
/// Represents a change in the [`AddressBalanceLocation`] of a transparent address
/// in the finalized state.
+#[derive(Clone)]
pub struct AddressBalanceLocationChange(AddressBalanceLocationInner);
/// Represents a set of updates to address balance locations in the database.
+#[derive(Clone)]
pub enum AddressBalanceLocationUpdates {
/// A set of [`AddressBalanceLocationChange`]s that should be merged into the existing values in the database.
Merge(HashMap),
diff --git a/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs b/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs
index ad059d7d844..8c6770a9100 100644
--- a/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs
+++ b/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs
@@ -250,6 +250,17 @@ impl DiskFormatUpgrade for Upgrade {
return Ok(Ok(()));
};
+ // Deferred-transparent reconcile: in the checkpoint-trusted range the per-height
+ // `BlockInfo` (and the value pool) are written by the reconcile worker a window
+ // behind the committed tip, so a transient gap below the tip is expected and not
+ // corruption. The auxiliary address index is also off in this mode, so the
+ // received-balance check below does not apply either. Skip the new-blocks check
+ // while deferral is active and the tip is still inside the deferred range; above
+ // the last checkpoint blocks commit inline and the check resumes normally.
+ if db.defer_reconcile_configured() && tip_height <= network.checkpoint_list().max_height() {
+ return Ok(Ok(()));
+ }
+
// Check any outputs in the last 1000 blocks.
let start_height = (tip_height - 1_000).unwrap_or(Height::MIN);
diff --git a/zebra-state/src/service/finalized_state/pipeline.rs b/zebra-state/src/service/finalized_state/pipeline.rs
new file mode 100644
index 00000000000..4b3afc7a082
--- /dev/null
+++ b/zebra-state/src/service/finalized_state/pipeline.rs
@@ -0,0 +1,486 @@
+//! In-memory state for the run-ahead finalized-commit pipeline.
+//!
+//! When [`Config::finalized_block_pipeline_depth`](crate::config::Config::finalized_block_pipeline_depth)
+//! is greater than zero, the finalized committer assembles a block's
+//! [`DiskWriteBatch`](super::DiskWriteBatch) on the assembler thread and hands it
+//! to a dedicated disk-writer thread, so the next block's assembly overlaps this
+//! block's flush.
+//!
+//! Assembling block `X + 1` while block `X` is still being flushed means `X + 1`'s
+//! reads can target state that `X` created but has not yet written to disk. This
+//! module holds that not-yet-durable state so those reads can be served from
+//! memory before falling back to RocksDB:
+//!
+//! - **Threaded tip state** — the history tree, note-commitment trees, value pool,
+//! `vct_upgrade_height` marker, and the (height, hash) tip cursor. These advance
+//! every block and are otherwise read fresh from disk per block, so they must be
+//! carried forward in memory while the assembler is ahead of the writer.
+//! - **A read-through overlay** — the transparent outputs created, and the absolute
+//! address balances updated, by not-yet-flushed blocks. Entries are retired once
+//! the writer has flushed up to (or past) the height that wrote them, so the
+//! overlay is bounded by the pipeline depth.
+//!
+//! This is only used below the last checkpoint, the reorg-free region, so the
+//! overlay never has to roll back: a crash mid-pipeline simply resumes from the
+//! durable finalized tip and re-downloads the gap.
+//!
+//! The committer (in [`write`](crate::service::write)) [`seed`]s the pipeline from
+//! disk, [`record_block`]s each assembled block's contribution before handing its
+//! batch to the disk-writer thread, and [`retire_through`]s the overlay as the
+//! writer makes blocks durable.
+//!
+//! [`seed`]: FinalizedPipeline::seed
+//! [`record_block`]: FinalizedPipeline::record_block
+//! [`retire_through`]: FinalizedPipeline::retire_through
+
+use std::{collections::HashMap, sync::Arc};
+
+use zebra_chain::{
+ amount::{DeferredPoolBalanceChange, NonNegative},
+ block::{self, Block, Height},
+ history_tree::HistoryTree,
+ transparent,
+ value_balance::ValueBalance,
+};
+
+use crate::{
+ service::finalized_state::{
+ disk_format::transparent::{AddressBalanceLocation, AddressBalanceLocationUpdates},
+ disk_format::OutputLocation,
+ FinalizedState, ZebraDb,
+ },
+ BoxError,
+};
+
+use super::NoteCommitmentTrees;
+
+/// The pipeline-relevant outputs of [`DiskWriteBatch::prepare_block_batch`], returned
+/// so the run-ahead committer can thread them forward in memory.
+///
+/// [`DiskWriteBatch::prepare_block_batch`]: super::DiskWriteBatch::prepare_block_batch
+pub(crate) struct BlockBatchOutputs {
+ /// The chain value pool after this block.
+ pub value_pool: ValueBalance,
+ /// The absolute address balances this block updated, captured only when the
+ /// pipeline is running ahead (`None` on the synchronous path).
+ pub address_balances: Option,
+ /// Whether this block wrote the set-once `vct_upgrade_height` marker.
+ pub wrote_vct_upgrade_marker: bool,
+}
+
+/// The pipeline contribution captured by [`ZebraDb::assemble_block_batch`] for a
+/// single block: its value pool, created outputs, and updated address balances.
+/// Combined with the block's tip/history/note trees by the committer to form a
+/// [`BlockPipelineContribution`].
+///
+/// [`ZebraDb::assemble_block_batch`]: super::ZebraDb::assemble_block_batch
+pub(crate) struct PipelineBatchContribution {
+ /// The chain value pool after this block.
+ pub value_pool: ValueBalance,
+ /// Whether this block wrote the set-once `vct_upgrade_height` marker.
+ pub wrote_vct_upgrade_marker: bool,
+ /// The transparent outputs this block created: `(outpoint, location, utxo)`.
+ pub created_outputs: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>,
+ /// The absolute address balances this block updated.
+ pub updated_balances: Vec<(transparent::Address, AddressBalanceLocation)>,
+ /// Per-checkpoint reconcile: the spent outpoints this block deferred (empty
+ /// unless [`ZebraDb::defers_transparent_spends`] is on).
+ pub deferred_spent: Vec,
+ /// The deferring block, held for the value-pool recompute at the checkpoint.
+ pub deferred_block: Option>,
+ /// The block's deferred-pool balance change, needed by the reconcile's
+ /// value-pool recompute (`Some` only after the deferred-pool activation).
+ pub deferred_pool_change: Option,
+}
+
+/// A transparent output created by a not-yet-flushed block.
+#[derive(Clone, Debug)]
+struct OverlayUtxo {
+ /// The on-disk location this output will occupy once flushed.
+ out_loc: OutputLocation,
+ /// The unspent output.
+ utxo: transparent::Utxo,
+ /// The height of the block that created it, for retirement.
+ height: Height,
+}
+
+/// An absolute address balance written by a not-yet-flushed block.
+#[derive(Clone, Debug)]
+struct OverlayBalance {
+ /// The absolute balance/location after the latest not-yet-flushed update.
+ balance: AddressBalanceLocation,
+ /// The height of the latest block that updated it, for retirement.
+ height: Height,
+}
+
+/// Everything block assembly captures about a committed-but-not-yet-flushed block,
+/// so the next block's assembly can read its effects from memory.
+#[derive(Clone, Debug)]
+pub(crate) struct BlockPipelineContribution {
+ /// The committed block's height and hash (the new in-memory tip cursor).
+ pub tip: (Height, block::Hash),
+ /// The history tree after this block.
+ pub history_tree: Arc,
+ /// The note-commitment trees after this block.
+ pub note_commitment_trees: NoteCommitmentTrees,
+ /// The value pool after this block.
+ pub value_pool: ValueBalance,
+ /// Whether this block wrote the set-once `vct_upgrade_height` marker.
+ pub wrote_vct_upgrade_marker: bool,
+ /// The transparent outputs this block created: `(outpoint, location, utxo)`.
+ pub created_outputs: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>,
+ /// The absolute address balances this block updated.
+ pub updated_balances: Vec<(transparent::Address, AddressBalanceLocation)>,
+ /// Per-checkpoint reconcile: the spent outpoints this block deferred (empty
+ /// unless [`ZebraDb::defers_transparent_spends`] is on).
+ pub deferred_spent: Vec,
+ /// The deferring block, held for the value-pool recompute at the checkpoint.
+ pub deferred_block: Option>,
+ /// The block's deferred-pool balance change, needed by the reconcile's
+ /// value-pool recompute.
+ pub deferred_pool_change: Option,
+}
+
+/// One block held in the [`DeferralWindow`] for the per-checkpoint transparent
+/// reconcile: its height, the block (for the value-pool recompute), the spent
+/// outpoints to resolve and delete, and its deferred-pool change.
+#[derive(Clone, Debug)]
+pub(crate) struct ReconcileBlock {
+ /// The deferring block's height.
+ pub height: Height,
+ /// The deferring block, for the value-pool recompute.
+ pub block: Arc,
+ /// The transparent outpoints this block spent, resolved + deleted at the reconcile.
+ pub spent: Vec,
+ /// The block's deferred-pool balance change, passed to `chain_value_pool_change`.
+ pub deferred_pool_change: Option,
+}
+
+/// The in-memory window of blocks whose transparent spend resolution has been
+/// deferred to the next checkpoint reconcile, in increasing height order.
+#[derive(Debug, Default)]
+struct DeferralWindow {
+ /// The deferred blocks, pushed in height order by [`FinalizedPipeline::record_block`].
+ blocks: Vec,
+}
+
+/// The in-memory tip state and read-through overlay for the run-ahead committer.
+///
+/// Seeded lazily from disk on the first block, then advanced in memory by
+/// [`record_block`](Self::record_block) and trimmed by
+/// [`retire_through`](Self::retire_through) as the writer makes blocks durable.
+#[derive(Debug)]
+pub(crate) struct FinalizedPipeline {
+ /// `true` once the threaded tip state has been seeded from disk.
+ seeded: bool,
+ /// The in-memory finalized tip `(height, hash)`, ahead of the durable tip.
+ tip: Option<(Height, block::Hash)>,
+ /// The history tree after the latest assembled block.
+ history_tree: Arc,
+ /// The note-commitment trees after the latest assembled block.
+ note_commitment_trees: NoteCommitmentTrees,
+ /// The value pool after the latest assembled block.
+ value_pool: ValueBalance,
+ /// Whether a not-yet-flushed block has written the `vct_upgrade_height` marker.
+ vct_upgrade_marker_set: bool,
+ /// Transparent outputs created by not-yet-flushed blocks, keyed by outpoint.
+ utxos: HashMap,
+ /// Absolute address balances updated by not-yet-flushed blocks.
+ address_balances: HashMap,
+ /// Blocks whose transparent spend resolution is deferred to the next
+ /// checkpoint reconcile (empty unless deferral is enabled).
+ deferral: DeferralWindow,
+}
+
+impl FinalizedPipeline {
+ /// Create an empty, unseeded pipeline. The threaded tip state is read from
+ /// `db` on the first [`seed`](Self::seed).
+ pub(crate) fn new() -> Self {
+ Self {
+ seeded: false,
+ tip: None,
+ history_tree: Arc::new(HistoryTree::default()),
+ note_commitment_trees: NoteCommitmentTrees::default(),
+ value_pool: ValueBalance::zero(),
+ vct_upgrade_marker_set: false,
+ utxos: HashMap::new(),
+ address_balances: HashMap::new(),
+ deferral: DeferralWindow::default(),
+ }
+ }
+
+ /// Seed the threaded tip state from the durable database, if not already seeded.
+ ///
+ /// Called before assembling the first block of a pipeline run, so the in-memory
+ /// tip state starts exactly equal to disk.
+ pub(crate) fn seed(&mut self, db: &ZebraDb) {
+ if self.seeded {
+ return;
+ }
+
+ self.tip = db
+ .finalized_tip_height()
+ .map(|height| (height, db.finalized_tip_hash()));
+ self.history_tree = db.history_tree();
+ self.note_commitment_trees = db.note_commitment_trees_for_tip();
+ self.value_pool = db.finalized_value_pool();
+ self.vct_upgrade_marker_set = db.vct_upgrade_height().is_some();
+ self.seeded = true;
+ }
+
+ /// The in-memory tip `(height, hash)`, used for the commit ordering asserts.
+ pub(crate) fn tip(&self) -> Option<(Height, block::Hash)> {
+ self.tip
+ }
+
+ /// The note-commitment trees after the latest assembled block (the parent
+ /// trees for the next block).
+ pub(crate) fn note_commitment_trees(&self) -> NoteCommitmentTrees {
+ self.note_commitment_trees.clone()
+ }
+
+ /// The history tree after the latest assembled block.
+ pub(crate) fn history_tree(&self) -> Arc {
+ self.history_tree.clone()
+ }
+
+ /// The value pool after the latest assembled block (the starting pool for the
+ /// next block).
+ pub(crate) fn value_pool(&self) -> ValueBalance {
+ self.value_pool
+ }
+
+ /// Refresh the threaded value pool from the durable database.
+ ///
+ /// In the deferred range the reconcile worker is the sole writer of the value
+ /// pool, so the pipeline's threaded copy stays at its seed (deferred blocks pass
+ /// it through unchanged). After the handoff drain barrier the worker has written
+ /// the current pool to disk; this adopts it so the inline (non-deferred) commits
+ /// above the last checkpoint start from the correct pool.
+ pub(crate) fn reseed_value_pool(&mut self, db: &ZebraDb) {
+ self.value_pool = db.finalized_value_pool();
+ }
+
+ /// Whether a not-yet-flushed block has already written the set-once
+ /// `vct_upgrade_height` marker, so the next block must not write it again.
+ pub(crate) fn vct_upgrade_marker_set(&self) -> bool {
+ self.vct_upgrade_marker_set
+ }
+
+ /// Overlay lookup for a spent output: returns its location and value if it was
+ /// created by a not-yet-flushed block.
+ pub(crate) fn spent_utxo_override(
+ &self,
+ outpoint: &transparent::OutPoint,
+ ) -> Option<(OutputLocation, transparent::Utxo)> {
+ self.utxos
+ .get(outpoint)
+ .map(|entry| (entry.out_loc, entry.utxo.clone()))
+ }
+
+ /// Overlay lookup for an address balance: returns the absolute balance if a
+ /// not-yet-flushed block updated it.
+ pub(crate) fn address_balance_override(
+ &self,
+ address: &transparent::Address,
+ ) -> Option {
+ self.address_balances
+ .get(address)
+ .map(|entry| entry.balance)
+ }
+
+ /// Record a freshly-assembled block's effects, advancing the in-memory tip
+ /// state and overlay so the next block's assembly observes them.
+ pub(crate) fn record_block(&mut self, contribution: BlockPipelineContribution) {
+ let BlockPipelineContribution {
+ tip,
+ history_tree,
+ note_commitment_trees,
+ value_pool,
+ wrote_vct_upgrade_marker,
+ created_outputs,
+ updated_balances,
+ deferred_spent,
+ deferred_block,
+ deferred_pool_change,
+ } = contribution;
+
+ let (height, _hash) = tip;
+
+ self.tip = Some(tip);
+ self.history_tree = history_tree;
+ self.note_commitment_trees = note_commitment_trees;
+ self.value_pool = value_pool;
+ self.vct_upgrade_marker_set |= wrote_vct_upgrade_marker;
+
+ // Per-checkpoint reconcile: record this block's deferred spends + block for
+ // the batched resolve/delete/value-pool pass at the next checkpoint. Only
+ // populated when deferral is on (`deferred_block` is `Some`).
+ if let Some(block) = deferred_block {
+ self.deferral.blocks.push(ReconcileBlock {
+ height,
+ block,
+ spent: deferred_spent,
+ deferred_pool_change,
+ });
+ }
+
+ for (outpoint, out_loc, utxo) in created_outputs {
+ self.utxos.insert(
+ outpoint,
+ OverlayUtxo {
+ out_loc,
+ utxo,
+ height,
+ },
+ );
+ }
+
+ for (address, balance) in updated_balances {
+ self.address_balances
+ .insert(address, OverlayBalance { balance, height });
+ }
+ }
+
+ /// Retire overlay entries the writer has made durable, after it flushes
+ /// `flushed_height`.
+ ///
+ /// Once a block is on disk, its created outputs and updated balances are served
+ /// from RocksDB, so the overlay entries written at or below the durable height
+ /// are redundant and can be dropped, keeping the overlay bounded by depth.
+ pub(crate) fn retire_through(&mut self, flushed_height: Height) {
+ self.utxos.retain(|_, entry| entry.height > flushed_height);
+ self.address_balances
+ .retain(|_, entry| entry.height > flushed_height);
+ }
+
+ /// Removes and returns the deferral-window prefix at or below `boundary_height`,
+ /// in height order, leaving any later (next-interval) blocks in the window.
+ ///
+ /// Acks drain lazily, so by the time a boundary becomes durable the window may
+ /// already hold blocks past it; those belong to the next interval and are kept.
+ pub(crate) fn take_reconcile_prefix(&mut self, boundary_height: Height) -> Vec {
+ let split = self
+ .deferral
+ .blocks
+ .iter()
+ .position(|block| block.height > boundary_height)
+ .unwrap_or(self.deferral.blocks.len());
+ self.deferral.blocks.drain(..split).collect()
+ }
+
+ /// Reconcile the deferred transparent spends for the interval ending at
+ /// `height`, inline on the caller's thread (the v1 path).
+ ///
+ /// Resolves every window spend from disk, deletes the spent `utxo_by_out_loc`
+ /// entries, recomputes the value pool, and writes the deletes + value pool
+ /// (tip + per-height `BlockInfo`) in one atomic batch. Only the window prefix at
+ /// or below `height` is reconciled.
+ ///
+ /// v2 moves this work to a dedicated worker thread (see
+ /// [`reconcile_window`])); this method remains for the inline fallback and is
+ /// byte-identical to it.
+ pub(crate) fn reconcile_checkpoint(
+ &mut self,
+ finalized_state: &mut FinalizedState,
+ height: Height,
+ ) -> Result<(), BoxError> {
+ let records = self.take_reconcile_prefix(height);
+ if records.is_empty() {
+ return Ok(());
+ }
+
+ let db = &finalized_state.db;
+ let network = db.network();
+ let new_value_pool = reconcile_window(db, &network, &records, self.value_pool)?;
+ self.value_pool = new_value_pool;
+
+ Ok(())
+ }
+}
+
+/// Resolve, delete, and re-debit a window of deferred transparent spends, writing
+/// the result in one atomic batch, and return the value pool after the window.
+///
+/// This is the shared core of the per-interval reconcile, called either inline
+/// ([`FinalizedPipeline::reconcile_checkpoint`]) or off the assembler thread by the
+/// dedicated reconcile worker. `start_value_pool` is the running pool before the
+/// first block in `records`; the caller chains the returned pool into the next
+/// window so the value pool stays exactly the sum of the per-block deltas.
+///
+/// # Correctness
+///
+/// Every spent UTXO is durable here: each window block is flushed before its
+/// boundary becomes durable (pipeline depth is far below the interval), so even
+/// same-interval creates are on disk. The deletes (old spent outputs) are disjoint
+/// from the creates the assembler writes inline, and the assembler does not write
+/// the value pool when deferring, so this runs concurrently with assembly without
+/// key conflicts. The resulting value pool and UTXO set are byte-identical to the
+/// non-deferred per-block path.
+pub(crate) fn reconcile_window(
+ db: &ZebraDb,
+ network: &zebra_chain::parameters::Network,
+ records: &[ReconcileBlock],
+ start_value_pool: ValueBalance,
+) -> Result, BoxError> {
+ if records.is_empty() {
+ return Ok(start_value_pool);
+ }
+
+ // Collect the window's spent outpoints, deduped (each UTXO is spent once, so the
+ // dedup is defensive). Sorting by txid also groups repeated txids for the
+ // `output_location`/`tx_loc_by_hash` reads.
+ let mut outpoints: Vec = records
+ .iter()
+ .flat_map(|r| r.spent.iter().copied())
+ .collect();
+ outpoints.sort_unstable_by_key(|outpoint| (outpoint.hash.0, outpoint.index));
+ outpoints.dedup();
+
+ // Batch-resolve every spent outpoint from disk in parallel. Each resolve is
+ // `output_location` (txid → location, `tx_loc_by_hash`) then `utxo_by_location`
+ // (location → value, `utxo_by_out_loc`); both are CPU-bound (page-cache-resident
+ // decode + index traversal) and independent across outpoints, and RocksDB reads
+ // are thread-safe through the cloneable `DiskDb`. This runs off the assembler
+ // thread on the worker, spreading the resolution across otherwise-idle cores.
+ let resolve_start = std::time::Instant::now();
+ use rayon::prelude::*;
+ let resolved: HashMap =
+ outpoints
+ .par_iter()
+ .map(
+ |outpoint| -> Result<
+ (transparent::OutPoint, (OutputLocation, transparent::Utxo)),
+ BoxError,
+ > {
+ let out_loc = db.output_location(outpoint).ok_or_else(|| -> BoxError {
+ format!(
+ "deferred spent outpoint missing from state at reconcile: {outpoint:?}"
+ )
+ .into()
+ })?;
+ let utxo = db
+ .utxo_by_location(out_loc)
+ .ok_or_else(|| -> BoxError {
+ format!(
+ "deferred spent UTXO missing from state at reconcile: {outpoint:?}"
+ )
+ .into()
+ })?
+ .utxo;
+ Ok((*outpoint, (out_loc, utxo)))
+ },
+ )
+ .collect::, BoxError>>()?;
+
+ if std::env::var("ZRB_RECONCILE_DEBUG").is_ok() {
+ let last_h = records.last().map(|r| r.height.0).unwrap_or(0);
+ eprintln!(
+ "[recon-resolve] last_h={last_h} n_outpoints={} resolve_ms={:.1}",
+ outpoints.len(),
+ resolve_start.elapsed().as_secs_f64() * 1e3,
+ );
+ }
+
+ db.commit_checkpoint_reconcile(network, start_value_pool, records, &resolved)
+}
diff --git a/zebra-state/src/service/finalized_state/tests/prop.rs b/zebra-state/src/service/finalized_state/tests/prop.rs
index 2e6a533c78a..b7fd0987cc1 100644
--- a/zebra-state/src/service/finalized_state/tests/prop.rs
+++ b/zebra-state/src/service/finalized_state/tests/prop.rs
@@ -21,7 +21,7 @@ use crate::{
use super::super::{
commitment_aux, serve_block_roots, vct::validate_final_frontiers_bytes,
- CheckpointVerifiedBlock, DiskWriteBatch, FinalizedState,
+ CheckpointVerifiedBlock, DiskWriteBatch, FinalizedPipeline, FinalizedState,
};
const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 1;
@@ -176,6 +176,100 @@ fn blocks_with_v5_transactions() -> Result<()> {
Ok(())
}
+/// Committing a chain through the run-ahead pipeline produces byte-identical
+/// finalized state to the synchronous committer.
+///
+/// Every block is assembled against the in-memory overlay *before any of them are
+/// flushed*, so later blocks read earlier blocks' not-yet-durable effects from the
+/// overlay (spent UTXOs, address balances, value pool, history/note trees, tip
+/// cursor). The batches are then flushed in height order. The resulting finalized
+/// tip, height, and value pool must match a synchronous `commit_finalized_direct`
+/// of the same chain.
+#[test]
+fn pipelined_commit_matches_synchronous() -> Result<()> {
+ let _init_guard = zebra_test::init();
+ proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES")
+ .ok()
+ .and_then(|v| v.parse().ok())
+ .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)),
+ |((chain, count, network, _history_tree) in PreparedChain::default())| {
+ let blocks: Vec<_> = chain.iter().take(count).cloned().collect();
+
+ // Synchronous baseline.
+ let mut sync_state = FinalizedState::new(
+ &Config::ephemeral(),
+ &network,
+ #[cfg(feature = "elasticsearch")]
+ false,
+ );
+ for block in &blocks {
+ let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone());
+ sync_state
+ .commit_finalized_direct(
+ checkpoint_verified.into(),
+ None,
+ None,
+ None,
+ "pipelined_commit_matches_synchronous sync",
+ )
+ .unwrap();
+ }
+
+ // Run-ahead: assemble every block against the overlay (threading the
+ // note-commitment trees forward), buffer the batches, then flush them in
+ // height order — mirroring what the disk-writer thread does, single-threaded.
+ let mut pipe_state = FinalizedState::new(
+ &Config::ephemeral(),
+ &network,
+ #[cfg(feature = "elasticsearch")]
+ false,
+ );
+ let mut pipeline = FinalizedPipeline::new();
+ let mut prev_trees = None;
+ let mut pending = std::collections::VecDeque::new();
+ for block in &blocks {
+ let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone());
+ let assembled = pipe_state
+ .commit_finalized_pipelined(
+ checkpoint_verified.into(),
+ prev_trees.take(),
+ None,
+ None,
+ &mut pipeline,
+ )
+ .unwrap();
+ prev_trees = Some(assembled.note_commitment_trees.clone());
+ pending.push_back(assembled);
+ }
+ for assembled in pending {
+ let height = assembled.height;
+ pipe_state
+ .db
+ .flush_block_batch(
+ assembled.batch,
+ assembled.commit_trace,
+ "pipelined_commit_matches_synchronous flush",
+ );
+ pipeline.retire_through(height);
+ }
+
+ prop_assert_eq!(
+ sync_state.finalized_tip_hash(),
+ pipe_state.finalized_tip_hash()
+ );
+ prop_assert_eq!(
+ sync_state.finalized_tip_height(),
+ pipe_state.finalized_tip_height()
+ );
+ prop_assert_eq!(
+ sync_state.finalized_value_pool(),
+ pipe_state.finalized_value_pool()
+ );
+ });
+
+ Ok(())
+}
+
/// Test if committing blocks from all upgrades work correctly, to make
/// sure the contextual validation done by the finalized state works.
/// Also test if a block with the wrong commitment is correctly rejected.
diff --git a/zebra-state/src/service/finalized_state/tests/transparent.rs b/zebra-state/src/service/finalized_state/tests/transparent.rs
index c10689d1868..0cba782fe47 100644
--- a/zebra-state/src/service/finalized_state/tests/transparent.rs
+++ b/zebra-state/src/service/finalized_state/tests/transparent.rs
@@ -183,6 +183,7 @@ fn intra_block_self_spend_chain_in_finalized_state() {
#[cfg(feature = "indexer")]
&HashMap::new(),
address_balances,
+ false,
);
// Write the batch and confirm the final on-disk balance matches the consensus value
diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs
index be72207dfca..57de6b6002c 100644
--- a/zebra-state/src/service/finalized_state/zebra_db.rs
+++ b/zebra-state/src/service/finalized_state/zebra_db.rs
@@ -63,6 +63,22 @@ pub mod transparent;
// TODO: when the database is split out of zebra-state, always expose these methods.
pub mod arbitrary;
+/// Benchmark-only throughput-ceiling probe (`ZEBRA_BENCH_SKIP_TRANSPARENT_READS=1`).
+///
+/// When set, the committer skips the per-block spent-UTXO resolution entirely, to
+/// measure the upper bound of deferring that work off the commit critical path (the
+/// per-checkpoint transparent reconcile). This deliberately produces an incorrect
+/// value pool and UTXO set, so it is a measurement tool only — never a shipped path.
+pub(crate) fn bench_skip_transparent_reads() -> bool {
+ static PROBE: std::sync::OnceLock = std::sync::OnceLock::new();
+ *PROBE.get_or_init(|| {
+ matches!(
+ std::env::var("ZEBRA_BENCH_SKIP_TRANSPARENT_READS").as_deref(),
+ Ok("1") | Ok("true")
+ )
+ })
+}
+
/// Wrapper struct to ensure high-level `zebra-state` database access goes through the correct API.
///
/// `rocksdb` allows concurrent writes through a shared reference,
@@ -234,6 +250,30 @@ impl ZebraDb {
&self.config
}
+ /// Whether the per-block committer defers transparent spend resolution (the
+ /// `utxo_by_out_loc` deletes and the transparent value-pool debit) off the commit
+ /// critical path, to a batched reconcile on the worker thread, for the block at
+ /// `height`.
+ ///
+ /// Only defers in the checkpoint-trusted range (`height <= max_checkpoint_height`):
+ /// above the last checkpoint the semantic verifier validates spends against the
+ /// live UTXO set and value pool, so those must stay current (non-deferred). The
+ /// auxiliary address index must also be off ([`Config::skip_address_index`]),
+ /// because the deferred path skips the per-block spent-UTXO reads the
+ /// address-balance update depends on.
+ pub(crate) fn defers_transparent_spends(&self, height: Height) -> bool {
+ self.config()
+ .defers_transparent_spends_at(&self.network(), height)
+ }
+
+ /// Whether the deferred-transparent reconcile is configured for this node (the
+ /// height-independent lifecycle predicate: worker spawn, drain triggers, the
+ /// handoff barrier). The per-block decision is the height-bound
+ /// [`defers_transparent_spends`](Self::defers_transparent_spends).
+ pub(crate) fn defer_reconcile_configured(&self) -> bool {
+ self.config().defer_reconcile_configured()
+ }
+
/// Returns the configured database kind for this database.
pub fn db_kind(&self) -> String {
self.db.db_kind()
diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs
index 75598a299c1..2c26b99c961 100644
--- a/zebra-state/src/service/finalized_state/zebra_db/block.rs
+++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs
@@ -37,7 +37,7 @@ use crate::{
MAX_BLOCK_REORG_HEIGHT, MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_PRUNE_HEIGHTS_PER_COMMIT,
},
error::{CommitCheckpointVerifiedError, CommitHeaderRangeError},
- request::FinalizedBlock,
+ request::{AuthenticatedCheckpointHash, FinalizedBlock},
service::check,
service::finalized_state::{
disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk},
@@ -56,6 +56,8 @@ use crate::{
#[cfg(feature = "indexer")]
use crate::request::Spend;
+use super::super::pipeline::{BlockBatchOutputs, FinalizedPipeline, PipelineBatchContribution};
+
#[cfg(test)]
mod tests;
@@ -403,6 +405,83 @@ impl ZebraDb {
}
}
+ /// Returns the hash at `height` from a committed body, falling back to a persisted Zakura
+ /// header-only row. `None` if neither is present.
+ fn header_hash_at(&self, height: block::Height) -> Option {
+ self.hash(height)
+ .or_else(|| self.zakura_header_hash(height))
+ }
+
+ /// Returns the highest checkpoint height whose header hash is positively re-established against
+ /// the hardcoded checkpoint list. Every height at or below this is checkpoint-authenticated.
+ ///
+ /// Used to bound checkpoint-class body scheduling (airtight ordering): the Zakura checkpoint
+ /// fast-commit path has no fallback, so a checkpoint-class body must never be requested before
+ /// its height is authenticated here.
+ pub fn authenticated_header_tip(&self) -> Option {
+ let checkpoints = self.network().checkpoint_list();
+ let persisted_tip = self.best_header_tip().map(|(height, _)| height)?;
+
+ // Walk checkpoints down from the highest at/below the persisted tip until one is positively
+ // authenticated. Normally the first matches; we do not trust persistence to have checked it
+ // (see `authenticated_checkpoint_hash`). The body tip's enclosing checkpoint is always
+ // authenticated because it was committed from a verified block, so this terminates.
+ let mut candidate = checkpoints.max_height_in_range(..=persisted_tip);
+ while let Some(c) = candidate {
+ if self.header_hash_at(c) == checkpoints.hash(c) {
+ return Some(c);
+ }
+ candidate = c
+ .previous()
+ .ok()
+ .and_then(|below| checkpoints.max_height_in_range(..=below));
+ }
+
+ None
+ }
+
+ /// Returns the Zakura-authenticated header hash at `height`, if `height` is a header-only
+ /// frontier height (above the committed body tip) whose enclosing checkpoint has been reached
+ /// and authenticated; otherwise `None`.
+ ///
+ /// The hash is taken **strictly from the Zakura header store**, never from committed body state,
+ /// so the token means exactly "the checkpoint-authenticated Zakura header hash for `height`".
+ /// Authentication is established positively at the gate: for the next checkpoint `C >= height`,
+ /// the persisted Zakura header hash at `C` must equal the hardcoded `checkpoint_list.hash(C)`.
+ ///
+ /// Continuity is sound because `height` and `C` are both in the un-trimmed frontier above the
+ /// body tip (the Zakura store releases a row only once its body commits), so every height in
+ /// `[height, C]` is also in the frontier; header sync persists only anchor-linked contiguous
+ /// ranges, so continuity from `height` up to the pinned `C` authenticates `height`'s hash. An
+ /// already-committed height returns `None` here (its Zakura row has been released), so the fast
+ /// path never re-commits or authenticates from body state.
+ ///
+ /// This is the only constructor of [`AuthenticatedCheckpointHash`].
+ pub fn authenticated_checkpoint_hash(
+ &self,
+ height: block::Height,
+ ) -> Option {
+ let checkpoints = self.network().checkpoint_list();
+
+ // The next checkpoint at or above `height`. `None` => above the last checkpoint, i.e. not a
+ // checkpoint-anchored height (the caller must not fast-commit it).
+ let next_checkpoint = checkpoints.min_height_in_range(height..)?;
+
+ // Construct strictly from the Zakura header frontier: `height` must be a header-only frontier
+ // height. (`None` for an already-committed height, so we never authenticate from body
+ // state.)
+ let header_hash = self.zakura_header_hash(height)?;
+
+ // Positively re-check the checkpoint anchor against the Zakura frontier. If the frontier has
+ // not reached `C`, or the persisted hash there does not match the hardcoded checkpoint,
+ // `height` is not yet authenticated.
+ if self.zakura_header_hash(next_checkpoint) != checkpoints.hash(next_checkpoint) {
+ return None;
+ }
+
+ Some(AuthenticatedCheckpointHash::new(header_hash))
+ }
+
/// Returns a contiguous ascending header range from full blocks and Zakura header rows.
pub fn headers_by_height_range(
&self,
@@ -482,10 +561,21 @@ impl ZebraDb {
let count = limit.min(best_header_tip.0.saturating_sub(start.0).saturating_add(1));
+ // Airtight ordering: never schedule a checkpoint-class body (`height <= checkpoint_max`)
+ // before header sync has checkpoint-authenticated that height. The Zakura checkpoint
+ // fast-commit path has no fallback, so requesting such a body early would later trip a hard
+ // invariant. Full-class heights (above the last checkpoint) are unaffected.
+ let checkpoint_max = self.network().checkpoint_list().max_height();
+ let authenticated_header_tip = self.authenticated_header_tip();
+
self.headers_by_height_range(start, count)
.into_iter()
.map(|(height, _, _)| height)
.filter(|height| !self.contains_body_at_height(*height))
+ .filter(|height| {
+ *height > checkpoint_max
+ || authenticated_header_tip.is_some_and(|tip| *height <= tip)
+ })
.take(limit as usize)
.collect()
}
@@ -919,18 +1009,26 @@ impl ZebraDb {
// Write block methods
- /// Write `finalized` to the finalized state.
+ /// Commit `finalized` to the finalized state: assemble its [`DiskWriteBatch`]
+ /// and immediately flush it to disk.
+ ///
+ /// This is the synchronous (tip-mode) entry point. The run-ahead committer
+ /// instead calls [`assemble_block_batch`](Self::assemble_block_batch) on the
+ /// assembler thread and [`flush_block_batch`](Self::flush_block_batch) on a
+ /// separate disk-writer thread, so the next block's assembly overlaps this
+ /// block's flush.
///
- /// Uses:
- /// - `history_tree`: the current tip's history tree
- /// - `network`: the configured network
- /// - `source`: the source of the block in log messages
+ /// Production commits go through [`FinalizedState::assemble_finalized_direct`]
+ /// and [`FinalizedState::flush_finalized_direct`], which call the two halves
+ /// below directly; this combined wrapper is retained as a synchronous one-shot
+ /// for tests, so it is unused in lib-only builds.
///
/// # Errors
///
/// - Propagates any errors from writing to the DB
/// - Propagates any errors from computing the block's chain value balance change or
/// from applying the change to the chain value balance
+ #[allow(dead_code)]
#[allow(clippy::unwrap_in_result)]
#[allow(clippy::too_many_arguments)]
pub(in super::super) fn write_block(
@@ -940,12 +1038,64 @@ impl ZebraDb {
network: &Network,
source: &str,
retention: RetentionPlan,
+ vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>,
+ vct_sync_below: Option,
+ ) -> Result {
+ let (batch, hash, _contribution, commit_trace) = self.assemble_block_batch(
+ finalized,
+ prev_note_commitment_trees,
+ network,
+ retention,
+ vct_anchor_roots,
+ vct_sync_below,
+ None,
+ )?;
+ self.flush_block_batch(batch, commit_trace, source);
+ Ok(hash)
+ }
+
+ /// Assemble the [`DiskWriteBatch`] for `finalized` without writing it to disk.
+ ///
+ /// This is the read/compute half of a block commit: it reads the spent UTXOs,
+ /// changed-address balances, and current value pool, applies the transparent
+ /// and shielded batch preparation, and returns the fully-built batch plus the
+ /// committed block hash. It performs no writes, so it can run on the assembler
+ /// thread (or the look-ahead) while a previously-assembled batch is being
+ /// flushed by the disk-writer thread.
+ ///
+ /// The reads it performs (`read_spent_utxo`, [`address_balance_location`], the
+ /// value pool) depend on the parent block. In run-ahead (sync) mode the parent
+ /// may not yet be durable on disk, so those reads must be served from the
+ /// in-memory pipeline overlay before falling back to disk; see the caller.
+ ///
+ /// [`address_balance_location`]: ZebraDb::address_balance_location
+ #[allow(clippy::too_many_arguments)]
+ pub(in super::super) fn assemble_block_batch(
+ &self,
+ finalized: FinalizedBlock,
+ prev_note_commitment_trees: Option,
+ network: &Network,
+ retention: RetentionPlan,
// When `Some`, skip per-height tree writes and fold these roots into
// the anchor set.
vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>,
// When `Some(height)`, mark the database as vct-synced.
vct_sync_below: Option,
- ) -> Result {
+ // The run-ahead pipeline's in-memory tip state and overlay. When `Some`,
+ // the parent-block reads (spent UTXOs, address balances, value pool, and
+ // the `vct_upgrade_height` marker) are served from it before falling back
+ // to disk, and this block's contribution is captured and returned.
+ overlay: Option<&FinalizedPipeline>,
+ ) -> Result<
+ (
+ DiskWriteBatch,
+ block::Hash,
+ Option,
+ super::super::commit_pressure::PreparedCommitTrace,
+ ),
+ CommitCheckpointVerifiedError,
+ > {
+ let write_start = std::time::Instant::now();
let tx_hash_indexes: HashMap = finalized
.transaction_hashes
.iter()
@@ -970,13 +1120,52 @@ impl ZebraDb {
.collect();
// Get a list of the spent UTXOs, before we delete any from the database.
- let outpoints: Vec = finalized
- .block
- .transactions
- .iter()
- .flat_map(|tx| tx.inputs().iter())
- .flat_map(|input| input.outpoint())
- .collect();
+ //
+ // Per-checkpoint transparent reconcile (`defer_transparent_reconcile`): in the
+ // deferred range, record this block's spent outpoints into the reconcile window
+ // and pass NO outpoints to the read loop, so the per-block spent-UTXO resolution,
+ // the `utxo_by_out_loc` deletes, and the transparent value-pool debit are all
+ // skipped here. Unlike the probe below, this is correct: the checkpoint reconcile
+ // resolves, deletes, and debits in one batched pass. The recorded outpoints (and
+ // the block) are threaded into the pipeline contribution below; deferral only
+ // takes effect on the run-ahead pipeline path, which the bench always uses.
+ //
+ // Benchmark-only ceiling probe (`ZEBRA_BENCH_SKIP_TRANSPARENT_READS=1`): drop the
+ // spent outpoints with NO reconcile, so the spend work is skipped entirely. This
+ // measures the throughput ceiling of deferring that work off the commit critical
+ // path. It produces an INCORRECT value pool and UTXO set, so it is never a shipped
+ // path — only a measurement of the upper bound.
+ let defer_spends = self.defers_transparent_spends(finalized.height);
+ // Deferral records into the run-ahead pipeline's reconcile window, so it is
+ // only correct when an overlay is present. Without one there is nowhere to
+ // record the spends and the checkpoint reconcile would never run, silently
+ // corrupting the value pool / UTXO set. The bench always runs pipelined.
+ assert!(
+ !defer_spends || overlay.is_some(),
+ "defer_transparent_reconcile requires the run-ahead pipeline \
+ (finalized_block_pipeline_depth > 0)"
+ );
+ let mut deferred_spent: Vec = Vec::new();
+ let outpoints: Vec = if defer_spends {
+ deferred_spent = finalized
+ .block
+ .transactions
+ .iter()
+ .flat_map(|tx| tx.inputs().iter())
+ .flat_map(|input| input.outpoint())
+ .collect();
+ Vec::new()
+ } else if super::bench_skip_transparent_reads() {
+ Vec::new()
+ } else {
+ finalized
+ .block
+ .transactions
+ .iter()
+ .flat_map(|tx| tx.inputs().iter())
+ .flat_map(|input| input.outpoint())
+ .collect()
+ };
// Serialize the raw transaction bytes for `tx_by_loc` concurrently with the
// spent-UTXO reads. Serialization is CPU-bound while the reads wait on disk,
@@ -985,6 +1174,25 @@ impl ZebraDb {
// `None` it serializes inline (e.g. the semantic path).
let store_raw_txs = retention.stores_raw_transactions();
let db: &ZebraDb = self;
+ // Resolve a spent output from the run-ahead overlay first (it may have been
+ // created by a not-yet-flushed block), then from disk / this block's own
+ // new outputs. With `overlay = None` this is exactly the disk read.
+ let read_one_spent =
+ |outpoint: transparent::OutPoint| -> (transparent::OutPoint, OutputLocation, transparent::Utxo) {
+ if let Some(overlay) = overlay {
+ if let Some((out_loc, utxo)) = overlay.spent_utxo_override(&outpoint) {
+ return (outpoint, out_loc, utxo);
+ }
+ }
+ read_spent_utxo(
+ db,
+ finalized.height,
+ outpoint,
+ &tx_hash_indexes,
+ &finalized.new_outputs,
+ )
+ };
+ let spent_reads_start = std::time::Instant::now();
let (spent_utxos, precomputed_raw_txs): (
Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>,
Option>,
@@ -992,31 +1200,9 @@ impl ZebraDb {
|| {
if outpoints.len() >= super::PARALLEL_BLOCK_READ_THRESHOLD {
use rayon::prelude::*;
- outpoints
- .into_par_iter()
- .map(|outpoint| {
- read_spent_utxo(
- db,
- finalized.height,
- outpoint,
- &tx_hash_indexes,
- &finalized.new_outputs,
- )
- })
- .collect()
+ outpoints.into_par_iter().map(&read_one_spent).collect()
} else {
- outpoints
- .into_iter()
- .map(|outpoint| {
- read_spent_utxo(
- db,
- finalized.height,
- outpoint,
- &tx_hash_indexes,
- &finalized.new_outputs,
- )
- })
- .collect()
+ outpoints.into_iter().map(&read_one_spent).collect()
}
},
|| {
@@ -1035,6 +1221,44 @@ impl ZebraDb {
}
},
);
+ let reads_dur = spent_reads_start.elapsed();
+ #[cfg(feature = "commit-metrics")]
+ metrics::histogram!("zebra.state.write.spent_utxo_reads.duration_seconds")
+ .record(reads_dur.as_secs_f64());
+
+ // Spend-distance instrumentation (trace-only): how many blocks back was each
+ // spent UTXO created? The cumulative buckets give the hit rate an in-memory
+ // UTXO cache of a given size would achieve, since the run-ahead overlay already
+ // serves only the pipeline-depth window. Built only when stage timing is on.
+ if zebra_chain::stage_timing::enabled() {
+ let spend_h = finalized.height.0;
+ let (mut le4k, mut le16k, mut le65k, mut le262k, mut le1m) =
+ (0u64, 0u64, 0u64, 0u64, 0u64);
+ for (_op, out_loc, _utxo) in &spent_utxos {
+ let d = spend_h.saturating_sub(out_loc.height().0);
+ if d <= 4_096 {
+ le4k += 1;
+ }
+ if d <= 16_384 {
+ le16k += 1;
+ }
+ if d <= 65_536 {
+ le65k += 1;
+ }
+ if d <= 262_144 {
+ le262k += 1;
+ }
+ if d <= 1_048_576 {
+ le1m += 1;
+ }
+ }
+ zebra_chain::stage_timing::record_val(spend_h, "spent_total", spent_utxos.len() as u64);
+ zebra_chain::stage_timing::record_val(spend_h, "spent_le4k", le4k);
+ zebra_chain::stage_timing::record_val(spend_h, "spent_le16k", le16k);
+ zebra_chain::stage_timing::record_val(spend_h, "spent_le65k", le65k);
+ zebra_chain::stage_timing::record_val(spend_h, "spent_le262k", le262k);
+ zebra_chain::stage_timing::record_val(spend_h, "spent_le1m", le1m);
+ }
let spent_utxos_by_outpoint: HashMap =
spent_utxos
@@ -1053,21 +1277,6 @@ impl ZebraDb {
.map(|(_outpoint, out_loc, utxo)| (out_loc, utxo))
.collect();
- // Get the transparent addresses with changed balances/UTXOs
- let changed_addresses: HashSet = spent_utxos_by_out_loc
- .values()
- .chain(
- finalized
- .new_outputs
- .values()
- .map(|ordered_utxo| &ordered_utxo.utxo),
- )
- .filter_map(|utxo| utxo.output.address(network))
- .unique()
- .collect();
-
- // Get the current address balances, before the transactions in this block
-
// Like the spent-UTXO reads above, the per-address balance lookups are
// cache-served but serial. Fan them across the rayon pool once a block
// touches enough addresses to amortize the fork-join cost.
@@ -1101,20 +1310,71 @@ impl ZebraDb {
// reading all of the pending merge operands (potentially hundreds), and applying pending merge operands to the
// fully-merged value such that it's much faster to read entries that have been updated with insertions than it
// is to read entries that have been updated with merge operations.
- let address_balances: AddressBalanceLocationUpdates = if self.finished_format_upgrades() {
- AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| {
- self.address_balance_location(addr)
- }))
+ //
+ // When the address index is skipped (fast-sync), none of the per-address
+ // balance reads happen and `address_balances` is left empty; the gated
+ // transparent index passes below then write no address entries.
+ let address_reads_start = std::time::Instant::now();
+ let address_balances: AddressBalanceLocationUpdates = if self.config().skip_address_index()
+ {
+ AddressBalanceLocationUpdates::Insert(HashMap::new())
} else {
- AddressBalanceLocationUpdates::Merge(read_addr_locs(changed_addresses, |addr| {
- Some(self.address_balance_location(addr)?.into_new_change())
- }))
+ // Transparent addresses with changed balances/UTXOs in this block.
+ let changed_addresses: HashSet = spent_utxos_by_out_loc
+ .values()
+ .chain(
+ finalized
+ .new_outputs
+ .values()
+ .map(|ordered_utxo| &ordered_utxo.utxo),
+ )
+ .filter_map(|utxo| utxo.output.address(network))
+ .unique()
+ .collect();
+ // Resolve an address balance from the run-ahead overlay first (it may
+ // have been updated by a not-yet-flushed block), then from disk.
+ let lookup_balance = |addr: &transparent::Address| {
+ if let Some(overlay) = overlay {
+ if let Some(balance) = overlay.address_balance_override(addr) {
+ return Some(balance);
+ }
+ }
+ self.address_balance_location(addr)
+ };
+ if self.finished_format_upgrades() {
+ AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| {
+ lookup_balance(addr)
+ }))
+ } else {
+ AddressBalanceLocationUpdates::Merge(read_addr_locs(changed_addresses, |addr| {
+ Some(lookup_balance(addr)?.into_new_change())
+ }))
+ }
};
-
+ let address_reads_dur = address_reads_start.elapsed();
+ #[cfg(feature = "commit-metrics")]
+ metrics::histogram!("zebra.state.write.address_reads.duration_seconds")
+ .record(address_reads_dur.as_secs_f64());
+
+ // The value pool and `vct_upgrade_height` marker are threaded forward in
+ // memory by the run-ahead pipeline; with `overlay = None` they come from
+ // disk exactly as before. `capture_pipeline_outputs` is set only when
+ // running ahead, so the synchronous path avoids the balance clone.
+ let value_pool = overlay
+ .map(|overlay| overlay.value_pool())
+ .unwrap_or_else(|| self.finalized_value_pool());
+ let pipeline_vct_marker_set = overlay
+ .map(|overlay| overlay.vct_upgrade_marker_set())
+ .unwrap_or(false);
+ let capture_pipeline_outputs = overlay.is_some();
+
+ let assembly_start = std::time::Instant::now();
let mut batch = DiskWriteBatch::new();
// In case of errors, propagate and do not write the batch.
- batch.prepare_block_batch(
+ #[cfg(feature = "commit-metrics")]
+ let batch_prep_start = std::time::Instant::now();
+ let block_outputs = batch.prepare_block_batch(
self,
network,
&finalized,
@@ -1124,12 +1384,14 @@ impl ZebraDb {
#[cfg(feature = "indexer")]
out_loc_by_outpoint,
address_balances,
- self.finalized_value_pool(),
+ value_pool,
prev_note_commitment_trees,
store_raw_txs,
precomputed_raw_txs,
vct_anchor_roots,
vct_sync_below,
+ pipeline_vct_marker_set,
+ capture_pipeline_outputs,
)?;
// In pruned storage mode, delete raw transaction history that has fallen
@@ -1139,18 +1401,111 @@ impl ZebraDb {
// block commit path. In archive mode the plan is always `Store`, so this
// is a no-op.
retention.prepare_prune(&mut batch, self, &finalized);
+ let batch_assembly_dur = assembly_start.elapsed();
+ #[cfg(feature = "commit-metrics")]
+ {
+ metrics::histogram!("zebra.state.write.batch_prep.duration_seconds")
+ .record(batch_prep_start.elapsed().as_secs_f64());
+ metrics::histogram!("zebra.state.write.batch_bytes")
+ .record(batch.size_in_bytes() as f64);
+ }
+ // When running ahead, capture this block's overlay contribution: the
+ // outputs it created (so the next block can spend them from memory) and
+ // the absolute address balances it updated.
+ let contribution = overlay.map(|_| {
+ let created_outputs = finalized
+ .new_outputs
+ .iter()
+ .map(|(outpoint, ordered_utxo)| {
+ (
+ *outpoint,
+ lookup_out_loc(finalized.height, outpoint, &tx_hash_indexes),
+ ordered_utxo.utxo.clone(),
+ )
+ })
+ .collect();
+ let updated_balances = match block_outputs.address_balances {
+ Some(AddressBalanceLocationUpdates::Insert(balances)) => {
+ balances.into_iter().collect()
+ }
+ // The pipeline only runs once format upgrades are finished (the
+ // `Insert` path), so `Merge`/`None` contribute no balances.
+ _ => Vec::new(),
+ };
+
+ PipelineBatchContribution {
+ value_pool: block_outputs.value_pool,
+ wrote_vct_upgrade_marker: block_outputs.wrote_vct_upgrade_marker,
+ created_outputs,
+ updated_balances,
+ // Per-checkpoint reconcile: when deferring, hand the spent outpoints
+ // + block + deferred-pool change to the pipeline window so the
+ // checkpoint reconcile can resolve, delete, and re-debit them. Empty
+ // / `None` when not deferring (no behavior change).
+ deferred_spent,
+ deferred_block: defer_spends.then(|| finalized.block.clone()),
+ deferred_pool_change: finalized.deferred_pool_balance_change,
+ }
+ });
+
+ let commit_trace = super::super::commit_pressure::PreparedCommitTrace {
+ height: finalized.height.0,
+ block: finalized.block.clone(),
+ tx_count: finalized.transaction_hashes.len(),
+ output_count: finalized.new_outputs.len(),
+ batch_keys: batch.len(),
+ batch_bytes: batch.size_in_bytes(),
+ reads: reads_dur,
+ address_reads: address_reads_dur,
+ batch_assembly: batch_assembly_dur,
+ // The VCT fold runs before `assemble_block_batch`; the caller
+ // (`assemble_finalized_direct`) sets this after it returns.
+ fold: std::time::Duration::ZERO,
+ // Self-time of the read/compute half (excludes the queue-wait + flush
+ // that `commit_total` also captures).
+ assemble_self: write_start.elapsed(),
+ write_start,
+ };
+
+ Ok((batch, finalized.hash, contribution, commit_trace))
+ }
+
+ /// Flush a previously [`assemble_block_batch`](Self::assemble_block_batch)d
+ /// batch to disk.
+ ///
+ /// This is the write half of a block commit. In run-ahead (sync) mode it runs
+ /// on the dedicated disk-writer thread so it overlaps the next block's
+ /// assembly; in tip mode it runs inline. A rocksdb write failure is fatal, as
+ /// before.
+ pub(crate) fn flush_block_batch(
+ &self,
+ batch: DiskWriteBatch,
+ commit_trace: super::super::commit_pressure::PreparedCommitTrace,
+ source: &str,
+ ) {
// Track batch commit latency for observability
let batch_start = std::time::Instant::now();
self.db
.write(batch)
.expect("unexpected rocksdb error while writing block");
+ let batch_commit = batch_start.elapsed();
metrics::histogram!("zebra.state.rocksdb.batch_commit.duration_seconds")
- .record(batch_start.elapsed().as_secs_f64());
+ .record(batch_commit.as_secs_f64());
+ let total_dur = commit_trace.write_start.elapsed();
+ // Optional per-slow-commit pressure row (inert unless ZEBRA_COMMIT_PRESSURE_TRACE
+ // is set): pairs this commit's latency with RocksDB L0/compaction/flush state.
+ // Timed separately because, when enabled, it re-serializes the block and samples
+ // RocksDB stats on the disk-writer's critical path (a Heisenberg cost to subtract).
+ let record_start = std::time::Instant::now();
+ super::super::commit_pressure::record_commit(&self.db, commit_trace, batch_commit);
+ metrics::histogram!("zebra.state.write.commit_trace_record.duration_seconds")
+ .record(record_start.elapsed().as_secs_f64());
+
+ metrics::histogram!("zebra.state.write.total.duration_seconds")
+ .record(total_dur.as_secs_f64());
tracing::trace!(?source, "committed block from");
-
- Ok(finalized.hash)
}
/// Writes the given batch to the database.
@@ -1191,6 +1546,42 @@ impl ZebraDb {
error: error.to_string(),
})
}
+
+ /// Seeds the Zakura header store with frontier header rows for a contiguous run
+ /// of blocks whose bodies have **not** been committed (heights strictly above the
+ /// body tip), writing in chunked batches.
+ ///
+ /// Offline-bench only: this stands in for what header sync would persist, so the
+ /// header-authenticated checkpoint fast path (`authenticated_checkpoint_hash`) can
+ /// run against a snapshot with no live header sync. Each row is independent and the
+ /// insert is idempotent, so callers may seed in any order.
+ pub fn seed_zakura_headers_from_blocks(
+ &self,
+ blocks: impl IntoIterator)>,
+ ) -> Result<(), CommitHeaderRangeError> {
+ let write = |batch: DiskWriteBatch| {
+ self.db
+ .write(batch)
+ .map_err(|error| CommitHeaderRangeError::StorageWriteError {
+ error: error.to_string(),
+ })
+ };
+
+ let mut batch = DiskWriteBatch::new();
+ let mut pending = 0usize;
+ for (height, block) in blocks {
+ batch.prepare_zakura_header_from_committed_block(&self.db, height, &block)?;
+ pending += 1;
+ if pending >= 2000 {
+ write(std::mem::take(&mut batch))?;
+ pending = 0;
+ }
+ }
+ if pending > 0 {
+ write(batch)?;
+ }
+ Ok(())
+ }
}
/// Read a spent transparent UTXO and its output location before deleting it from the database.
@@ -1462,7 +1853,7 @@ impl DiskWriteBatch {
/// - Propagates any errors from computing the block's chain value balance change or
/// from applying the change to the chain value balance
#[allow(clippy::too_many_arguments)]
- pub fn prepare_block_batch(
+ pub(crate) fn prepare_block_batch(
&mut self,
zebra_db: &ZebraDb,
network: &Network,
@@ -1481,7 +1872,19 @@ impl DiskWriteBatch {
precomputed_raw_txs: Option>,
vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>,
vct_sync_below: Option,
- ) -> Result<(), CommitCheckpointVerifiedError> {
+ // When `true`, a not-yet-flushed pipeline block already wrote the set-once
+ // `vct_upgrade_height` marker, so this block must not write it again.
+ pipeline_vct_marker_set: bool,
+ // When `true`, capture and return this block's post-update value pool and
+ // absolute address balances for the run-ahead pipeline overlay.
+ capture_pipeline_outputs: bool,
+ ) -> Result {
+ // Per-category key-count attribution (env-gated; no-op when stage timing
+ // is off). Each step's `batch.len()` delta is how many keys/deletes that
+ // category contributes to the commit batch (the compaction input).
+ let kh = finalized.height.0;
+ let mut kp = self.len();
+
// Commit block, transaction, and note commitment tree data.
self.prepare_block_header_and_transaction_data_batch(
zebra_db,
@@ -1489,11 +1892,16 @@ impl DiskWriteBatch {
store_raw_transactions,
precomputed_raw_txs,
)?;
+ zebra_chain::stage_timing::record_val(kh, "keys_header_tx", (self.len() - kp) as u64);
+ kp = self.len();
+
let zakura_header_commitment_roots_by_height = zebra_db
.db
.cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT)
.unwrap();
self.zs_delete(&zakura_header_commitment_roots_by_height, finalized.height);
+ zebra_chain::stage_timing::record_val(kh, "keys_vct_evict", (self.len() - kp) as u64);
+ kp = self.len();
// The consensus rules are silent on shielded transactions in the genesis block,
// because there aren't any in the mainnet or testnet genesis blocks.
@@ -1502,13 +1910,26 @@ impl DiskWriteBatch {
//
// In Zebra we include the nullifiers and note commitments in the genesis block because it simplifies our code.
self.prepare_shielded_transaction_batch(zebra_db, finalized);
+ zebra_chain::stage_timing::record_val(kh, "keys_shielded", (self.len() - kp) as u64);
+ kp = self.len();
+
+ // The `vct_upgrade_height` marker is written once, by the first block this
+ // binary commits. In the run-ahead pipeline an earlier not-yet-flushed block
+ // may have already set it in memory (`pipeline_vct_marker_set`), in which case
+ // its disk write is still pending, so the disk read would wrongly look absent;
+ // suppress the duplicate write here and report which block actually wrote it.
+ let wrote_vct_upgrade_marker =
+ !pipeline_vct_marker_set && zebra_db.vct_upgrade_height().is_none();
self.prepare_trees_batch(
zebra_db,
finalized,
prev_note_commitment_trees,
vct_anchor_roots,
vct_sync_below,
+ wrote_vct_upgrade_marker,
);
+ zebra_chain::stage_timing::record_val(kh, "keys_trees", (self.len() - kp) as u64);
+ kp = self.len();
// # Consensus
//
@@ -1520,9 +1941,10 @@ impl DiskWriteBatch {
// So we ignore the genesis UTXO, transparent address index, and value pool updates
// for the genesis block. This also ignores genesis shielded value pool updates, but there
// aren't any of those on mainnet or testnet.
+ let mut captured_address_balances = None;
if !finalized.height.is_min() {
// Commit transaction indexes
- self.prepare_transparent_transaction_batch(
+ captured_address_balances = self.prepare_transparent_transaction_batch(
zebra_db,
network,
finalized,
@@ -1532,21 +1954,31 @@ impl DiskWriteBatch {
#[cfg(feature = "indexer")]
&out_loc_by_outpoint,
address_balances,
+ capture_pipeline_outputs,
);
}
+ zebra_chain::stage_timing::record_val(kh, "keys_transparent", (self.len() - kp) as u64);
+ kp = self.len();
- // Commit UTXOs and value pools
- self.prepare_chain_value_pools_batch(
+ // Commit UTXOs and value pools. This runs for every height, including
+ // genesis (which writes the initial pool and block info), matching the
+ // original commit path.
+ let new_value_pool = self.prepare_chain_value_pools_batch(
zebra_db,
finalized,
spent_utxos_by_outpoint,
value_pool,
)?;
+ zebra_chain::stage_timing::record_val(kh, "keys_valuepool", (self.len() - kp) as u64);
// The block has passed contextual validation, so update the metrics
block_precommit_metrics(&finalized.block, finalized.hash, finalized.height);
- Ok(())
+ Ok(BlockBatchOutputs {
+ value_pool: new_value_pool,
+ address_balances: captured_address_balances,
+ wrote_vct_upgrade_marker,
+ })
}
/// Adds deletes for pruned raw transaction data to this batch, for the
diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs
index 2c412f9488e..1cd9fb4d6d0 100644
--- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs
+++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs
@@ -499,6 +499,30 @@ fn committed_block_releases_matching_zakura_header() {
.is_none());
}
+#[test]
+fn authenticated_checkpoint_hash_requires_zakura_frontier_row() {
+ let _init_guard = zebra_test::init();
+ let genesis = mainnet_block(0);
+ let block1 = mainnet_block(1);
+ let network = checkpoint_test_network(genesis.hash(), block1.hash());
+ let state = state_with_genesis(&network, genesis.clone());
+
+ commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header));
+
+ let authenticated_hash = state
+ .authenticated_checkpoint_hash(Height(1))
+ .expect("checkpoint-authenticated header-only frontier hash is available");
+ assert_eq!(authenticated_hash.hash(), block1.hash());
+
+ write_full_block_header_and_transactions(&state, block1.clone());
+
+ assert_eq!(
+ state.headers_by_height_range(Height(1), 1),
+ vec![(Height(1), block1.hash(), block1.header.clone())],
+ );
+ assert!(state.authenticated_checkpoint_hash(Height(1)).is_none());
+}
+
/// Committing a body at the bottom of a header-only frontier releases just that
/// height from the Zakura header store while the higher frontier rows remain,
/// and reads span the body/frontier boundary.
diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs
index e3532ad69a1..0b16a171261 100644
--- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs
+++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs
@@ -17,10 +17,11 @@ use std::{
};
use zebra_chain::{
- amount::NonNegative,
+ amount::{NegativeAllowed, NonNegative},
block::Height,
block_info::BlockInfo,
history_tree::HistoryTree,
+ parameters::Network,
serialization::{CompactSizeMessage, ZcashSerialize as _},
transparent,
value_balance::ValueBalance,
@@ -30,11 +31,13 @@ use crate::{
request::FinalizedBlock,
service::finalized_state::{
disk_db::{DiskWriteBatch, ReadDisk},
- disk_format::{chain::HistoryTreeParts, RawBytes},
+ disk_format::transparent::AddressBalanceLocationUpdates,
+ disk_format::{chain::HistoryTreeParts, OutputLocation, RawBytes},
+ pipeline::ReconcileBlock,
zebra_db::{metrics::value_pool_metrics, ZebraDb},
TypedColumnFamily,
},
- HashOrHeight, ValidateContextError,
+ BoxError, HashOrHeight, ValidateContextError,
};
/// The name of the History Tree column family.
@@ -219,6 +222,26 @@ impl ZebraDb {
.unwrap_or_else(ValueBalance::zero)
}
+ /// Verification helper (offline tools): the count of `block_info` entries and
+ /// the lowest/highest heights missing a `BlockInfo` in `[lo, hi]`.
+ pub fn block_info_coverage(&self, lo: Height, hi: Height) -> (u64, Option, u64) {
+ let cf = self.block_info_cf();
+ let mut count = 0u64;
+ let mut first_missing = None;
+ let mut missing = 0u64;
+ for height in lo.0..=hi.0 {
+ if cf.zs_get(&Height(height)).is_some() {
+ count += 1;
+ } else {
+ if first_missing.is_none() {
+ first_missing = Some(Height(height));
+ }
+ missing += 1;
+ }
+ }
+ (count, first_missing, missing)
+ }
+
/// Returns the stored `BlockInfo` for the given block.
pub fn block_info(&self, hash_or_height: HashOrHeight) -> Option {
let height = hash_or_height.height_or_else(|hash| self.height(hash))?;
@@ -227,6 +250,141 @@ impl ZebraDb {
block_info_cf.zs_get(&height)
}
+
+ /// Per-checkpoint transparent reconcile: write, in one atomic batch, the
+ /// `utxo_by_out_loc` deletes for every deferred spend in `records` and the
+ /// recomputed chain value pool (the tip pool plus per-height `BlockInfo`).
+ /// Returns the value pool after the last block in `records`.
+ ///
+ /// `start_value_pool` is the pool before the first block in `records`,
+ /// `resolved` maps each spent outpoint to its on-disk location and UTXO.
+ ///
+ /// This reproduces, in a batched pass, exactly what the per-block
+ /// [`prepare_spent_transparent_outputs_batch`](DiskWriteBatch::prepare_spent_transparent_outputs_batch)
+ /// (with the address index off) and
+ /// [`prepare_chain_value_pools_batch`](DiskWriteBatch::prepare_chain_value_pools_batch)
+ /// would have written inline, so the resulting state is byte-identical.
+ #[allow(clippy::unwrap_in_result)]
+ pub(crate) fn commit_checkpoint_reconcile(
+ &self,
+ network: &Network,
+ start_value_pool: ValueBalance,
+ records: &[ReconcileBlock],
+ resolved: &HashMap,
+ ) -> Result, BoxError> {
+ // Env-gated per-phase timing for the A2 bottleneck study.
+ let debug = std::env::var("ZRB_RECONCILE_DEBUG").is_ok();
+ let last_h = records.last().map(|r| r.height.0).unwrap_or(0);
+
+ let mut batch = DiskWriteBatch::new();
+
+ // The deletes: every resolved spend, keyed by output location. The address
+ // index is off in the deferred range, so `skip_index = true` makes this
+ // delete only the `utxo_by_out_loc` entries (no address-link deletes), and
+ // the empty `address_balances` is never consulted.
+ let deletes_start = std::time::Instant::now();
+ let spent_utxos_by_out_loc: BTreeMap = resolved
+ .values()
+ .map(|(out_loc, utxo)| (*out_loc, utxo.clone()))
+ .collect();
+ batch.prepare_spent_transparent_outputs_batch(
+ &self.db,
+ network,
+ &spent_utxos_by_out_loc,
+ &AddressBalanceLocationUpdates::Insert(HashMap::new()),
+ true,
+ );
+ let deletes_dur = deletes_start.elapsed();
+ let n_deletes = spent_utxos_by_out_loc.len();
+
+ // Compute each block's independent value-pool delta and serialized size in
+ // parallel — `chain_value_pool_change` re-folds every transaction's value
+ // balance and the size sums every transaction's serialized length, so this is
+ // the CPU-heavy pass, and it is independent across the window's blocks. Then
+ // apply the deltas in block order sequentially (cheap integer adds) to get the
+ // per-height running pool for `BlockInfo`. Splitting the parallel per-block
+ // work from the ordered running-sum keeps the result byte-identical to the
+ // inline per-block path (same deltas, same application order).
+ let vp_start = std::time::Instant::now();
+ use rayon::prelude::*;
+ let per_block: Vec<(ValueBalance, usize)> = records
+ .par_iter()
+ .map(
+ |record| -> Result<(ValueBalance, usize), BoxError> {
+ let spent_by_block: HashMap = record
+ .spent
+ .iter()
+ .map(|outpoint| {
+ let (_out_loc, utxo) = resolved
+ .get(outpoint)
+ .expect("every deferred spend was resolved above");
+ (*outpoint, utxo.clone())
+ })
+ .collect();
+
+ let delta = record
+ .block
+ .chain_value_pool_change(&spent_by_block, record.deferred_pool_change)?;
+
+ // Block size, summed per-transaction (byte-identical to serializing the
+ // whole block), as in `prepare_chain_value_pools_batch`.
+ let transactions = &record.block.transactions;
+ let transactions_size: usize = transactions
+ .iter()
+ .map(|transaction| transaction.zcash_serialized_size())
+ .sum();
+ let tx_count_size = CompactSizeMessage::try_from(transactions.len())
+ .expect("block must have a valid transaction count")
+ .zcash_serialized_size();
+ let block_size = record.block.header.zcash_serialized_size()
+ + tx_count_size
+ + transactions_size;
+
+ Ok((delta, block_size))
+ },
+ )
+ .collect::, BoxError>>()?;
+ let vp_dur = vp_start.elapsed();
+
+ let bi_start = std::time::Instant::now();
+ let mut value_pool = start_value_pool;
+ for (record, (delta, block_size)) in records.iter().zip(per_block) {
+ value_pool = value_pool.add_chain_value_pool_change(delta)?;
+ value_pool_metrics(&value_pool);
+
+ let _ = self
+ .block_info_cf()
+ .with_batch_for_writing(&mut batch)
+ .zs_insert(
+ &record.height,
+ &BlockInfo::new(value_pool, block_size as u32),
+ );
+ }
+
+ let _ = self
+ .chain_value_pools_cf()
+ .with_batch_for_writing(&mut batch)
+ .zs_insert(&(), &value_pool);
+ let bi_dur = bi_start.elapsed();
+
+ let write_start = std::time::Instant::now();
+ self.write_batch(batch)?;
+ let write_dur = write_start.elapsed();
+
+ if debug {
+ eprintln!(
+ "[recon-commit] last_h={last_h} n_blocks={} n_deletes={n_deletes} \
+ deletes_ms={:.1} vp_ms={:.1} blockinfo_ms={:.1} write_ms={:.1}",
+ records.len(),
+ deletes_dur.as_secs_f64() * 1e3,
+ vp_dur.as_secs_f64() * 1e3,
+ bi_dur.as_secs_f64() * 1e3,
+ write_dur.as_secs_f64() * 1e3,
+ );
+ }
+
+ Ok(value_pool)
+ }
}
impl DiskWriteBatch {
@@ -286,13 +444,25 @@ impl DiskWriteBatch {
/// [`chain_value_pool_change`]: zebra_chain::block::Block::chain_value_pool_change
/// [`add_chain_value_pool_change`]: ValueBalance::add_chain_value_pool_change
#[allow(clippy::unwrap_in_result)]
+ /// Returns the chain value pool after applying this block, so the run-ahead
+ /// committer can thread it forward in memory to the next block's assembly.
pub fn prepare_chain_value_pools_batch(
&mut self,
db: &ZebraDb,
finalized: &FinalizedBlock,
utxos_spent_by_block: HashMap,
value_pool: ValueBalance,
- ) -> Result<(), ValidateContextError> {
+ ) -> Result, ValidateContextError> {
+ // Per-checkpoint reconcile / ceiling probe: the value-pool change needs the spent
+ // values. When deferring (`defers_transparent_spends`), the spent reads are skipped
+ // here and recomputed in the batched checkpoint reconcile, so leave the pool
+ // unchanged (it is threaded forward and corrected at the next checkpoint). The
+ // benchmark probe (`bench_skip_transparent_reads`) skips it with no reconcile,
+ // leaving the pool permanently stale (measurement only, never shipped).
+ if db.defers_transparent_spends(finalized.height) || super::bench_skip_transparent_reads() {
+ return Ok(value_pool);
+ }
+
let block_value_pool_change = finalized
.block
.chain_value_pool_change(
@@ -364,6 +534,6 @@ impl DiskWriteBatch {
&BlockInfo::new(new_value_pool, block_size as u32),
);
- Ok(())
+ Ok(new_value_pool)
}
}
diff --git a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs
index bdcfc6dbf58..6d142fc6f93 100644
--- a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs
+++ b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs
@@ -653,6 +653,11 @@ impl DiskWriteBatch {
prev_note_commitment_trees: Option,
vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>,
vct_sync_below: Option,
+ // Whether to write the set-once `vct_upgrade_height` marker for this block.
+ // Computed by the caller (`!already_set && marker_is_absent`) so the run-ahead
+ // pipeline can suppress a duplicate write when an earlier not-yet-flushed block
+ // already set it.
+ write_vct_upgrade_marker: bool,
) {
let FinalizedBlock {
height,
@@ -670,7 +675,7 @@ impl DiskWriteBatch {
// a node that upgrades above the last checkpoint (legacy path only). Set-once: the marker
// is never moved, so the boundary stays stable as the chain grows. Commits are sequential,
// so the absent check sees the previous block's committed marker, not a half-written batch.
- if zebra_db.vct_upgrade_height().is_none() {
+ if write_vct_upgrade_marker {
self.update_vct_upgrade_marker(zebra_db, *height);
}
diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs
index a5aec765587..6c10a727012 100644
--- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs
+++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs
@@ -37,7 +37,7 @@ use crate::{
AddressBalanceLocationUpdates, AddressLocation, AddressTransaction,
AddressUnspentOutput, OutputLocation,
},
- TransactionLocation,
+ RawBytes, TransactionLocation,
},
zebra_db::ZebraDb,
},
@@ -83,6 +83,60 @@ pub type TransactionLocationBySpentOutputLocationCf<'cf> =
impl ZebraDb {
// Column family convenience methods
+ /// Verification helper (offline tools): a stable digest of the `utxo_by_out_loc`
+ /// column family — the entry count plus order-independent rolling hashes of the
+ /// raw key and value bytes — for comparing two databases' UTXO sets without
+ /// dumping every entry.
+ ///
+ /// Returns `(count, key_sum, key_xor, value_sum, value_xor)`. Two databases
+ /// have an identical UTXO set iff all five fields match. Used by the
+ /// per-checkpoint reconcile byte-match verification.
+ pub fn utxo_by_out_loc_verification_digest(&self) -> (u64, u64, u64, u64, u64) {
+ self.cf_verification_digest("utxo_by_out_loc")
+ }
+
+ /// Verification helper (offline tools): the same stable digest as
+ /// [`Self::utxo_by_out_loc_verification_digest`], over the `block_info` column
+ /// family — so two databases' per-height value pools can be compared.
+ pub fn block_info_verification_digest(&self) -> (u64, u64, u64, u64, u64) {
+ self.cf_verification_digest("block_info")
+ }
+
+ /// Order-independent rolling digest of a column family's raw key/value bytes:
+ /// `(count, key_sum, key_xor, value_sum, value_xor)`.
+ fn cf_verification_digest(&self, cf_name: &str) -> (u64, u64, u64, u64, u64) {
+ // FNV-1a over a byte slice.
+ fn fnv1a(bytes: &[u8]) -> u64 {
+ let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
+ for &byte in bytes {
+ hash ^= u64::from(byte);
+ hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
+ }
+ hash
+ }
+
+ let cf = self
+ .db
+ .cf_handle(cf_name)
+ .expect("column family was created when database was created");
+
+ let (mut count, mut key_sum, mut key_xor, mut value_sum, mut value_xor) = (0, 0, 0, 0, 0);
+ for (key, value) in self
+ .db
+ .zs_forward_range_iter::<_, RawBytes, RawBytes, _>(&cf, ..)
+ {
+ let key_hash = fnv1a(key.raw_bytes());
+ let value_hash = fnv1a(value.raw_bytes());
+ count += 1;
+ key_sum = u64::wrapping_add(key_sum, key_hash);
+ key_xor ^= key_hash;
+ value_sum = u64::wrapping_add(value_sum, value_hash);
+ value_xor ^= value_hash;
+ }
+
+ (count, key_sum, key_xor, value_sum, value_xor)
+ }
+
/// Returns a typed handle to the transaction location by spent output location column family.
pub(crate) fn tx_loc_by_spent_output_loc_cf(
&self,
@@ -417,6 +471,10 @@ impl DiskWriteBatch {
/// If this method returns an error, it will be propagated,
/// and the batch should not be written to the database.
#[allow(clippy::too_many_arguments)]
+ /// When `capture_balances` is true, returns the absolute address balances
+ /// after applying this block, so the run-ahead committer can serve the
+ /// next block's address reads from memory before the write is durable.
+ /// Returns `None` otherwise, avoiding a clone on the synchronous path.
pub fn prepare_transparent_transaction_batch(
&mut self,
zebra_db: &ZebraDb,
@@ -430,22 +488,32 @@ impl DiskWriteBatch {
OutputLocation,
>,
mut address_balances: AddressBalanceLocationUpdates,
- ) {
+ capture_balances: bool,
+ ) -> Option {
let db = &zebra_db.db;
let FinalizedBlock { block, height, .. } = finalized;
+ // A pruned, checkpoint-syncing node skips the auxiliary transparent address
+ // index entirely. The UTXO-set passes still run (they write/delete
+ // `utxo_by_out_loc`), but the address-balance update, the per-tx address
+ // indexing, and the address balance CF are all elided. The UTXO/value-pool/
+ // nullifier state is unchanged.
+ let skip_index = zebra_db.config().skip_address_index();
+
// Update the in-memory `address_balances` transaction-by-transaction, debiting inputs
// before crediting outputs within each transaction. This ordering keeps every
// intermediate per-address balance within the consensus range, even when the block
// contains a same-address transparent self-spend chain whose batch credit-first
// intermediate balance would otherwise exceed MAX_MONEY.
- Self::prepare_transparent_address_balance_updates(
- network,
- *height,
- &block.transactions,
- spent_utxos_by_outpoint,
- &mut address_balances,
- );
+ if !skip_index {
+ Self::prepare_transparent_address_balance_updates(
+ network,
+ *height,
+ &block.transactions,
+ spent_utxos_by_outpoint,
+ &mut address_balances,
+ );
+ }
// Write the new and spent transparent output index entries. These passes no longer
// touch `address_balances`; they only read each entry's `address_location()`.
@@ -454,31 +522,43 @@ impl DiskWriteBatch {
network,
new_outputs_by_out_loc,
&address_balances,
+ skip_index,
);
self.prepare_spent_transparent_outputs_batch(
db,
network,
spent_utxos_by_out_loc,
&address_balances,
+ skip_index,
);
// Index the transparent addresses that spent in each transaction
- for (tx_index, transaction) in block.transactions.iter().enumerate() {
- let spending_tx_location = TransactionLocation::from_usize(*height, tx_index);
+ if !skip_index {
+ for (tx_index, transaction) in block.transactions.iter().enumerate() {
+ let spending_tx_location = TransactionLocation::from_usize(*height, tx_index);
+
+ self.prepare_spending_transparent_tx_ids_batch(
+ zebra_db,
+ network,
+ spending_tx_location,
+ transaction,
+ spent_utxos_by_outpoint,
+ #[cfg(feature = "indexer")]
+ out_loc_by_outpoint,
+ &address_balances,
+ );
+ }
+ }
- self.prepare_spending_transparent_tx_ids_batch(
- zebra_db,
- network,
- spending_tx_location,
- transaction,
- spent_utxos_by_outpoint,
- #[cfg(feature = "indexer")]
- out_loc_by_outpoint,
- &address_balances,
- );
+ // Capture the post-block absolute balances for the pipeline overlay before
+ // they are moved into the write batch (cheap clone, only when running ahead).
+ let captured = capture_balances.then(|| address_balances.clone());
+
+ if !skip_index {
+ self.prepare_transparent_balances_batch(db, address_balances);
}
- self.prepare_transparent_balances_batch(db, address_balances);
+ captured
}
/// Update `address_balances` in memory for the transparent transfers in `transactions`,
@@ -591,6 +671,7 @@ impl DiskWriteBatch {
network: &Network,
new_outputs_by_out_loc: &BTreeMap,
address_balances: &AddressBalanceLocationUpdates,
+ skip_index: bool,
) {
let utxo_by_out_loc = db.cf_handle("utxo_by_out_loc").unwrap();
let utxo_loc_by_transparent_addr_loc =
@@ -601,7 +682,9 @@ impl DiskWriteBatch {
// Index all new transparent outputs
for (new_output_location, utxo) in new_outputs_by_out_loc {
let unspent_output = &utxo.output;
- let receiving_address = unspent_output.address(network);
+ let receiving_address = (!skip_index)
+ .then(|| unspent_output.address(network))
+ .flatten();
if let Some(receiving_address) = receiving_address {
let receiving_address_location = match address_balances {
@@ -663,6 +746,7 @@ impl DiskWriteBatch {
network: &Network,
spent_utxos_by_out_loc: &BTreeMap,
address_balances: &AddressBalanceLocationUpdates,
+ skip_index: bool,
) {
let utxo_by_out_loc = db.cf_handle("utxo_by_out_loc").unwrap();
let utxo_loc_by_transparent_addr_loc =
@@ -673,7 +757,9 @@ impl DiskWriteBatch {
// Coinbase inputs represent new coins, so there are no UTXOs to mark as spent.
for (spent_output_location, utxo) in spent_utxos_by_out_loc {
let spent_output = &utxo.output;
- let sending_address = spent_output.address(network);
+ let sending_address = (!skip_index)
+ .then(|| spent_output.address(network))
+ .flatten();
// Fetch the link from the address to the AddressLocation, from memory.
if let Some(sending_address) = sending_address {
diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs
index 81ad0d0c48c..f17d585961b 100644
--- a/zebra-state/src/service/tests.rs
+++ b/zebra-state/src/service/tests.rs
@@ -239,6 +239,59 @@ fn out_of_order_committing_strategy() -> BoxedStrategy>> {
Just(blocks).prop_shuffle().boxed()
}
+/// Committing checkpoint-verified blocks through the real `StateService` with the
+/// run-ahead pipeline enabled (`finalized_block_pipeline_depth > 0`) must succeed,
+/// exactly like the synchronous committer. This drives the full write-worker path
+/// (which the `FinalizedState`-only equivalence test bypasses), where the pipeline
+/// actually activates.
+#[tokio::test(flavor = "multi_thread")]
+async fn pipelined_state_service_commits_chain() -> Result<()> {
+ let _init_guard = zebra_test::init();
+ let network = Network::Mainnet;
+
+ let blocks: Vec> = zebra_test::vectors::MAINNET_BLOCKS
+ .range(0..=20)
+ .map(|(_, b)| b.zcash_deserialize_into::>().unwrap())
+ .collect();
+ let expected = blocks.len();
+
+ let mut config = Config::ephemeral();
+ config.finalized_block_pipeline_depth = 2;
+
+ let (state, _read, _latest, _change) =
+ StateService::new(config, &network, Height::MAX, 0).await;
+ let mut state = Buffer::new(BoxService::new(state), 10);
+
+ // Queue all the commits (so several are in flight, exercising the pipeline depth),
+ // then await each response in order.
+ let mut rsps = Vec::new();
+ for block in blocks {
+ let rsp = state
+ .ready()
+ .await
+ .expect("state service ready")
+ .call(Request::CommitCheckpointVerifiedBlock(block.into()));
+ rsps.push(rsp);
+ }
+ let mut committed = 0;
+ for (i, rsp) in rsps.into_iter().enumerate() {
+ rsp.await
+ .unwrap_or_else(|e| panic!("pipelined commit of block {i} failed: {e:?}"));
+ committed += 1;
+ }
+
+ assert_eq!(
+ committed, expected,
+ "every queued block should commit through the pipeline"
+ );
+ assert!(
+ expected >= 2,
+ "the test chain must have at least two blocks"
+ );
+
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn empty_state_still_responds_to_requests() -> Result<()> {
let _init_guard = zebra_test::init();
diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs
index 2a3f373b2cf..256c79d273b 100644
--- a/zebra-state/src/service/write.rs
+++ b/zebra-state/src/service/write.rs
@@ -17,7 +17,9 @@ use tokio::sync::{
};
use tracing::Span;
+use zebra_chain::amount::NonNegative;
use zebra_chain::block::{self, Height};
+use zebra_chain::value_balance::ValueBalance;
use zebra_chain::parallel::{
commitment_aux::BlockCommitmentRoots,
@@ -29,12 +31,15 @@ use crate::{
error::CommitHeaderRangeError,
service::{
check,
- finalized_state::{spawn_note_precompute, FinalizedState, ZebraDb},
+ finalized_state::{
+ reconcile_window, spawn_note_precompute, DiskWriteBatch, FinalizedPipeline,
+ FinalizedState, PreparedCommitTrace, ReconcileBlock, ZebraDb,
+ },
non_finalized_state::NonFinalizedState,
queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified},
ChainTipBlock, ChainTipSender, InvalidateError, ReconsiderError,
},
- SemanticallyVerifiedBlock, ValidateContextError,
+ CommitCheckpointVerifiedError, SemanticallyVerifiedBlock, ValidateContextError,
};
// These types are used in doc links
@@ -53,6 +58,276 @@ type PendingPrecompute = (
Arc,
);
+/// A finalized block batch handed to the disk-writer thread for flushing, in the
+/// run-ahead committer.
+///
+/// The assembler thread builds these against the in-memory overlay and sends them
+/// over a bounded channel; the disk-writer thread flushes each one to disk in
+/// height order, sends the per-block commit response only once durable, then acks
+/// the flushed height back so the assembler can advance the chain tip and retire
+/// the overlay.
+struct PipelineFlush {
+ batch: DiskWriteBatch,
+ commit_trace: PreparedCommitTrace,
+ hash: block::Hash,
+ height: Height,
+ rsp_tx: oneshot::Sender>,
+}
+
+/// The disk-writer thread loop for the run-ahead finalized committer.
+///
+/// Flushes each assembled batch to disk in arrival (height) order, sends the
+/// per-block commit response only once the write is durable (ack-after-flush),
+/// then acks the flushed height back to the assembler. A rocksdb write failure is
+/// fatal, as on the synchronous path.
+fn run_finalized_writer(
+ db: ZebraDb,
+ batch_receiver: std::sync::mpsc::Receiver,
+ ack_sender: std::sync::mpsc::Sender,
+) {
+ loop {
+ // Time blocked waiting for the next batch: ~0 means the disk-writer is the
+ // bottleneck (a full queue feeding it), high means it is starved (the
+ // assembler upstream is the bottleneck).
+ let recv_start = std::time::Instant::now();
+ let Ok(PipelineFlush {
+ batch,
+ commit_trace,
+ hash,
+ height,
+ rsp_tx,
+ }) = batch_receiver.recv()
+ else {
+ break;
+ };
+ metrics::histogram!("zebra.state.write.disk_writer_recv_wait.duration_seconds")
+ .record(recv_start.elapsed().as_secs_f64());
+
+ // Full per-block service time on the disk-writer (flush + acks): the inverse
+ // of this is the disk-writer's max throughput.
+ let service_start = std::time::Instant::now();
+ db.flush_block_batch(
+ batch,
+ commit_trace,
+ "commit checkpoint-verified request (pipelined)",
+ );
+
+ // Ack-after-flush: the commit response only resolves once the block is
+ // durable on disk.
+ let _ = rsp_tx.send(Ok(hash));
+
+ // If the assembler has gone away (shutdown), stop.
+ let stop = ack_sender.send(height).is_err();
+ metrics::histogram!("zebra.state.write.disk_writer_service.duration_seconds")
+ .record(service_start.elapsed().as_secs_f64());
+ if stop {
+ break;
+ }
+ }
+}
+
+/// A window of deferred transparent spends handed to the reconcile worker thread.
+///
+/// The window blocks (height-ordered, all at or below the boundary) are resolved,
+/// deleted, and re-debited off the assembler thread. Empty jobs are never sent.
+struct ReconcileJob {
+ records: Vec,
+}
+
+/// The deferred-transparent reconcile worker thread loop (v2).
+///
+/// Owns the running chain value pool (seeded once from disk by the spawner) and is
+/// the **sole** writer of `chain_value_pools`, `block_info`, and the spent
+/// `utxo_by_out_loc` deletes in the deferred range — the assembler and disk writer
+/// write only the disjoint UTXO creates / headers / nullifiers / trees, and skip
+/// the value pool entirely when deferring. So this runs concurrently with assembly
+/// without shared-state races or key conflicts.
+///
+/// Jobs are processed FIFO, so the value-pool chaining stays ordered. A reconcile
+/// failure is fatal, as a rocksdb write failure is on the disk writer.
+fn run_reconcile_worker(
+ db: ZebraDb,
+ mut value_pool: ValueBalance,
+ job_receiver: std::sync::mpsc::Receiver,
+) {
+ let network = db.network();
+ let debug = std::env::var("ZRB_RECONCILE_DEBUG").is_ok();
+ loop {
+ // Env-gated: time how long the worker sits idle waiting for the next window. A
+ // large wait means the worker is faster than the assembler produces intervals.
+ let recv_start = std::time::Instant::now();
+ let Ok(ReconcileJob { records }) = job_receiver.recv() else {
+ break;
+ };
+ if debug {
+ let last_h = records.last().map(|r| r.height.0).unwrap_or(0);
+ eprintln!(
+ "[recon-recv] last_h={last_h} recv_wait_ms={:.1}",
+ recv_start.elapsed().as_secs_f64() * 1e3,
+ );
+ }
+ // Run the interval's parallel passes on the dedicated reconcile pool so they
+ // do not share the global pool's queue with the assembler's `par_iter`s.
+ let result = crate::service::finalized_state::RECONCILE_POOL
+ .install(|| reconcile_window(&db, &network, &records, value_pool));
+ match result {
+ Ok(new_pool) => value_pool = new_pool,
+ Err(error) => panic!("deferred transparent reconcile worker failed: {error}"),
+ }
+ }
+}
+
+/// Handle to the reconcile worker thread and its bounded job channel.
+///
+/// The channel is capacity-1: handing off the next window blocks the assembler only
+/// while the worker is still busy with the previous one, bounding the in-flight
+/// `Arc` to ~two windows.
+struct ReconcileWorker {
+ sender: Option>,
+ handle: Option>,
+}
+
+impl ReconcileWorker {
+ /// Hand a window prefix to the worker, applying backpressure if it is still busy.
+ /// Empty windows are ignored.
+ fn send(&self, records: Vec) {
+ if records.is_empty() {
+ return;
+ }
+ if let Some(sender) = self.sender.as_ref() {
+ // Env-gated: time how long the bounded send blocks. A large wait means the
+ // worker is behind (the capacity-1 channel is full), i.e. the reconcile
+ // gates the assembler.
+ let debug = std::env::var("ZRB_RECONCILE_DEBUG").is_ok();
+ let last_h = records.last().map(|r| r.height.0).unwrap_or(0);
+ let send_start = std::time::Instant::now();
+ // The worker only stops on channel close, so a send error means it
+ // panicked mid-reconcile; that is fatal (inconsistent value pool / UTXOs).
+ sender
+ .send(ReconcileJob { records })
+ .expect("reconcile worker thread has gone away");
+ if debug {
+ eprintln!(
+ "[recon-send] last_h={last_h} backpressure_ms={:.1}",
+ send_start.elapsed().as_secs_f64() * 1e3,
+ );
+ }
+ }
+ }
+
+ /// Drop the job channel and wait for the worker to finish every queued window, so
+ /// all reconcile writes are durable. Idempotent.
+ fn flush_and_join(&mut self) {
+ drop(self.sender.take());
+ if let Some(handle) = self.handle.take() {
+ handle.join().expect("reconcile worker thread panicked");
+ }
+ }
+}
+
+impl Drop for ReconcileWorker {
+ /// Clean-shutdown safety net: if the worker was not already drained at the
+ /// checkpoint handoff (the common path), close the job channel and join the
+ /// thread so a clean stop finishes any queued reconcile before the state service
+ /// is torn down. Idempotent — a no-op once `flush_and_join` has run. Join errors
+ /// are swallowed here to avoid panicking during unwinding; crash recovery from a
+ /// mid-window stop is out of scope (the node is treated as re-snapshottable).
+ fn drop(&mut self) {
+ drop(self.sender.take());
+ if let Some(handle) = self.handle.take() {
+ let _ = handle.join();
+ }
+ }
+}
+
+/// Process the disk-writer thread's flush acknowledgements: for each acked height,
+/// advance the finalized chain tip, retire the now-durable overlay entries, and run
+/// the configured stop-height check (which exits the process if matched).
+///
+/// With `block_until_empty`, waits for every in-flight block to be acked (used at
+/// the checkpoint→non-finalized handoff and after a stop-height block); otherwise
+/// drains only the acks already available.
+fn drain_finalized_acks(
+ finalized_state: &mut FinalizedState,
+ pipeline: &mut FinalizedPipeline,
+ ack_receiver: &std::sync::mpsc::Receiver,
+ in_flight: &mut VecDeque<(Height, block::Hash, ChainTipBlock)>,
+ chain_tip_sender: &mut ChainTipSender,
+ mut reconcile_worker: Option<&mut ReconcileWorker>,
+ block_until_empty: bool,
+) {
+ while !in_flight.is_empty() {
+ let acked = if block_until_empty {
+ match ack_receiver.recv() {
+ Ok(height) => height,
+ // The writer thread is gone; stop draining.
+ Err(_) => break,
+ }
+ } else {
+ match ack_receiver.try_recv() {
+ Ok(height) => height,
+ Err(_) => break,
+ }
+ };
+
+ let (height, hash, tip_block) = in_flight
+ .pop_front()
+ .expect("each ack corresponds to an in-flight block");
+ debug_assert_eq!(height, acked, "the writer acks blocks in height order");
+
+ chain_tip_sender.set_finalized_tip(tip_block);
+ pipeline.retire_through(height);
+
+ // Deferred transparent reconcile: when deferral is on, reconcile the window
+ // prefix at or below this now-durable height at each reconcile boundary and
+ // at the stop height (so the final partial window is reconciled before the
+ // process exits). The boundary is a fixed block interval when
+ // `defer_reconcile_interval > 0` (mainnet checkpoints are only ~30-40 blocks
+ // apart here, too frequent to amortize the reconcile cost), otherwise each
+ // checkpoint.
+ //
+ // v2: hand the window to the dedicated reconcile worker thread so its disk
+ // reads + value-pool recompute overlap continued assembly. v1 fallback (no
+ // worker): reconcile inline. Fatal on error either way: a failed reconcile
+ // would leave the value pool / UTXO set inconsistent.
+ if finalized_state.db.defer_reconcile_configured() {
+ let interval = finalized_state.db.config().defer_reconcile_interval;
+ let is_boundary = if interval > 0 {
+ height.0 % (interval as u32) == 0
+ } else {
+ finalized_state
+ .db
+ .network()
+ .checkpoint_list()
+ .contains(height)
+ };
+ let at_stop = finalized_state.is_at_stop_height(height);
+ if is_boundary || at_stop {
+ if let Some(worker) = reconcile_worker.as_deref_mut() {
+ worker.send(pipeline.take_reconcile_prefix(height));
+ // The process is about to exit at the stop height; flush the
+ // worker so every queued reconcile is durable first.
+ if at_stop {
+ worker.flush_and_join();
+ }
+ } else {
+ pipeline
+ .reconcile_checkpoint(finalized_state, height)
+ .expect("deferred transparent reconcile failed");
+ }
+ }
+ }
+
+ // The block is now durable, so the stop-height check (which exits the
+ // process) is safe to run here.
+ finalized_state.finalized_stop_at_height_if_configured(
+ height,
+ hash,
+ "commit checkpoint-verified request",
+ );
+ }
+}
+
/// Delay between retryable VCT root-miss commit attempts while the peer cache refills.
const VCT_ROOT_RETRY_WAIT: Duration = Duration::from_millis(500);
@@ -395,6 +670,75 @@ impl WriteBlockWorkerTask {
let mut vct_root_stall: Option<(Height, Instant)> = None;
let mut vct_root_stall_logged = false;
+ // Run-ahead finalized-commit pipeline. Enabled only below the last
+ // checkpoint (the reorg-free region, where the committer is processing the
+ // finalized block channel) and once format upgrades are finished (so the
+ // overlay serves absolute `Insert` address balances, not merge deltas). When
+ // active, the committer assembles each block's batch against the in-memory
+ // overlay and hands it to a dedicated disk-writer thread; the chain tip and
+ // overlay retirement trail the writer's flush acks (ack-after-flush).
+ let pipeline_depth = finalized_state.db.config().finalized_block_pipeline_depth;
+ let pipeline_active = pipeline_depth > 0 && finalized_state.db.finished_format_upgrades();
+
+ let mut pipeline_state: Option = None;
+ let mut pipeline_flush_sender: Option> = None;
+ let mut pipeline_ack_receiver: Option> = None;
+ let mut pipeline_writer_handle: Option> = None;
+ let mut pipeline_in_flight: VecDeque<(Height, block::Hash, ChainTipBlock)> =
+ VecDeque::new();
+
+ if pipeline_active {
+ // The bounded batch channel is the backpressure: at most `pipeline_depth`
+ // assembled-but-not-yet-flushed blocks are in flight, bounding the overlay.
+ let (flush_sender, flush_receiver) =
+ std::sync::mpsc::sync_channel::(pipeline_depth);
+ let (ack_sender, ack_receiver) = std::sync::mpsc::channel::();
+ let writer_db = finalized_state.db.clone();
+ let writer_span = Span::current();
+ let handle = std::thread::Builder::new()
+ .name("zebra-finalized-writer".to_string())
+ .spawn(move || {
+ writer_span
+ .in_scope(|| run_finalized_writer(writer_db, flush_receiver, ack_sender))
+ })
+ .expect("failed to spawn the finalized disk-writer thread");
+
+ pipeline_state = Some(FinalizedPipeline::new());
+ pipeline_flush_sender = Some(flush_sender);
+ pipeline_ack_receiver = Some(ack_receiver);
+ pipeline_writer_handle = Some(handle);
+ }
+
+ // Deferred-transparent reconcile worker (v2). When deferral is on (and not
+ // forced inline), spawn a dedicated thread that resolves/deletes/re-debits
+ // each window off the assembler thread. It owns the running value pool,
+ // seeded once here from disk (the worker is the sole writer of the value pool
+ // in the deferred range, so this stays exclusively owned with no races).
+ let mut reconcile_worker: Option = None;
+ if pipeline_active
+ && finalized_state.db.defer_reconcile_configured()
+ && !finalized_state.db.config().defer_reconcile_inline
+ {
+ // Capacity 1: hand-off blocks the assembler only while the worker is
+ // still busy with the previous window (~two windows of blocks in flight).
+ let (job_sender, job_receiver) = std::sync::mpsc::sync_channel::(1);
+ let worker_db = finalized_state.db.clone();
+ let start_value_pool = finalized_state.db.finalized_value_pool();
+ let worker_span = Span::current();
+ let handle = std::thread::Builder::new()
+ .name("zebra-reconcile-worker".to_string())
+ .spawn(move || {
+ worker_span.in_scope(|| {
+ run_reconcile_worker(worker_db, start_value_pool, job_receiver)
+ })
+ })
+ .expect("failed to spawn the reconcile worker thread");
+ reconcile_worker = Some(ReconcileWorker {
+ sender: Some(job_sender),
+ handle: Some(handle),
+ });
+ }
+
// Write all the finalized blocks sent by the state,
// until the state closes the finalized block channel's sender.
loop {
@@ -427,9 +771,23 @@ impl WriteBlockWorkerTask {
{
Some(block) => block,
None => match finalized_block_write_receiver.try_recv() {
- Ok(block) => block,
+ Ok(block) => {
+ zebra_chain::stage_timing::record(
+ block.0.height.0,
+ "write_worker_received",
+ );
+ block
+ }
Err(TryRecvError::Empty) => {
+ // Starved: no finalized block available. Time the park so we can
+ // tell "assembler is upstream-starved (verify/driver/feed slow)"
+ // apart from "assembler is the bottleneck".
+ let park = std::time::Instant::now();
std::thread::park_timeout(Duration::from_millis(10));
+ metrics::histogram!(
+ "zebra.state.write.assembler_empty_park.duration_seconds"
+ )
+ .record(park.elapsed().as_secs_f64());
continue;
}
Err(TryRecvError::Disconnected) => break,
@@ -449,9 +807,19 @@ impl WriteBlockWorkerTask {
// So if there has been a block commit error,
// we need to drop all the descendants of that block,
// until we receive a block at the required next height.
- let next_valid_height = finalized_state
- .db
- .finalized_tip_height()
+ //
+ // The next block must be the child of the current tip. In the run-ahead
+ // pipeline the assembler runs *ahead* of the durable disk tip (ack-after-
+ // flush), so the next valid height follows the in-memory assembled tip
+ // (`pipeline.tip()`), not the on-disk tip — otherwise a block whose parent
+ // is assembled-but-not-yet-flushed is wrongly dropped as "wrong height".
+ // `pipeline.tip()` is `None` until the first block is assembled (seeded),
+ // so the first block correctly falls back to the durable tip.
+ let next_valid_height = pipeline_state
+ .as_ref()
+ .and_then(|pipeline| pipeline.tip())
+ .map(|(height, _hash)| height)
+ .or_else(|| finalized_state.db.finalized_tip_height())
.map(|height| (height + 1).expect("committed heights are valid"))
.unwrap_or(Height(0));
@@ -481,6 +849,7 @@ impl WriteBlockWorkerTask {
// sizes plus this block's note counts (the sizes after this block).
if finalized_lookahead.is_empty() {
if let Ok(next) = finalized_block_write_receiver.try_recv() {
+ zebra_chain::stage_timing::record(next.0.height.0, "write_worker_received");
finalized_lookahead.push_back(next);
}
}
@@ -497,8 +866,19 @@ impl WriteBlockWorkerTask {
hash = ?ordered_block.0.hash,
"VCT: deferring fast checkpoint commit until successor is buffered"
);
+ // First time this iteration we discover N's successor isn't buffered:
+ // anchors "worker started waiting for N+1" so an offline join can compare
+ // it against when N+1 was actually sent to the worker (state_to_worker).
+ zebra_chain::stage_timing::record(ordered_block.0.height.0, "successor_wait_start");
retry_finalized_block = Some(ordered_block);
+ // VCT one-block look-ahead stall: this fast block can't commit until its
+ // successor is buffered (needed to authenticate its supplied roots). If
+ // this dominates, the 10ms poll granularity (not real work) is the gate.
+ let park = std::time::Instant::now();
std::thread::park_timeout(Duration::from_millis(10));
+ metrics::counter!("zebra.state.write.assembler_successor_defer.count").increment(1);
+ metrics::histogram!("zebra.state.write.assembler_successor_park.duration_seconds")
+ .record(park.elapsed().as_secs_f64());
continue;
}
@@ -548,17 +928,95 @@ impl WriteBlockWorkerTask {
let prev_note_commitment_trees = prev_finalized_note_commitment_trees.take();
let prev_note_commitment_trees_for_retry = prev_note_commitment_trees.clone();
- let next_block_took_vct_path =
- finalized_state.vct_fast_will_apply(ordered_block.0.height);
+ let committed_height = ordered_block.0.height;
+
+ // Handoff drain barrier (consensus-critical). Deferral is only valid below
+ // the last checkpoint (guard 2); above it the semantic verifier validates
+ // each block's spends against the live UTXO set and value pool, so those
+ // must be current. Before the first above-checkpoint block commits, flush
+ // every in-flight block to disk (so the window's spent UTXOs are durable),
+ // drain the pending reconcile window, and refresh the pipeline's threaded
+ // value pool from the now-current disk pool for the inline commits that
+ // follow. The finalized committer normally only sees checkpoint-verified
+ // blocks, so this rarely fires here — the common handoff is the
+ // channel-close drain below — but it guards the boundary itself.
+ if reconcile_worker.is_some()
+ && committed_height > finalized_state.db.network().checkpoint_list().max_height()
+ {
+ if let (Some(pipeline), Some(ack_receiver)) =
+ (pipeline_state.as_mut(), pipeline_ack_receiver.as_ref())
+ {
+ drain_finalized_acks(
+ finalized_state,
+ pipeline,
+ ack_receiver,
+ &mut pipeline_in_flight,
+ chain_tip_sender,
+ reconcile_worker.as_mut(),
+ true,
+ );
+ if let Some(mut worker) = reconcile_worker.take() {
+ worker.send(pipeline.take_reconcile_prefix(committed_height));
+ worker.flush_and_join();
+ }
+ pipeline.reseed_value_pool(&finalized_state.db);
+ }
+ }
- // Try committing the block
- match finalized_state.commit_finalized(
- ordered_block,
- prev_note_commitment_trees,
- note_precompute,
- next_checkpoint,
- ) {
- Ok((finalized, note_commitment_trees)) => {
+ let next_block_took_vct_path = finalized_state.vct_fast_will_apply(committed_height);
+
+ // Commit the block. When the run-ahead pipeline is active, assemble the
+ // block against the overlay and hand its batch to the disk-writer thread
+ // (the chain tip advances later, on the writer's flush ack); otherwise
+ // assemble and flush it synchronously here, advancing the tip inline.
+ let commit_result: Result<
+ NoteCommitmentTrees,
+ (QueuedCheckpointVerified, CommitCheckpointVerifiedError),
+ > = if let Some(pipeline) = pipeline_state.as_mut() {
+ let (checkpoint_verified, rsp_tx) = ordered_block;
+ let tip_block = ChainTipBlock::from(checkpoint_verified.clone());
+ match finalized_state.commit_finalized_pipelined(
+ checkpoint_verified.clone().into(),
+ prev_note_commitment_trees,
+ note_precompute,
+ next_checkpoint,
+ pipeline,
+ ) {
+ Ok(assembled) => {
+ pipeline_in_flight.push_back((assembled.height, assembled.hash, tip_block));
+ // Bounded send: applies backpressure once `pipeline_depth`
+ // blocks are already in flight to the disk-writer thread.
+ let _ = pipeline_flush_sender
+ .as_ref()
+ .expect("flush sender is present while the pipeline is active")
+ .send(PipelineFlush {
+ batch: assembled.batch,
+ commit_trace: assembled.commit_trace,
+ hash: assembled.hash,
+ height: assembled.height,
+ rsp_tx,
+ });
+ Ok(assembled.note_commitment_trees)
+ }
+ Err(error) => Err(((checkpoint_verified, rsp_tx), error)),
+ }
+ } else {
+ finalized_state
+ .commit_finalized(
+ ordered_block,
+ prev_note_commitment_trees,
+ note_precompute,
+ next_checkpoint,
+ )
+ .map(|(finalized, note_commitment_trees)| {
+ let tip_block = ChainTipBlock::from(finalized);
+ chain_tip_sender.set_finalized_tip(tip_block);
+ note_commitment_trees
+ })
+ };
+
+ match commit_result {
+ Ok(note_commitment_trees) => {
// Whether this successful commit consumed header-carried
// tree-aux roots to skip the note-commitment frontier rebuild.
if next_block_took_vct_path {
@@ -581,9 +1039,37 @@ impl WriteBlockWorkerTask {
vct_root_stall_logged = false;
}
- let tip_block = ChainTipBlock::from(finalized);
prev_finalized_note_commitment_trees = Some(note_commitment_trees);
- chain_tip_sender.set_finalized_tip(tip_block);
+
+ // In the pipeline, advance the chain tip and retire the overlay
+ // for any blocks the writer has already flushed. If this block is
+ // the configured stop height, then wait for it to become durable,
+ // so the stop check fires after the flush and no later block is
+ // assembled past it.
+ if let (Some(pipeline), Some(ack_receiver)) =
+ (pipeline_state.as_mut(), pipeline_ack_receiver.as_ref())
+ {
+ drain_finalized_acks(
+ finalized_state,
+ pipeline,
+ ack_receiver,
+ &mut pipeline_in_flight,
+ chain_tip_sender,
+ reconcile_worker.as_mut(),
+ false,
+ );
+ if finalized_state.is_at_stop_height(committed_height) {
+ drain_finalized_acks(
+ finalized_state,
+ pipeline,
+ ack_receiver,
+ &mut pipeline_in_flight,
+ chain_tip_sender,
+ reconcile_worker.as_mut(),
+ true,
+ );
+ }
+ }
}
Err((ordered_block, error)) => {
// Retryable VCT root stalls (an absent/evicted root, or one not yet
@@ -678,6 +1164,53 @@ impl WriteBlockWorkerTask {
}
}
+ // The finalized block channel has closed (the checkpoint→non-finalized
+ // handoff). Drain the run-ahead pipeline to durability before switching to
+ // the non-finalized loop: drop the batch sender so the writer finishes its
+ // queue and exits, advance the chain tip and retire the overlay for every
+ // remaining in-flight block, then join the writer thread.
+ if pipeline_active {
+ drop(pipeline_flush_sender.take());
+ if let (Some(pipeline), Some(ack_receiver)) =
+ (pipeline_state.as_mut(), pipeline_ack_receiver.as_ref())
+ {
+ drain_finalized_acks(
+ finalized_state,
+ pipeline,
+ ack_receiver,
+ &mut pipeline_in_flight,
+ chain_tip_sender,
+ reconcile_worker.as_mut(),
+ true,
+ );
+
+ // Deferred reconcile: the trailing blocks committed past the last
+ // boundary form a partial window that nothing reconciled. This is the
+ // common checkpoint→non-finalized handoff: drain the window (through the
+ // durable tip) so the value pool and UTXO set are current before the
+ // semantic verifier reads them. (The in-loop barrier above guards the
+ // rarer case of an above-checkpoint block arriving on this channel.)
+ if finalized_state.db.defer_reconcile_configured() {
+ let tip_height = finalized_state.db.tip().map(|(height, _)| height);
+ if let Some(tip_height) = tip_height {
+ if let Some(worker) = reconcile_worker.as_mut() {
+ // v2: send the final window, then wait for the worker to
+ // make every queued reconcile durable.
+ worker.send(pipeline.take_reconcile_prefix(tip_height));
+ worker.flush_and_join();
+ } else {
+ pipeline
+ .reconcile_checkpoint(finalized_state, tip_height)
+ .expect("final deferred transparent reconcile failed");
+ }
+ }
+ }
+ }
+ if let Some(handle) = pipeline_writer_handle.take() {
+ let _ = handle.join();
+ }
+ }
+
// Do this check even if the channel got closed before any finalized blocks were sent.
// This can happen if we're past the finalized tip.
if invalid_block_reset_sender.is_closed() {
diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml
index aca6b4647cf..d77338ec796 100644
--- a/zebrad/Cargo.toml
+++ b/zebrad/Cargo.toml
@@ -65,6 +65,11 @@ indexer = ["zebra-state/indexer"]
# Per-block commit-phase timing histograms (zebra.state.write.*), for perf profiling
commit-metrics = ["zebra-state/commit-metrics"]
+# Exposes the production Zakura block-sync apply driver (`drive_block_sync_actions`)
+# from the crate root so the offline `zebra-replay-bench` can drive the real apply
+# pipeline. Not part of any release; no production behavior change.
+internal-bench = ["zebra-network/internal-bench"]
+
# Experimental internal miner support
internal-miner = [
"thread-priority",
diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs
index 98dec3a2af5..f3eb05c553b 100644
--- a/zebrad/src/commands/start.rs
+++ b/zebrad/src/commands/start.rs
@@ -73,7 +73,9 @@
//!
//! Some of the diagnostic features are optional, and need to be enabled at compile-time.
-mod zakura;
+// `pub(crate)` (not private) so the crate root can re-export the block-sync apply
+// driver under the `internal-bench` feature for `zebra-replay-bench`.
+pub(crate) mod zakura;
use std::{net::SocketAddr, path::Path, sync::Arc};
diff --git a/zebrad/src/commands/start/zakura/block_sync_driver.rs b/zebrad/src/commands/start/zakura/block_sync_driver.rs
index 1c8c405725d..0aa2ff69df9 100644
--- a/zebrad/src/commands/start/zakura/block_sync_driver.rs
+++ b/zebrad/src/commands/start/zakura/block_sync_driver.rs
@@ -91,8 +91,16 @@ impl CheckpointFrontierRefresh {
}
}
+/// Drives the Zakura block-sync apply pipeline: consumes `BlockSyncAction`s from the
+/// sequencer, applies submitted bodies through the consensus router into state
+/// (checkpoint-class blocks take the header-authenticated fast path), drains applies
+/// concurrently under the checkpoint/full/combined limits, and reports completions
+/// back to the sequencer via `block_sync`. Runs until `shutdown` resolves.
+///
+/// This is the production sync driver; it is also reused by the offline
+/// `zebra-replay-bench` (via `zebrad::bench_api`) to benchmark the real apply path.
#[allow(clippy::too_many_arguments)]
-pub(crate) async fn drive_block_sync_actions(
+pub async fn drive_block_sync_actions(
mut actions: mpsc::Receiver,
// Retained so the disconnect capability stays wired into the driver, even
// though peer scoring no longer drives disconnects (misbehavior is record-only).
@@ -469,6 +477,14 @@ pub(crate) async fn drive_block_sync_actions(
BlockSyncAction::SubmitBlock { token, block } => {
let class = block_apply_class(block.as_ref(), max_checkpoint_height);
let height = block.coinbase_height();
+ // The pending-apply backlog when this SubmitBlock arrives: a large
+ // backlog means the sequencer is emitting well ahead and the driver's
+ // dispatch is the limiter; ~0 means the sequencer feeds just-in-time.
+ zebra_chain::stage_timing::record_val(
+ height.map_or(0, |h| h.0),
+ "submit_received",
+ pending_applies.len() as u64,
+ );
emit_commit_state(
&trace,
cs_trace::BLOCK_SUBMIT_QUEUED,
@@ -754,6 +770,15 @@ fn drain_pending_block_applies(
}
}
+ // How many applies are concurrently in flight at the moment this one is
+ // dispatched: ~1-2 means the driver is paced one-at-a-time (verify can't run
+ // ahead); a large depth means the driver is sprinting and the gate is elsewhere.
+ zebra_chain::stage_timing::record_val(
+ pending.block.coinbase_height().map_or(0, |h| h.0),
+ "apply_dispatch",
+ (*checkpoint_in_flight + *full_in_flight) as u64,
+ );
+
let class = pending.class;
in_flight_applies.push(
apply_block_sync_body(
@@ -934,6 +959,7 @@ where
};
};
+ zebra_chain::stage_timing::record(height.0, "driver_commit_start");
emit_commit_state(&trace, cs_trace::COMMIT_START, "block_sync_driver", |row| {
insert_cs_u64(row, cs_trace::APPLY_TOKEN, token);
insert_cs_str(row, cs_trace::APPLY_CLASS, block_apply_class_label(class));
@@ -946,19 +972,53 @@ where
// normal mode the frontier comes from re-reading committed state below.
let (result, probe_frontier) = match throughput_probe.as_ref() {
Some(probe) => probe.apply_block(block.as_ref()),
- None => (
- commit_block_sync_body_with_stall_trace(
- block_verifier.clone(),
- block,
- class,
- &trace,
- token,
- height,
- expected_hash,
- )
- .await,
- None,
- ),
+ None => {
+ // For a checkpoint-class body, obtain the checkpoint-authenticated expected hash. This
+ // also enforces the airtight-ordering invariant: the body must not reach the verifier
+ // before header sync has authenticated its height (the needed-bodies gate ensures it).
+ let checkpoint_auth_hash = if class == BlockApplyClass::Checkpoint {
+ query_authenticated_checkpoint_hash(read_state.clone(), height).await
+ } else {
+ None
+ };
+ zebra_chain::stage_timing::record(height.0, "auth_hash_done");
+
+ if class == BlockApplyClass::Checkpoint && checkpoint_auth_hash.is_none() {
+ // Hard sync-invariant violation: there is no fallback into range accumulation. We
+ // refuse the commit and surface it; airtight ordering means this should not happen.
+ warn!(
+ ?height,
+ ?expected_hash,
+ "Zakura checkpoint body is not header-authenticated; refusing fast commit"
+ );
+ emit_commit_state(
+ &trace,
+ "checkpoint_auth_invariant",
+ "block_sync_driver",
+ |row| {
+ insert_cs_u64(row, cs_trace::APPLY_TOKEN, token);
+ insert_cs_height(row, cs_trace::HEIGHT, height);
+ insert_cs_hash(row, cs_trace::HASH, expected_hash);
+ },
+ );
+ (BlockApplyResult::Rejected, None)
+ } else {
+ (
+ commit_block_sync_body_with_stall_trace(
+ block_verifier.clone(),
+ block,
+ class,
+ &trace,
+ token,
+ height,
+ expected_hash,
+ checkpoint_auth_hash,
+ )
+ .await,
+ None,
+ )
+ }
+ }
};
emit_commit_state(
&trace,
@@ -971,6 +1031,9 @@ where
insert_cs_hash(row, cs_trace::HASH, expected_hash);
insert_cs_str(row, cs_trace::RESULT, block_apply_result_label(result));
insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started));
+ if class == BlockApplyClass::Checkpoint && result == BlockApplyResult::Committed {
+ insert_cs_str(row, "checkpoint_commit_path", "zakura_authenticated");
+ }
},
);
emit_commit_state(
@@ -1083,6 +1146,7 @@ async fn commit_block_sync_body_with_stall_trace(
token: BlockApplyToken,
height: block::Height,
expected_hash: block::Hash,
+ checkpoint_auth_hash: Option,
) -> BlockApplyResult
where
BlockVerifier:
@@ -1090,9 +1154,16 @@ where
BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static,
BlockVerifier::Future: Send + 'static,
{
- let commit = block_verifier
- .clone()
- .oneshot(zebra_consensus::Request::Commit(block));
+ // Checkpoint-class bodies use the header-authenticated fast path (per-block validate + release,
+ // no range accumulation). Full-class bodies take the normal semantic commit.
+ let request = match checkpoint_auth_hash {
+ Some(expected_hash) => zebra_consensus::Request::CommitCheckpointAuthenticated {
+ block,
+ expected_hash,
+ },
+ None => zebra_consensus::Request::Commit(block),
+ };
+ let commit = block_verifier.clone().oneshot(request);
match class {
BlockApplyClass::Checkpoint => {
@@ -1189,6 +1260,41 @@ fn block_commit_timed_out(
BlockApplyResult::TimedOut
}
+/// Reads the checkpoint-authenticated hash for `height`, if header sync has authenticated it.
+///
+/// Returns the provenance token required by the Zakura checkpoint fast-commit path, or `None` if
+/// the height is not (yet) checkpoint-authenticated or the read fails.
+async fn query_authenticated_checkpoint_hash(
+ read_state: ReadState,
+ height: block::Height,
+) -> Option
+where
+ ReadState: Service<
+ zebra_state::ReadRequest,
+ Response = zebra_state::ReadResponse,
+ Error = zebra_state::BoxError,
+ > + Clone
+ + Send
+ + 'static,
+ ReadState::Future: Send + 'static,
+{
+ match read_state
+ .oneshot(zebra_state::ReadRequest::AuthenticatedCheckpointHash { height })
+ .await
+ {
+ Ok(zebra_state::ReadResponse::AuthenticatedCheckpointHash(token)) => token,
+ Ok(_) => None,
+ Err(error) => {
+ warn!(
+ ?height,
+ ?error,
+ "failed to read authenticated checkpoint hash"
+ );
+ None
+ }
+ }
+}
+
async fn refresh_block_sync_frontiers_for_checkpoint_window(
read_state: ReadState,
latest_chain_tip: impl ChainTip + Clone + Send + Sync + 'static,
diff --git a/zebrad/src/commands/start/zakura/mod.rs b/zebrad/src/commands/start/zakura/mod.rs
index bff8f791fd8..82a5028c592 100644
--- a/zebrad/src/commands/start/zakura/mod.rs
+++ b/zebrad/src/commands/start/zakura/mod.rs
@@ -12,7 +12,7 @@ pub(crate) mod frontier;
pub(crate) mod header_sync_driver;
pub(crate) mod throughput_probe;
-pub(crate) use block_sync_driver::drive_block_sync_actions;
+pub use block_sync_driver::drive_block_sync_actions;
#[cfg(test)]
pub(crate) use block_sync_driver::{
apply_block_sync_body, block_apply_class, block_sync_missing_body_window,
diff --git a/zebrad/src/commands/start/zakura/throughput_probe.rs b/zebrad/src/commands/start/zakura/throughput_probe.rs
index 6ddf5325e5e..989828711c8 100644
--- a/zebrad/src/commands/start/zakura/throughput_probe.rs
+++ b/zebrad/src/commands/start/zakura/throughput_probe.rs
@@ -24,7 +24,7 @@ use zebra_network::zakura::{BlockApplyResult, BlockSyncFrontiers};
/// expected block at the next height; the probe only confirms contiguity and
/// takes the body's own hash as the new tip.
#[derive(Clone, Debug)]
-pub(crate) struct BlocksyncThroughputProbe {
+pub struct BlocksyncThroughputProbe {
inner: Arc>,
}
diff --git a/zebrad/src/lib.rs b/zebrad/src/lib.rs
index fe7efa59bcf..e1af8d4067b 100644
--- a/zebrad/src/lib.rs
+++ b/zebrad/src/lib.rs
@@ -150,5 +150,14 @@ pub mod components;
pub mod config;
pub mod prelude;
+/// Narrow internal API for the offline `zebra-replay-bench`: the production Zakura
+/// block-sync apply driver, so the bench can drive the real apply/commit/report loop
+/// instead of a hand-rolled one. Feature-gated; not part of any release.
+#[cfg(feature = "internal-bench")]
+pub mod bench_api {
+ pub use crate::commands::start::zakura::drive_block_sync_actions;
+ pub use crate::commands::start::zakura::throughput_probe::BlocksyncThroughputProbe;
+}
+
#[cfg(feature = "sentry")]
pub(crate) mod sentry;