fix(manual-qa): async-dispatch metrics fixes + full tokenomics parity in bundle

bundles/manual-qa/hooks/README.md

@@ -30,7 +30,7 @@ This merges 5 hook events (`SessionStart`, `PreToolUse`, `PostToolUse`,
 `SubagentStop`, `SessionEnd`) into `.claude/settings.json` under groups
 tagged `"_bundle": "manual-qa"` (merge-not-clobber — your other hooks and
 other bundles' hooks are left untouched; `.claude/settings.json.bak` is
-written first), and copies 10 files into `.claude/hooks/manual-qa/`.
+written first), and copies 12 files into `.claude/hooks/manual-qa/`.
 Nothing in `agents/` or `skills/` is touched — this is additive only.
 
 ## What to expect after a run

bundles/manual-qa/hooks/scripts/benchmark-preflight

@@ -2,14 +2,49 @@
 # PreToolUse[Agent] hook — fires before every sub-agent call.
 # On the first call of a session: captures first_dispatch_at timestamp,
 # resolves a ccusage pre-snapshot, then creates the benchmark state file.
-# Subsequent calls are no-ops (state file already exists).
+# Subsequent calls are no-ops for THAT part (state file already exists) but
+# still refresh LAST_DISPATCH_FILE below — see its own comment for why.
 #
 # Files are scoped by session_id (2026-07-22 fix) — see benchmark-session-start
 # for why (prevents concurrent-session/test-fixture collisions).
+#
+# RETRY-FOR-SCOPING RACE (ported 2026-07-31 from qa-project, round 4):
+# benchmark-session-start now retry-waits for `ccusage session --json` to
+# actually contain this session's own entry before writing SESSION_PRE_FILE
+# — see ccusage-wait.sh for the timing evidence and why session-start's own
+# retry can still, legitimately, sometimes come back empty-handed (its retry
+# budget is bounded; the gap it's racing against isn't fully understood yet
+# — see that file).
+#
+# That means SESSION_PRE_FILE existing is NOT the same thing as it being
+# actually SCOPED — it may be session-start's own unscoped give-up snapshot.
+# Blindly `cp`-ing it forward (the old behavior) would propagate that failure
+# into this run's real metrics even though, empirically, a real dispatch
+# fires WAY later than session start (48s in the one real case measured so
+# far) — by which point the session has near-certainly already appeared in
+# ccusage (its own first usage entry showed up in ~2-3s in every real sample
+# collected). So: check whether SESSION_PRE_FILE is actually scoped before
+# trusting it; if not, this hook gets its OWN, much-later-timed shot via the
+# same retry helper rather than inheriting session-start's early failure.
+#
+# Both this hook and session-start are registered `async: true`, so a fast
+# orchestrator's first dispatch can also fire before session-start's own
+# retry loop has finished AT ALL (SESSION_PRE_FILE doesn't exist yet) — same
+# fix applies there too (see the `elif` below). Worst case this means two
+# independent `ccusage session --json` retry loops run briefly in parallel —
+# wasteful in CLI calls, not incorrect (each just converges to whatever it
+# finds); simpler than adding a separate "wait for the OTHER hook's file
+# instead" coordination layer for a race this narrow.
 set -euo pipefail
 
 PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$PWD}"
 STATE_DIR="${PROJECT_DIR}/.claude"
+# ccusage-wait.sh lives next to this script (bundle-installed), not at
+# <project>/scripts/ — resolve self-relatively, same pattern benchmark-stop
+# already uses for build-run-metrics.mjs.
+HOOK_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=ccusage-wait.sh
+. "${HOOK_DIR}/ccusage-wait.sh"
 
 payload="$(cat 2>/dev/null || true)"
 sid="$(printf '%s' "$payload" | node -e "
@@ -25,8 +60,21 @@ STATE_FILE="${STATE_DIR}/benchmark-state${SID_SUFFIX}.json"
 TRACE_FILE="${STATE_DIR}/benchmark-tc-trace${SID_SUFFIX}.jsonl"
 PRE_FILE="${STATE_DIR}/benchmark-pre${SID_SUFFIX}.json"
 SESSION_PRE_FILE="${STATE_DIR}/benchmark-session-pre${SID_SUFFIX}.json"
+LAST_DISPATCH_FILE="${STATE_DIR}/benchmark-last-dispatch-at${SID_SUFFIX}.txt"
+CCUSAGE_WAIT_DEBUG_LOG="${STATE_DIR}/benchmark-ccusage-wait-debug.log"
+
+# Refresh on EVERY dispatch (ported 2026-07-31 from qa-project — see
+# benchmark-stop's own "PREMATURE-FIRING GUARD" comment for the full
+# incident writeup). Cheap unconditional write, independent of the
+# first-dispatch-only STATE_FILE init below: benchmark-stop uses "how long
+# since the most recent dispatch" to tell a genuine test-reporter completion
+# (always 40s+ in every real sample collected) apart from a premature
+# same-tool-call-cycle firing (always under 12s in every real sample
+# collected) — see that script.
+mkdir -p "$STATE_DIR"
+date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_DISPATCH_FILE" 2>/dev/null || true
 
-# Only act on the first dispatch of this session.
+# Only act on the first dispatch of this session for everything below.
 [ -f "$STATE_FILE" ] && exit 0
 
 mkdir -p "$STATE_DIR"
@@ -40,9 +88,33 @@ first_dispatch_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 # from this dispatch onward, matching the old behavior) if session-start's
 # snapshot isn't there — e.g. that hook wasn't wired, or this project was
 # updated mid-session and only got the new benchmark-preflight.
-if [ -f "$SESSION_PRE_FILE" ]; then
+session_pre_is_scoped=0
+if [ -f "$SESSION_PRE_FILE" ] && [ -n "$sid" ]; then
+  if node -e "
+    try{
+      const p=JSON.parse(require('fs').readFileSync('${SESSION_PRE_FILE}','utf8'));
+      process.exit((p.session||[]).some(s=>s.period==='${sid}')?0:1);
+    }catch{process.exit(1);}
+  " 2>/dev/null; then
+    session_pre_is_scoped=1
+  fi
+fi
+
+if [ "$session_pre_is_scoped" = 1 ]; then
+  cp "$SESSION_PRE_FILE" "$PRE_FILE"
+elif [ -n "$sid" ]; then
+  # Either session-start hasn't written SESSION_PRE_FILE yet (race), or it
+  # did but gave up unscoped (see header note) — this hook's own, later
+  # timing gives it a much better real shot; retry fresh rather than
+  # inheriting session-start's outcome either way.
+  wait_for_scoped_ccusage "$sid" "$PRE_FILE" "$CCUSAGE_WAIT_DEBUG_LOG" "preflight-fallback" || true
+elif [ -f "$SESSION_PRE_FILE" ]; then
+  # No session_id to retry-match against, but session-start's snapshot
+  # (unscoped or not) is still better than nothing.
   cp "$SESSION_PRE_FILE" "$PRE_FILE"
 else
+  # No session_id and no session-start snapshot at all -- nothing to
+  # retry-match against, one immediate snapshot as last resort.
   ccusage session --json > "$PRE_FILE" 2>/dev/null \
     || printf '{}' > "$PRE_FILE"
 fi

bundles/manual-qa/hooks/scripts/benchmark-session-start

@@ -18,10 +18,31 @@
 # correctly re-baselines too, matching Claude Code's own semantics (today,
 # with fixed filenames, /clear did NOT reset the baseline — a deliberate
 # behavior improvement, confirmed with Olha).
+#
+# RETRY-FOR-SCOPING (ported 2026-07-31 from qa-project, round 4): this hook
+# fires at the very first moment of a brand-new session, before Claude
+# Code's own usage logs (which ccusage reads from) have anything for this
+# session id yet — a single un-retried `ccusage session --json` call here
+# almost always misses its own session's `period` entry, silently degrading
+# every downstream token/cost figure to "full_session_unscoped" (summed
+# across every session ever recorded on the machine — confirmed on a real
+# qa-project run, and confirmed AGAIN on the very next real run after an
+# initial attempt-count-based retry fix — see ccusage-wait.sh for the full
+# incident writeup: what actually failed wasn't the timing estimate, still
+# unresolved is whether `sleep` reliably blocks in this hook's real
+# execution context). See ccusage-wait.sh (sourced below) for the retry
+# mechanism itself and why benchmark-preflight now does more than just
+# reuse SESSION_PRE_FILE when it exists.
 set -euo pipefail
 
 PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$PWD}"
 STATE_DIR="${PROJECT_DIR}/.claude"
+# ccusage-wait.sh lives next to this script (bundle-installed), not at
+# <project>/scripts/ — resolve self-relatively, same pattern benchmark-stop
+# already uses for build-run-metrics.mjs.
+HOOK_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=ccusage-wait.sh
+. "${HOOK_DIR}/ccusage-wait.sh"
 
 payload="$(cat 2>/dev/null || true)"
 sid="$(printf '%s' "$payload" | node -e "
@@ -35,12 +56,24 @@ SID_SUFFIX="${sid:+-$sid}"       # empty sid -> empty suffix -> today's exact fi
 
 TS_FILE="${STATE_DIR}/benchmark-session-started-at${SID_SUFFIX}.txt"
 SESSION_PRE_FILE="${STATE_DIR}/benchmark-session-pre${SID_SUFFIX}.json"
+CCUSAGE_WAIT_DEBUG_LOG="${STATE_DIR}/benchmark-ccusage-wait-debug.log"
 
 mkdir -p "$STATE_DIR"
 if [ ! -f "$TS_FILE" ]; then
   date -u +%Y-%m-%dT%H:%M:%SZ > "$TS_FILE" 2>/dev/null || true
-  ccusage session --json > "$SESSION_PRE_FILE" 2>/dev/null \
-    || printf '{}' > "$SESSION_PRE_FILE"
+  if [ -n "$sid" ]; then
+    # async: true in hooks.json -- this hook never blocks the user's
+    # interactive session, so it's safe to spend up to the full budget here
+    # (currently 60s of real elapsed time, not attempt-count-based -- see
+    # ccusage-wait.sh header for why that changed).
+    wait_for_scoped_ccusage "$sid" "$SESSION_PRE_FILE" "$CCUSAGE_WAIT_DEBUG_LOG" "session-start" || true
+  else
+    # No session_id on this payload at all (older Claude Code / unusual
+    # invocation) -- nothing to retry-match against, take one snapshot as
+    # before rather than burning the full retry window for nothing.
+    ccusage session --json > "$SESSION_PRE_FILE" 2>/dev/null \
+      || printf '{}' > "$SESSION_PRE_FILE"
+  fi
 
   # Opportunistic cleanup: remove session-scoped ephemeral files older than 7
   # days — left behind by sessions that crashed/were force-killed before ever