dgx-online-serving.sh

#!/usr/bin/env bash
# Reproducible SERVE-GATE-ONLINE campaign driver.
#
# The accepted contract is .agents/specs/cuda-online-serving-gate.md. Timed
# requests are issued only by pinned vLLM `bench serve`; this script owns server
# lifecycle, interleaving, the one-model/one-lock boundary, memory return, and
# artifact capture.  --dry-run and --prepare-corpus never acquire /tmp/gpu;
# --trace-only performs the model gate plus the paired trace without the grid.
set -euo pipefail

usage() {
  cat >&2 <<'EOF'
usage:
  dgx-online-serving.sh --dry-run [--claim-root DIR] [--client PATH] [--vllm-cpp-sha SHA]
  dgx-online-serving.sh --prepare-corpus --model 27|27n|35 --source-corpus DIR --evidence DIR
  dgx-online-serving.sh --trace-only --model 27 --snapshot DIR --source-corpus DIR \
    --evidence DIR --build-dir DIR --configure-log FILE [--client PATH] [--port N] \
    [--trace-concurrency 2|16] [--gdn-ba-mode both] [--gdn-packed-mode both]
  dgx-online-serving.sh --execute --model 27|27n|35 --snapshot DIR --source-corpus DIR \
    --evidence DIR --build-dir DIR --configure-log FILE [--client PATH] [--port N]
  dgx-online-serving.sh --startup-only --model 27|27n|35|q3mxfp4 --snapshot DIR \
    --source-corpus DIR --evidence DIR --build-dir DIR --configure-log FILE \
    [--client PATH] [--port N]
EOF
}

mode=""
model=""
snapshot=""
source_corpus=""
claim_root="${HOME}/work/vllm.cpp-online-gate"
evidence=""
build_dir=""
configure_log=""
# The oracle venv is HOST state, not repo state, so this stays a symlink and
# `--client` still overrides it. What changed (#520) is that the symlink can no
# longer decide the run silently: `record-oracle` below asserts the resolved
# oracle's COMMIT against the parity pin, so pointing this at the preserved
# v0.25.0 rollback now ABORTS the gate instead of quietly relabelling it (#375).
client="${HOME}/venvs/vllm-oracle/bin/vllm"
vllm_cpp_sha=""
port=8001
num_blocks=4736
max_num_seqs=32
max_num_batched_tokens=""
trace_concurrency=16
gdn_ba_mode=""
gdn_packed_mode=""
# Readiness cadence. 0.2 s resolves a startup of a few tens of seconds to well
# under 1%; the previous 5 s cadence was itself ~12% of the measured quantity,
# which is why launch->ready was never a reportable number. The 1800 s budget is
# the old 360 x 5 s timeout, preserved exactly.
ready_poll_interval=0.2
ready_timeout_seconds=1800
# Hard ceiling on the SM-clock sampler's own lifetime (#543). See run_leg.
clock_sampler_max_seconds=7200

while (($#)); do
  case "$1" in
    --dry-run|--prepare-corpus|--trace-only|--execute|--startup-only)
      [[ -z ${mode} ]] || { echo "choose exactly one mode" >&2; exit 2; }
      mode=${1#--}
      shift
      ;;
    --model) model=${2:?}; shift 2 ;;
    --snapshot) snapshot=${2:?}; shift 2 ;;
    --source-corpus) source_corpus=${2:?}; shift 2 ;;
    --claim-root) claim_root=${2:?}; shift 2 ;;
    --evidence) evidence=${2:?}; shift 2 ;;
    --build-dir) build_dir=${2:?}; shift 2 ;;
    --configure-log) configure_log=${2:?}; shift 2 ;;
    --client) client=${2:?}; shift 2 ;;
    --vllm-cpp-sha) vllm_cpp_sha=${2:?}; shift 2 ;;
    --port) port=${2:?}; shift 2 ;;
    --num-blocks) num_blocks=${2:?}; shift 2 ;;
    --trace-concurrency) trace_concurrency=${2:?}; shift 2 ;;
    --gdn-ba-mode) gdn_ba_mode=${2:?}; shift 2 ;;
    --gdn-packed-mode) gdn_packed_mode=${2:?}; shift 2 ;;
    -h|--help) usage; exit 0 ;;
    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
  esac
done

[[ -n ${mode} ]] || { usage; exit 2; }
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
export PYTHONPATH="${repo_root}${PYTHONPATH:+:${PYTHONPATH}}"

if [[ -z ${vllm_cpp_sha} ]]; then
  vllm_cpp_sha=$(git -C "${repo_root}" rev-parse HEAD)
fi
if [[ -n ${gdn_ba_mode} &&
      ( ${mode} != trace-only || ${gdn_ba_mode} != both ) ]]; then
  echo "--gdn-ba-mode currently accepts only 'both' with --trace-only" >&2
  exit 2
fi
if [[ -n ${gdn_packed_mode} &&
      ( ${mode} != trace-only || ${gdn_packed_mode} != both ) ]]; then
  echo "--gdn-packed-mode currently accepts only 'both' with --trace-only" >&2
  exit 2
fi
if [[ -n ${gdn_ba_mode} && -n ${gdn_packed_mode} ]]; then
  echo "--gdn-ba-mode and --gdn-packed-mode are mutually exclusive" >&2
  exit 2
fi

if [[ ${mode} == dry-run ]]; then
  evidence_root="${claim_root}/evidence/${vllm_cpp_sha}"
  python3 "${repo_root}/tools/bench/online_gate.py" plan \
    --claim-root "${claim_root}" \
    --vllm-cpp-sha "${vllm_cpp_sha}" \
    --client "${client}" \
    --output "${evidence_root}/manifest.json"
  echo "dry-run manifest: ${evidence_root}/manifest.json" >&2
  exit 0
fi

[[ ${model} == 27 || ${model} == 27n || ${model} == 35 || ${model} == q3mxfp4 ]] || {
  echo "--model must be 27, 27n, 35 or q3mxfp4" >&2; exit 2; }
# 35B MoE prefills a wider chunk; the 27B NVFP4 dense arms (27 = unsloth,
# 27n = nvidia/ModelOpt) and the q3mxfp4 MXFP4 dense 8B all use the dense 2048
# batched-token gate value.
if [[ ${model} == 35 ]]; then
  max_num_batched_tokens=8192
else
  max_num_batched_tokens=2048
fi
[[ -n ${evidence} ]] || { echo "--evidence is required" >&2; exit 2; }
[[ -n ${source_corpus} ]] || { echo "--source-corpus is required" >&2; exit 2; }
expected_source_corpus="${evidence}/corpus/${model}"
[[ $(realpath -m "${source_corpus}") == "$(realpath -m "${expected_source_corpus}")" ]] || {
  echo "--source-corpus must be ${expected_source_corpus} so its hashes stay in evidence" >&2
  exit 2
}

prepare_corpus() {
  local output="${evidence}/corpus/${model}/vllm"
  if [[ -f ${output}/manifest.json ]]; then
    return
  fi
  python3 "${repo_root}/tools/bench/online_gate.py" prepare-corpus \
    --source "${source_corpus}" \
    --output "${output}" \
    --model-key "${model}"
}

if [[ ${mode} == prepare-corpus ]]; then
  prepare_corpus
  exit 0
fi

[[ ${mode} == execute || ${mode} == trace-only || ${mode} == startup-only ]] || { usage; exit 2; }
# The H1d-era unconditional --execute hold was LIFTED 2026-07-15: its stated
# precondition (H1d/G4 complete; separate production and trace builds) was met
# on 2026-07-13, and the W1D3 closure (b80663a) authorized the fresh
# binding/exact-grid rerun. Timed grids still require a production
# (profile-control-OFF) build via the recorded configure contract.
# The refusal below is about WHICH CHECKPOINT, not which architecture: 27n is a
# Qwen3.6-27B dense graph too. TRACE_PRIMARY_GRAPH_CONTRACTS holds node and
# kernel-family counts captured on the unsloth @890bdef7 27B alone, so no other
# key has a contract to validate a trace against.
if [[ ${mode} == trace-only && ${model} != 27 ]]; then
  echo "H1d trace-only control is defined only for the unsloth Qwen3.6-27B NVFP4 checkpoint (--model 27), the only key with captured graph contracts; 35B performance remains held" >&2
  exit 2
fi
if [[ ${mode} == trace-only && ${trace_concurrency} != 2 && ${trace_concurrency} != 16 ]]; then
  echo "--trace-concurrency must be 2 or 16" >&2
  exit 2
fi
if [[ -n ${gdn_ba_mode} &&
      ( ${model} != 27 || ${trace_concurrency} != 2 ) ]]; then
  echo "--gdn-ba-mode both is defined only for the 27B c2 trace-only gate" >&2
  exit 2
fi
if [[ -n ${gdn_packed_mode} &&
      ( ${model} != 27 || ${trace_concurrency} != 2 ) ]]; then
  echo "--gdn-packed-mode both is defined only for the 27B c2 trace-only gate" >&2
  exit 2
fi
[[ -n ${snapshot} && -d ${snapshot} ]] || { echo "--snapshot directory is required" >&2; exit 2; }
[[ -n ${build_dir} && -f ${build_dir}/CMakeCache.txt ]] || {
  echo "--build-dir must name a configured CMake build tree" >&2
  exit 2
}
[[ -n ${configure_log} && -s ${configure_log} ]] || {
  echo "--configure-log must name the non-empty log from this build configuration" >&2
  exit 2
}
[[ -x ${client} ]] || { echo "pinned vLLM client is not executable: ${client}" >&2; exit 2; }
[[ $(git -C "${repo_root}" rev-parse HEAD) == "${vllm_cpp_sha}" ]] || {
  echo "worktree HEAD does not match --vllm-cpp-sha" >&2
  exit 2
}
[[ -z $(git -C "${repo_root}" status --porcelain) ]] || {
  echo "vllm.cpp source tree is not completely clean" >&2
  exit 2
}
python3 "${repo_root}/tools/bench/online_gate.py" validate-plan \
  "${evidence}/manifest.json" \
  --vllm-cpp-sha "${vllm_cpp_sha}" >/dev/null
if [[ ${mode} == startup-only ]]; then
  # No timed request is issued, so no corpus is built; the directory still has
  # to exist because the cache-drop inventory is a fixed four-root artifact.
  mkdir -p "${source_corpus}"
else
  prepare_corpus
fi

# The execution manifest validates this environment before the GPU lock.  Run
# every model-bearing command from a fixed allowlist instead of inheriting an
# operator shell: hidden VT_*/VLLM_CPP_* controls must not alter the graph.
benchmark_system_path=/usr/local/cuda-13.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
benchmark_common_env=(
  "HOME=${HOME}"
  "PYTHONPATH=${repo_root}"
  "LANG=C.UTF-8"
  "LC_ALL=C.UTF-8"
  "TMPDIR=/tmp"
  "TZ=UTC"
)
benchmark_clean_prefix=(/usr/bin/env -i "${benchmark_common_env[@]}")
benchmark_clean_env=("${benchmark_clean_prefix[@]}" "PATH=${benchmark_system_path}")
h1d_plan_env=(
  "VT_FP4_AUTOTUNE=1"
  "VT_FP4_AUTOTUNE_CACHE_PATH=${evidence}/native-plan-must-not-exist.json"
  "VT_FP4_AUTOTUNE_CACHE_READONLY=1"
  "VT_FP4_AUTOTUNE_DELAY_US=5000"
  "VT_FP4_FLASHINFER_CACHE_PATH=${repo_root}/tests/fixtures/nvfp4_flashinfer_v025_gb10/autotune_configs.json"
  "VT_FP4_FULL_TACTICS=1"
  "VT_FP4_PERSISTENT_CACHE=1"
  "VT_FP4_PLAN_CACHE=1"
  "VT_FP4_PRE_SERVE_WARMUP=1"
)
flashinfer_plan_fixture="${repo_root}/tests/fixtures/nvfp4_flashinfer_v025_gb10/autotune_configs.json"
native_plan_target="${evidence}/native-plan-must-not-exist.json"
[[ -f ${flashinfer_plan_fixture} ]] || {
  echo "frozen FlashInfer plan fixture is absent" >&2
  exit 2
}
[[ ! -e ${native_plan_target} ]] || {
  echo "native plan target must be absent before execution" >&2
  exit 2
}

execution_dir="${evidence}/execution"
mkdir -p "${execution_dir}"
# Timed grids record the production manifest the grid summary reads
# (execution/<model>.json); the H1d trace campaign records the instrumented
# diagnostic manifest (execution/<model>-trace.json) linked from its status.
if [[ ${mode} == trace-only ]]; then
  execution_manifest="${execution_dir}/${model}-trace.json"
else
  execution_manifest="${execution_dir}/${model}.json"
fi
if [[ ${model} == 27 || ${model} == 27n ]]; then
  # Both dense 27B arms precondition on the same paged-engine suite, whose
  # committed goldens are the unsloth @890bdef7 checkpoint's -- so for 27n this
  # is BUILD SANITY, not a golden for @0893e160, and a 27n correctness claim
  # additionally owes a greedy continuation against the pinned oracle there.
  # This comment is NOT the record: record-model-gate writes golden_revision,
  # model_revision and golden_covers_benched_checkpoint into
  # preflight/model-gate/<key>.json and the summary revalidates them.
  test_name=test_qwen27_paged_engine
  gate_target=${test_name}
elif [[ ${model} == 35 ]]; then
  test_name=test_qwen36_paged_engine
  gate_target=${test_name}
else
  # q3mxfp4 (MXFP4 W4A16 keep-quant dense 8B) has no committed npy near-tie
  # paged-engine golden; its correctness precondition is the #44 MXFP4 e2e smoke
  # battery (vllm-cli greedy vs docs/bench-evidence/mxfp4-qwen/golden_marlin_w4a16.json),
  # so the model gate builds + runs vllm-cli instead of a ctest binary.
  test_name=mxfp4_smoke_battery
  gate_target=vllm-cli
fi
cmake_home=$(sed -n 's/^CMAKE_HOME_DIRECTORY:INTERNAL=//p' "${build_dir}/CMakeCache.txt")
[[ -n ${cmake_home} && $(realpath -e "${cmake_home}") == "$(realpath -e "${repo_root}")" ]] || {
  echo "build CMAKE_HOME_DIRECTORY does not match this worktree" >&2
  exit 2
}
vllm_python="$(dirname "${client}")/python"
[[ -x ${vllm_python} ]] || { echo "vLLM oracle Python is absent" >&2; exit 2; }
oracle_manifest="${execution_dir}/${model}-oracle.json"
"${benchmark_clean_env[@]}" "${vllm_python}" \
  "${repo_root}/tools/bench/online_gate.py" record-oracle \
  --output "${oracle_manifest}" \
  --client "${client}"

build_command="${execution_dir}/${model}-build-command.txt"
build_log="${execution_dir}/${model}-build.log"
[[ ! -e ${build_command} && ! -e ${build_log} ]] || {
  echo "refusing to overwrite build provenance for ${model}" >&2
  exit 1
}
build_jobs=$(nproc)
# --startup-only runs no model gate (it compares no token), so the gate binary
# is not built: on a full box that target is pure disk and wall-clock cost.
build_targets=(server)
if [[ ${mode} != startup-only ]]; then
  build_targets+=("${gate_target}")
fi
build_cmd=(
  cmake --build "${build_dir}"
  --target "${build_targets[@]}"
  --parallel "${build_jobs}"
)
printf '%q ' "${build_cmd[@]}" >"${build_command}"
printf '\n' >>"${build_command}"
if ! "${build_cmd[@]}" >"${build_log}" 2>&1; then
  cat "${build_log}" >&2
  exit 1
fi
# The `server` target's OUTPUT_NAME became `vllm-server` when the release
# packaging landed (examples/CMakeLists.txt, W6), so a current build tree has no
# `examples/server`. Resolve the new name first and keep the old one as a
# fallback so an evidence tree built before that rename still replays.
server_bin="${build_dir}/examples/vllm-server"
[[ -x ${server_bin} ]] || server_bin="${build_dir}/examples/server"
[[ -x ${server_bin} ]] || {
  echo "provenance-recorded build did not produce examples/vllm-server (nor the pre-rename examples/server)" >&2
  exit 1
}
[[ ${model} != q3mxfp4 || -x ${build_dir}/examples/vllm-cli ]] || {
  echo "provenance-recorded build did not produce examples/vllm-cli" >&2
  exit 1
}
# Timed grids record the production (profile-control-OFF) build; the H1d
# trace campaign records the instrumented ON build. The grid summary
# requires profile_control=false for timed evidence.
profile_control_flag=$([[ ${mode} == trace-only ]] && echo on || echo off)
"${benchmark_clean_env[@]}" "${h1d_plan_env[@]}" \
  python3 "${repo_root}/tools/bench/online_gate.py" record-execution \
  --output "${execution_manifest}" \
  --model-key "${model}" \
  --vllm-cpp-sha "${vllm_cpp_sha}" \
  --build-dir "${build_dir}" \
  --client "${client}" \
  --snapshot "${snapshot}" \
  --configure-log "${configure_log}" \
  --build-command "${build_command}" \
  --build-log "${build_log}" \
  --oracle-manifest "${oracle_manifest}" \
  --port "${port}" \
  --num-blocks "${num_blocks}" \
  --max-num-seqs "${max_num_seqs}" \
  --max-num-batched-tokens "${max_num_batched_tokens}" \
  --profile-control "${profile_control_flag}"

spid=""
mpid=""
# GLOBAL, exactly like mpid, because the EXIT trap has to be able to reap it. As
# a `local` in run_leg it was invisible to cleanup_server, so any failure inside
# the bench loop -- and `set -e` makes a failing `online_gate.py bench` exactly
# that -- left the sampler polling nvidia-smi once a second forever on a box
# another session may be measuring on.
clock_pid=""
startup_launch_epoch=""
startup_ready_epoch=""
profiled_pid=""
profiled_pgid=""
# Stop the clock sampler and REQUIRE it to have written its record. A leg whose
# clock was not captured is not a leg with an unknown clock -- it is a number
# that cannot be attributed to a box state, and online_gate_summary.py voids it.
# Reads and clears the GLOBAL clock_pid so that calling it twice -- once on the
# normal path, once from the EXIT trap -- is a no-op the second time.
stop_clock_sampler() {
  local pid=${clock_pid}
  clock_pid=""
  [[ -n ${pid} ]] || return 0
  kill -TERM "${pid}" 2>/dev/null || true
  wait "${pid}"
}
cleanup_server() {
  # Before the server, because the sampler's window ends with the leg and an
  # orphan here outlives the whole script.
  stop_clock_sampler || true
  if [[ -n ${profiled_pid} ]] && kill -0 "${profiled_pid}" 2>/dev/null; then
    if [[ -n ${profiled_pgid} && ${profiled_pgid} == "${profiled_pid}" ]]; then
      kill -TERM -- "-${profiled_pgid}" 2>/dev/null || true
    else
      kill -TERM "${profiled_pid}" 2>/dev/null || true
    fi
    for _ in $(seq 1 30); do
      kill -0 "${profiled_pid}" 2>/dev/null || break
      sleep 1
    done
    if kill -0 "${profiled_pid}" 2>/dev/null; then
      if [[ -n ${profiled_pgid} && ${profiled_pgid} == "${profiled_pid}" ]]; then
        kill -KILL -- "-${profiled_pgid}" 2>/dev/null || true
      else
        kill -KILL "${profiled_pid}" 2>/dev/null || true
      fi
    fi
  fi
  profiled_pid=""
  profiled_pgid=""
  if [[ -n ${spid} ]] && kill -0 "${spid}" 2>/dev/null; then
    kill -TERM -- "-${spid}" 2>/dev/null || true
    for _ in $(seq 1 30); do
      kill -0 "${spid}" 2>/dev/null || break
      sleep 1
    done
    if kill -0 "${spid}" 2>/dev/null; then
      kill -KILL -- "-${spid}" 2>/dev/null || true
    fi
    wait "${spid}" 2>/dev/null || true
  fi
  spid=""
  if [[ -n ${mpid} ]]; then
    wait "${mpid}" 2>/dev/null || true
  fi
  mpid=""
}
trap cleanup_server EXIT INT TERM

mem_available_kib() {
  awk '/^MemAvailable:/{print $2}' /proc/meminfo
}

gpu_idle() {
  [[ -z $(nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null | tr -d '[:space:]') ]]
}

drop_caches() {
  local output=$1
  python3 -m tools.bench.drop_file_cache \
    --root "${snapshot}" \
    --root "${source_corpus}" \
    --root "${server_bin}" \
    --root "${client}" \
    --output "${output}"
}

wait_ready() {
  local log=$1
  local deadline
  deadline=$(awk -v now="$(date +%s.%N)" -v budget="${ready_timeout_seconds}" \
    'BEGIN{printf "%.3f", now + budget}')
  while :; do
    if curl -fsS --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then
      return 0
    fi
    if ! kill -0 "${spid}" 2>/dev/null; then
      echo "server exited during readiness" >&2
      tail -n 80 "${log}" >&2 || true
      return 1
    fi
    if awk -v now="$(date +%s.%N)" -v deadline="${deadline}" \
      'BEGIN{exit !(now > deadline)}'; then
      break
    fi
    sleep "${ready_poll_interval}"
  done
  echo "server readiness timed out" >&2
  return 1
}

start_server() {
  local engine=$1 repetition=$2 memory_dir=$3
  local log_dir="${evidence}/logs/${model}/${engine}"
  local log="${log_dir}/r${repetition}-server.log"
  local command_file="${log_dir}/r${repetition}-server-command.txt"
  mkdir -p "${log_dir}"
  [[ ! -e ${log} && ! -e ${command_file} ]] || {
    echo "refusing to overwrite server evidence for ${model}/${engine}/r${repetition}" >&2
    return 1
  }
  local -a server_cmd
  if [[ ${engine} == ours ]]; then
    server_cmd=(
      "${server_bin}"
      --model "${snapshot}"
      --port "${port}"
      --num-blocks "${num_blocks}"
      --max-num-seqs "${max_num_seqs}"
      --max-num-batched-tokens "${max_num_batched_tokens}"
      --no-enable-prefix-caching
      --served-model-name gate
    )
  elif [[ ${model} == q3mxfp4 ]]; then
    # MXFP4 dense 8B oracle. On sm_121 GB10 the default FlashInfer cute-dsl mxf4
    # backend aborts engine start (BackendSupportedError mm_fp4 cap 121), so we
    # force the Marlin W4A16 keep-quant kernel our arm mirrors. Dense Qwen3 has no
    # mamba SSM state, so the hybrid-only SSM cache-dtype flag is omitted here.
    server_cmd=(
      env "PATH=$(dirname "${client}"):${PATH}"
      "VLLM_DISABLED_KERNELS=FlashInferMxFp4LinearKernel"
      "${client}" serve "${snapshot}"
      --served-model-name gate
      --gpu-memory-utilization 0.6
      --max-num-seqs "${max_num_seqs}"
      --max-num-batched-tokens "${max_num_batched_tokens}"
      --no-enable-prefix-caching
      --port "${port}"
    )
  else
    server_cmd=(
      env "PATH=$(dirname "${client}"):${PATH}"
      "${client}" serve "${snapshot}"
      --served-model-name gate
      --gpu-memory-utilization 0.6
      --max-num-seqs "${max_num_seqs}"
      --max-num-batched-tokens "${max_num_batched_tokens}"
      --no-enable-prefix-caching
      --mamba-ssm-cache-dtype float32
      --port "${port}"
    )
  fi
  printf '%q ' "${server_cmd[@]}" >"${command_file}"
  printf '\n' >>"${command_file}"
  # The launch stamp is taken immediately before the spawn so nothing else is
  # attributed to startup. The memory sampler below IS inside the measured
  # window, but it is launched identically on both arms, so it cancels in the
  # ratio; only the absolute number carries its (sub-100 ms) cost.
  local startup_status=0
  startup_launch_epoch=$(date +%s.%N)
  setsid "${server_cmd[@]}" >"${log}" 2>&1 &
  spid=$!
  python3 "${repo_root}/tools/bench/sample_process_memory.py" \
    --pid "${spid}" \
    --output "${memory_dir}/r${repetition}.samples.jsonl" \
    --interval 0.1 \
    --include-gpu >"${memory_dir}/r${repetition}.summary.json" &
  mpid=$!
  wait_ready "${log}" || startup_status=$?
  startup_ready_epoch=$(date +%s.%N)
  return "${startup_status}"
}

startup_elapsed_seconds() {
  awk -v ready="${startup_ready_epoch}" -v launch="${startup_launch_epoch}" \
    'BEGIN{printf "%.3f", ready - launch}'
}

# One cold launch->ready measurement. No timed client runs: the axis is server
# lifecycle, and paying for a full bench grid to read a startup number would
# make the measurement cost hours instead of minutes. Both arms go through the
# SAME start_server, so the launched commands are the grid's verbatim.
run_startup_leg() {
  local engine=$1 repetition=$2
  local memory_dir="${evidence}/startup-memory/${model}/${engine}"
  local cache_dir="${evidence}/startup-cache-drop/${model}/${engine}"
  local startup_dir="${evidence}/startup/${model}/${engine}"
  local before_cache="${cache_dir}/r${repetition}-before.json"
  local idle_ok=0 elapsed
  mkdir -p "${memory_dir}" "${cache_dir}" "${startup_dir}"

  gpu_idle || {
    echo "GPU is not idle before startup ${model}/${engine}/r${repetition}" >&2
    return 1
  }
  drop_caches "${before_cache}"
  start_server "${engine}" "${repetition}" "${memory_dir}"
  elapsed=$(startup_elapsed_seconds)
  cleanup_server

  for _ in $(seq 1 120); do
    if gpu_idle; then
      idle_ok=1
      break
    fi
    sleep 1
  done
  ((idle_ok == 1)) || {
    echo "GPU did not return after startup ${model}/${engine}/r${repetition}" >&2
    return 1
  }
  python3 "${repo_root}/tools/bench/online_gate.py" record-startup \
    --output "${startup_dir}/r${repetition}.json" \
    --engine "${engine}" \
    --model-key "${model}" \
    --repetition "${repetition}" \
    --elapsed-seconds "${elapsed}" \
    --poll-interval-seconds "${ready_poll_interval}" \
    --probe-url "http://127.0.0.1:${port}/health" \
    --cache-drop-report "${before_cache}"
  echo "startup ${model}/${engine}/r${repetition}: ${elapsed}s" >&2
}

run_leg() {
  local engine=$1 repetition=$2
  local baseline final idle_ok=0
  local memory_dir="${evidence}/memory/${model}/${engine}"
  local thermal_dir="${evidence}/thermal/${model}/${engine}"
  local clock_dir="${evidence}/clocks/${model}/${engine}"
  local return_dir="${evidence}/memory-return/${model}/${engine}"
  local cache_dir="${evidence}/cache-drop/${model}/${engine}"
  local preflight_dir="${evidence}/preflight/${model}/${engine}"
  local before_cache="${cache_dir}/r${repetition}-before.json"
  local after_cache="${cache_dir}/r${repetition}-after.json"
  mkdir -p \
    "${memory_dir}" "${thermal_dir}" "${clock_dir}" "${return_dir}" "${cache_dir}" \
    "${preflight_dir}"

  gpu_idle || { echo "GPU is not idle before ${model}/${engine}/r${repetition}" >&2; return 1; }
  drop_caches "${before_cache}"
  baseline=$(mem_available_kib)
  start_server "${engine}" "${repetition}" "${memory_dir}"
  nvidia-smi -q -d TEMPERATURE,POWER >"${thermal_dir}/r${repetition}-before.txt"

  python3 "${repo_root}/tools/bench/run_serve_low.py" stream \
    --url "http://127.0.0.1:${port}/v1/completions" \
    --corpus "${source_corpus}/c1-r${repetition}.jsonl" \
    --output-len 128 \
    --minimum-spread 0.05 \
    --output "${preflight_dir}/r${repetition}-stream.json"

  # q3mxfp4 (MXFP4 W4 row) is scoped to the low-concurrency decode regime; the
  # NVFP4 gate models sweep the full six points. Kept in lockstep with
  # online_gate.points_for so the summary never flags a missing result group.
  local concurrency_points="1 2 4 8 16 32"
  [[ ${model} == q3mxfp4 ]] && concurrency_points="1 2 4 8"
  # The SM clock the TIMED window actually ran at (#543). The clock differs
  # between boots on this box without throttling, and a 12.79% delta repriced a
  # byte-identical kernel by 9.65% -- larger than the deficits it was used to
  # rank. Sampling starts after the preflight stream so the window is the timed
  # grid's rather than the server's warm-up, and stops before the after-thermal
  # snapshot. One probe per second, launched identically on both arms, so its
  # cost cancels in the ratio exactly as the memory sampler's does.
  #
  # --max-duration is the sampler's OWN stop condition, carried over from the
  # memory sampler's --pid: that one dies with the process it watches, and this
  # one must likewise not depend on a signal an aborting caller may never send.
  # The ceiling is a safety net, not a budget -- it sits above the driver's own
  # readiness budget (ready_timeout_seconds, 1800 s) by 4x, so it cannot truncate
  # a leg the rest of the driver would still consider live, while an orphan dies
  # instead of polling nvidia-smi once a second forever on a SHARED box.
  python3 "${repo_root}/tools/bench/gpu_clock_state.py" sample \
    --output "${clock_dir}/r${repetition}.samples.jsonl" \
    --summary "${clock_dir}/r${repetition}.summary.json" \
    --interval 1 \
    --max-duration "${clock_sampler_max_seconds}" &
  clock_pid=$!

  local concurrency
  for concurrency in ${concurrency_points}; do
    kill -0 "${spid}" 2>/dev/null || {
      echo "server died before c${concurrency}" >&2
      stop_clock_sampler || true
      return 1
    }
    python3 "${repo_root}/tools/bench/online_gate.py" bench \
      --client "${client}" \
      --tokenizer "${snapshot}" \
      --evidence "${evidence}" \
      --model-key "${model}" \
      --engine "${engine}" \
      --base-url "http://127.0.0.1:${port}" \
      --concurrency "${concurrency}" \
      --repetition "${repetition}"
  done
  stop_clock_sampler || {
    echo "SM-clock sampler failed for ${model}/${engine}/r${repetition}" >&2
    cleanup_server
    return 1
  }
  nvidia-smi -q -d TEMPERATURE,POWER >"${thermal_dir}/r${repetition}-after.txt"
  cleanup_server

  for _ in $(seq 1 120); do
    if gpu_idle; then
      idle_ok=1
      break
    fi
    sleep 1
  done
  ((idle_ok == 1)) || { echo "GPU did not return after ${model}/${engine}/r${repetition}" >&2; return 1; }
  drop_caches "${after_cache}"
  final=$(mem_available_kib)
  python3 "${repo_root}/tools/bench/online_gate.py" record-memory-return \
    --output "${return_dir}/r${repetition}.json" \
    --baseline-kib "${baseline}" \
    --final-kib "${final}" \
    --tolerance-kib 1048576 \
    --before-cache-drop-report "${before_cache}" \
    --after-cache-drop-report "${after_cache}" \
    --gpu-idle
}

run_paired_traces() {
  local trace_arm=${1:-}
  local trace_dir="${evidence}/trace/${model}"
  local artifact_prefix=""
  local gdn_ba_env_value=""
  local gdn_packed_env_value=""
  case "${trace_arm}" in
    "") ;;
    merged)
      trace_dir="${evidence}/trace/${model}/gdn-ba-merged"
      artifact_prefix="gdn-ba-merged-"
      gdn_ba_env_value=1
      ;;
    split)
      trace_dir="${evidence}/trace/${model}/gdn-ba-split"
      artifact_prefix="gdn-ba-split-"
      gdn_ba_env_value=0
      ;;
    packed)
      trace_dir="${evidence}/trace/${model}/gdn-packed"
      artifact_prefix="gdn-packed-"
      gdn_packed_env_value=1
      ;;
    rollback)
      trace_dir="${evidence}/trace/${model}/gdn-rollback"
      artifact_prefix="gdn-rollback-"
      gdn_packed_env_value=0
      ;;
    *)
      echo "unknown internal GDN trace arm: ${trace_arm}" >&2
      return 2
      ;;
  esac
  local trace_prompts=48
  if [[ ${trace_concurrency} == 2 ]]; then
    trace_prompts=6
  fi
  local vllm_profile_dir="${trace_dir}/vllm-profile"
  local vllm_metadata="${trace_dir}/vllm-profile-metadata.json"
  local vllm_corpus="${evidence}/corpus/${model}/vllm/c${trace_concurrency}-r1.jsonl"
  local vllm_summary="${trace_dir}/vllm-kernels.json"
  local vllm_log="${trace_dir}/vllm-profile.log"
  local vllm_command="${trace_dir}/vllm-profile-command.txt"
  local status="${trace_dir}/status.json"
  if [[ ${trace_concurrency} == 2 ]]; then
    status="${trace_dir}/status-c2.json"
  fi
  local cache_before_ours="${trace_dir}/cache-before-ours.json"
  local cache_between="${trace_dir}/cache-between-engines.json"
  local cache_after_vllm="${trace_dir}/cache-after-vllm.json"
  local -a ours_reps=()
  local -a ours_sqlites=()
  local -a ours_summaries=()
  local -a ours_validations=()
  local -a ours_logs=()
  local -a ours_commands=()
  local -a ours_controls=()
  mkdir -p "${trace_dir}"
  [[ ! -e ${vllm_summary} && ! -e ${status} ]] || {
    echo "refusing to overwrite paired trace evidence for ${model}" >&2
    return 1
  }
  command -v nsys >/dev/null || { echo "nsys is required for our trace" >&2; return 1; }

  gpu_idle || { echo "GPU is not idle before our trace" >&2; return 1; }
  drop_caches "${cache_before_ours}"
  local trace_rep
  for trace_rep in 1 2 3; do
    local ours_prefix="${trace_dir}/ours-r${trace_rep}"
    local ours_log="${trace_dir}/ours-r${trace_rep}-profile.log"
    local ours_command="${trace_dir}/ours-r${trace_rep}-profile-command.txt"
    local ours_control="${trace_dir}/ours-r${trace_rep}-profile-control.json"
    local shutdown_fifo="${ours_prefix}-shutdown.fifo"
    [[ ! -e ${ours_log} && ! -e ${ours_command} && ! -e ${ours_control} &&
       ! -e ${shutdown_fifo} ]] || {
      echo "refusing to overwrite ours trace repetition ${trace_rep}" >&2
      return 1
    }
    local range_index
    for range_index in 1 2 3 4; do
      local range_prefix="${ours_prefix}.${range_index}"
      [[ ! -e ${range_prefix}.nsys-rep && ! -e ${range_prefix}.sqlite &&
         ! -e ${range_prefix}-nsys-validation.json &&
         ! -e ${range_prefix}-cuda_gpu_kern_sum.txt ]] || {
        echo "refusing to overwrite ours trace repetition ${trace_rep} range ${range_index}" >&2
        return 1
      }
    done
    mkfifo --mode=600 "${shutdown_fifo}"
    local -a server_cmd=(
      "${server_bin}"
      --model "${snapshot}"
      --port "${port}"
      --num-blocks "${num_blocks}"
      --max-num-seqs "${max_num_seqs}"
      --max-num-batched-tokens "${max_num_batched_tokens}"
      # No --max-model-len: the KV pool here is num_blocks x 32 tokens
      # (4736 x 32 = 151552), so a pinned 262144 asked for four times the
      # context the pool could ever hold and is now refused at startup by the
      # KV sizing check. Leaving it unset auto-fits the serving length to the
      # pool, which is the length this server could actually serve either way.
      --no-enable-prefix-caching
      --cuda-profile-graph-replays 4
    )
    if [[ ${trace_concurrency} == 2 ]]; then
      server_cmd+=(--cuda-profile-graph-batch 2)
    fi
    server_cmd+=(
      --benchmark-shutdown-fifo "${shutdown_fifo}"
      --served-model-name gate
    )
    local -a trace_env=("${h1d_plan_env[@]}")
    if [[ -n ${gdn_ba_env_value} ]]; then
      trace_env+=("VT_GDN_MERGED_BA=${gdn_ba_env_value}")
    fi
    if [[ -n ${gdn_packed_env_value} ]]; then
      trace_env+=("VT_GDN_PACKED_DECODE=${gdn_packed_env_value}")
    fi
    local -a profile_cmd=(
      "${benchmark_clean_env[@]}"
      "${trace_env[@]}"
      nsys profile
      --trace=cuda
      --capture-range=cudaProfilerApi
      --capture-range-end=repeat:4:sync
      --flush-on-cudaprofilerstop=true
      --cuda-flush-interval=0
      --cuda-graph-trace=node:host-only
      --cuda-event-trace=false
      --sample=none
      --cpuctxsw=none
      --stats=false
      --kill=none
      --force-overwrite=true
      --output "${ours_prefix}"
      "${server_cmd[@]}"
    )
    printf '%q ' "${profile_cmd[@]}" >"${ours_command}"
    printf '\n' >>"${ours_command}"
    setsid "${profile_cmd[@]}" >"${ours_log}" 2>&1 &
    spid=$!
    local nsys_pid=${spid}
    wait_ready "${ours_log}"
    local server_pid=""
    for _ in $(seq 1 60); do
      server_pid=$(sed -n 's/^\[VT_CUDA_PROFILE\] ready pid=\([0-9][0-9]*\) signal=SIGUSR2 target_replays=4$/\1/p' "${ours_log}")
      [[ $(wc -w <<<"${server_pid}") -eq 1 ]] && break
      sleep 1
    done
    [[ ${server_pid} =~ ^[0-9]+$ ]] || {
      echo "profiled server did not emit one exact ready marker" >&2
      return 1
    }
    local shutdown_ready_pid=""
    for _ in $(seq 1 60); do
      shutdown_ready_pid=$(sed -n 's/^\[VT_BENCH_SHUTDOWN\] ready pid=\([0-9][0-9]*\) control=fifo$/\1/p' "${ours_log}")
      [[ $(wc -w <<<"${shutdown_ready_pid}") -eq 1 ]] && break
      sleep 1
    done
    [[ ${shutdown_ready_pid} == "${server_pid}" ]] || {
      echo "profiled server did not arm one exact graceful-shutdown waiter" >&2
      return 1
    }
    profiled_pid=${server_pid}
    kill -0 "${server_pid}" 2>/dev/null || {
      echo "profiled server PID is not live" >&2
      return 1
    }
    local server_identity launcher_identity nsys_identity
    local server_ppid server_pgid server_sid
    local launcher_pid launcher_ppid launcher_pgid launcher_sid launcher_comm
    local nsys_pgid nsys_sid
    server_identity=$(ps -o ppid=,pgid=,sid= -p "${server_pid}")
    read -r server_ppid server_pgid server_sid <<<"${server_identity}"
    launcher_pid=${server_ppid}
    launcher_identity=$(ps -o ppid=,pgid=,sid=,comm= -p "${launcher_pid}")
    read -r launcher_ppid launcher_pgid launcher_sid launcher_comm <<<"${launcher_identity}"
    nsys_identity=$(ps -o pgid=,sid= -p "${nsys_pid}")
    read -r nsys_pgid nsys_sid <<<"${nsys_identity}"
    [[ ${nsys_pgid} == "${nsys_pid}" && ${nsys_sid} == "${nsys_pid}" &&
       ${launcher_ppid} == "${nsys_pid}" && ${launcher_pgid} == "${nsys_pid}" &&
       ${launcher_sid} == "${nsys_pid}" && ${launcher_comm} == nsys-launcher &&
       ${server_ppid} == "${launcher_pid}" && ${server_pgid} == "${server_pid}" &&
       ${server_sid} == "${server_pid}" ]] || {
      echo "profiled server does not have the expected nsys -> nsys-launcher -> target ancestry" >&2
      echo "nsys pid=${nsys_pid} pgid=${nsys_pgid} sid=${nsys_sid}; launcher pid=${launcher_pid} ppid=${launcher_ppid} pgid=${launcher_pgid} sid=${launcher_sid} comm=${launcher_comm}; server pid=${server_pid} ppid=${server_ppid} pgid=${server_pgid} sid=${server_sid}" >&2
      return 1
    }
    profiled_pgid=${server_pgid}
    python3 "${repo_root}/tools/bench/online_gate.py" bench \
      --client "${client}" \
      --tokenizer "${snapshot}" \
      --evidence "${evidence}" \
      --model-key "${model}" \
      --engine ours \
      --base-url "http://127.0.0.1:${port}" \
      --concurrency "${trace_concurrency}" \
      --repetition 1 \
      --num-prompts "${trace_prompts}" \
      --artifact-tag "${artifact_prefix}trace${trace_rep}"
    [[ ! -e ${native_plan_target} ]] || {
      echo "forbidden native plan target was created before capture" >&2
      return 1
    }
    kill -USR2 "${server_pid}"
    python3 "${repo_root}/tools/bench/online_gate.py" bench \
      --client "${client}" \
      --tokenizer "${snapshot}" \
      --evidence "${evidence}" \
      --model-key "${model}" \
      --engine ours \
      --base-url "http://127.0.0.1:${port}" \
      --concurrency "${trace_concurrency}" \
      --repetition 1 \
      --num-prompts "${trace_concurrency}" \
      --num-warmups 0 \
      --artifact-tag "${artifact_prefix}trace${trace_rep}-probe"
    local capture_closed=0
    for _ in $(seq 1 60); do
      if grep -q '\[VT_CUDA_PROFILE\] stopped captured_replays=4 graph=0x[0-9a-f][0-9a-f]*$' "${ours_log}"; then
        capture_closed=1
        break
      fi
      kill -0 "${server_pid}" 2>/dev/null || break
      sleep 1
    done
    ((capture_closed == 1)) || {
      echo "profiled server did not close the exact four-replay window" >&2
      return 1
    }
    printf 'Q' >"${shutdown_fifo}"
    local nsys_status=0
    local nsys_exited=0
    for _ in $(seq 1 60); do
      if ! kill -0 "${nsys_pid}" 2>/dev/null; then
        nsys_exited=1
        break
      fi
      sleep 1
    done
    if ((nsys_exited == 0)); then
      echo "nsys did not exit within 60 seconds after graceful target shutdown" >&2
      kill -INT -- "-${nsys_pid}" 2>/dev/null || true
      return 1
    fi
    wait "${nsys_pid}" || nsys_status=$?
    if ((nsys_status != 0)); then
      echo "nsys exited ${nsys_status} for trace repetition ${trace_rep}" >&2
      return 1
    fi
    grep -q '^\[VT_BENCH_SHUTDOWN\] requested control=fifo$' "${ours_log}" || {
      echo "profiled server did not record its graceful-shutdown request" >&2
      return 1
    }
    grep -q '^\[VT_BENCH_SHUTDOWN\] completed control=fifo$' "${ours_log}" || {
      echo "profiled server did not complete graceful shutdown" >&2
      return 1
    }
    rm -- "${shutdown_fifo}"
    [[ ! -e ${shutdown_fifo} ]] || {
      echo "profile shutdown FIFO still exists after target exit" >&2
      return 1
    }
    spid=""
    profiled_pid=""
    profiled_pgid=""
    python3 "${repo_root}/tools/bench/online_gate.py" record-profile-control \
      --output "${ours_control}" \
      --profile-log "${ours_log}" \
      --nsys-pid "${nsys_pid}" \
      --nsys-pgid "${nsys_pgid}" \
      --nsys-sid "${nsys_sid}" \
      --nsys-exit-status "${nsys_status}" \
      --launcher-pid "${launcher_pid}" \
      --launcher-ppid "${launcher_ppid}" \
      --launcher-pgid "${launcher_pgid}" \
      --launcher-sid "${launcher_sid}" \
      --launcher-comm "${launcher_comm}" \
      --server-pid "${server_pid}" \
      --server-ppid "${server_ppid}" \
      --server-pgid "${server_pgid}" \
      --server-sid "${server_sid}" \
      --shutdown-fifo "${shutdown_fifo}" \
      --expected-batch "${trace_concurrency}"
    for range_index in 1 2 3 4; do
      local range_prefix="${ours_prefix}.${range_index}"
      local ours_rep="${range_prefix}.nsys-rep"
      local ours_sqlite="${range_prefix}.sqlite"
      local ours_summary="${range_prefix}-cuda_gpu_kern_sum.txt"
      local ours_validation="${range_prefix}-nsys-validation.json"
      [[ -s ${ours_rep} ]] || {
        echo "nsys did not write ${ours_rep}" >&2
        return 1
      }
      nsys export --type=sqlite --lazy=false --force-overwrite=false \
        --output "${ours_sqlite}" "${ours_rep}" >/dev/null
      [[ -s ${ours_sqlite} ]] || {
        echo "ours Nsight SQLite export is empty for range ${range_index}" >&2
        return 1
      }
      local -a gdn_ba_validation_args=()
      local -a gdn_validation_args=()
      if [[ -n ${gdn_ba_env_value} ]]; then
        gdn_ba_validation_args=(--gdn-ba-mode "${trace_arm}")
      fi
      if [[ -n ${gdn_packed_env_value} ]]; then
        gdn_validation_args=(--gdn-packed-mode "${trace_arm}")
      fi
      python3 "${repo_root}/tools/bench/online_gate.py" validate-nsys-trace \
        --sqlite "${ours_sqlite}" \
        --model-key "${model}" \
        --range-index "${range_index}" \
        --expected-batch "${trace_concurrency}" \
        "${gdn_ba_validation_args[@]}" \
        "${gdn_validation_args[@]}" \
        --output "${ours_validation}"
      python3 "${repo_root}/tools/bench/online_gate.py" summarize-nsys-kernels \
        --sqlite "${ours_sqlite}" \
        --model-key "${model}" \
        --range-index "${range_index}" \
        --expected-batch "${trace_concurrency}" \
        "${gdn_ba_validation_args[@]}" \
        "${gdn_validation_args[@]}" \
        --output "${ours_summary}"
      ours_reps+=("${ours_rep}")
      ours_sqlites+=("${ours_sqlite}")
      ours_summaries+=("${ours_summary}")
      ours_validations+=("${ours_validation}")
    done
    ours_logs+=("${ours_log}")
    ours_commands+=("${ours_command}")
    ours_controls+=("${ours_control}")
    gpu_idle || {
      echo "GPU is not idle after ours trace repetition ${trace_rep}" >&2
      return 1
    }
  done

  gpu_idle || { echo "GPU is not idle before vLLM trace" >&2; return 1; }
  drop_caches "${cache_between}"
  local -a vllm_profile_cmd=(
    "${benchmark_clean_prefix[@]}"
    "PATH=$(dirname "${client}"):${benchmark_system_path}"
    "${vllm_python}" "${repo_root}/tools/bench/profile_vllm_online_gate.py"
    --model "${snapshot}"
    --corpus "${vllm_corpus}"
    --profile-dir "${vllm_profile_dir}"
    --metadata "${vllm_metadata}"
    --num-prompts "${trace_prompts}"
    --max-concurrency "${trace_concurrency}"
    --max-num-seqs "${max_num_seqs}"
    --max-num-batched-tokens "${max_num_batched_tokens}"
    --repetitions 3
  )
  printf '%q ' "${vllm_profile_cmd[@]}" >"${vllm_command}"
  printf '\n' >>"${vllm_command}"
  "${vllm_profile_cmd[@]}" >"${vllm_log}" 2>&1
  if [[ ${trace_concurrency} == 16 ]]; then
    python3 "${repo_root}/tools/bench/summarize_torch_kernels.py" \
      --profile-dir "${vllm_profile_dir}" \
      --model-key "${model}" \
      --output "${vllm_summary}"
  else
    python3 "${repo_root}/tools/bench/summarize_torch_kernels.py" \
      --profile-dir "${vllm_profile_dir}" \
      --output "${vllm_summary}"
  fi
  local vllm_trace
  vllm_trace=$(python3 - "${vllm_summary}" <<'PY'
import json
import pathlib
import sys
value = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
print(value["selected_trace"])
PY
)
  local -a status_args=(
    --output "${status}"
    --model-key "${model}"
    --vllm-torch-trace "${vllm_trace}"
    --vllm-kernel-summary "${vllm_summary}"
    --vllm-command "${vllm_command}"
    --vllm-profile-log "${vllm_log}"
    --vllm-metadata "${vllm_metadata}"
    --vllm-corpus "${vllm_corpus}"
    --cache-drop-report "${cache_before_ours}"
    --cache-drop-report "${cache_between}"
    --cache-drop-report "${cache_after_vllm}"
    --execution-manifest "${execution_manifest}"
    --vllm-cpp-sha "${vllm_cpp_sha}"
  )
  local trace_index
  for trace_index in $(seq 0 11); do
    status_args+=(
      --ours-nsys-report "${ours_reps[trace_index]}"
      --ours-nsys-sqlite "${ours_sqlites[trace_index]}"
      --ours-nsys-validation "${ours_validations[trace_index]}"
      --ours-kernel-summary "${ours_summaries[trace_index]}"
    )
  done
  for trace_index in 0 1 2; do
    trace_rep=$((trace_index + 1))
    status_args+=(
      --ours-command "${ours_commands[trace_index]}"
      --ours-profile-log "${ours_logs[trace_index]}"
      --ours-profile-control "${ours_controls[trace_index]}"
      --ours-client-result "${evidence}/raw/${model}/ours/c16-r1-trace${trace_rep}.json"
      --ours-client-log "${evidence}/logs/${model}/ours/c16-r1-trace${trace_rep}.log"
      --ours-probe-result "${evidence}/raw/${model}/ours/c16-r1-trace${trace_rep}-probe.json"
      --ours-probe-log "${evidence}/logs/${model}/ours/c16-r1-trace${trace_rep}-probe.log"
    )
  done
  gpu_idle || { echo "GPU did not return after paired traces" >&2; return 1; }
  drop_caches "${cache_after_vllm}"
  if [[ ${trace_concurrency} == 2 ]]; then
    if [[ -n ${gdn_ba_env_value} ]]; then
      echo "c2 GDN BA ${trace_arm} raw paired trace capture complete; status remains PENDING until GDN BA finalization" >&2
    elif [[ -n ${gdn_packed_env_value} ]]; then
      echo "c2 GDN packed ${trace_arm} raw paired trace capture complete; status remains PENDING until GDN packed finalization" >&2
    else
      echo "c2 raw paired trace capture complete; status remains PENDING until low-batch finalization" >&2
    fi
    return 0
  fi
  python3 "${repo_root}/tools/bench/online_gate.py" record-trace-status \
    "${status_args[@]}"
}

# One lock for every ours/vLLM leg of this model, including the model gate.
exec 9>/tmp/gpu
flock 9
gpu_idle || { echo "GPU has a compute owner after acquiring /tmp/gpu" >&2; exit 1; }

# --startup-only measures ONE axis: cold launch -> first /health, interleaved
# ours/vLLM under this single lock. It deliberately skips the token model gate
# and the timed grid: no generated token is compared and no throughput number is
# produced, so neither is a precondition for it (and running the gate here would
# collide with a real grid's gate evidence in the same root).
if [[ ${mode} == startup-only ]]; then
  # Loop variable is deliberately NOT the grid's `repetition`: the --execute
  # contract test anchors on that exact loop header to isolate the timed tail.
  for startup_repetition in 1 2 3; do
    run_startup_leg ours "${startup_repetition}"
    run_startup_leg vllm "${startup_repetition}"
  done
  python3 "${repo_root}/tools/bench/online_gate.py" summarize-startup \
    --evidence "${evidence}" \
    --model-key "${model}" \
    --repetitions 1 2 3 \
    --output "${evidence}/startup/${model}/summary.json"
  cat "${evidence}/startup/${model}/summary.json"
  exit 0
fi

gate_dir="${evidence}/preflight/model-gate"
gate_log="${gate_dir}/${model}.log"
gate_status="${gate_dir}/${model}.json"
mkdir -p "${gate_dir}"
[[ ! -e ${gate_log} && ! -e ${gate_status} ]] || {
  echo "refusing to overwrite model-gate evidence for ${model}" >&2
  exit 1
}
if [[ ${model} == q3mxfp4 ]]; then
  # MXFP4 correctness precondition = the #44 e2e smoke battery (default config,
  # async ON + the classic-dense device-mirror fix). vllm-cli links vllm::shared
  # (build-tree RPATH) and needs the operator CUDA environment, so it runs in the
  # script shell rather than the ultra-clean measurement env used for timed legs.
  if ! python3 "${repo_root}/tools/bench/mxfp4_smoke_gate.py" \
    --vllm-cli "${build_dir}/examples/vllm-cli" \
    --snapshot "${snapshot}" \
    --golden "${repo_root}/docs/bench-evidence/mxfp4-qwen/golden_marlin_w4a16.json" \
    >"${gate_log}" 2>&1; then
    cat "${gate_log}" >&2
    exit 1
  fi
# -V, not just --output-on-failure: a checkpoint-gated parity test emits a loud
# SKIP MESSAGE and returns 0, so a PASSING run must also land its own output in
# the log, or record-model-gate cannot tell a real gate from a skipped one.
elif ! "${benchmark_clean_env[@]}" "${h1d_plan_env[@]}" \
  ctest --test-dir "${build_dir}" -R "^${test_name}$" -V --output-on-failure \
  >"${gate_log}" 2>&1; then
  cat "${gate_log}" >&2
  exit 1
fi
cat "${gate_log}"
python3 "${repo_root}/tools/bench/online_gate.py" record-model-gate \
  --output "${gate_status}" \
  --log "${gate_log}" \
  --model-key "${model}" \
  --test-name "${test_name}" \
  --vllm-cpp-sha "${vllm_cpp_sha}"

if [[ ${mode} == trace-only ]]; then
  if [[ ${gdn_ba_mode} == both ]]; then
    run_paired_traces merged
    run_paired_traces split
    echo "model ${model} GDN BA merged/split node-level paired traces complete" >&2
  elif [[ ${gdn_packed_mode} == both ]]; then
    run_paired_traces packed
    run_paired_traces rollback
    echo "model ${model} GDN packed/rollback node-level paired traces complete" >&2
  else
    run_paired_traces
    echo "model ${model} node-level paired trace complete" >&2
  fi
  exit 0
fi

# --execute is a PURE TIMED production grid: model gate + interleaved timed
# legs (memory/memory-return/thermal/cache-drop capture) + summary, all from
# the production profile-control-OFF build. The H1d paired-trace machinery
# (nsys wrapping, --cuda-profile-graph-replays, record-profile-control) runs
# ONLY under --trace-only, which requires the instrumented build; mixing it
# into --execute is what the production server correctly refuses.
for repetition in 1 2 3; do
  run_leg ours "${repetition}"
  run_leg vllm "${repetition}"
done

model_summary_status=0
python3 "${repo_root}/tools/bench/online_gate_summary.py" \
  --evidence "${evidence}" \
  --model "${model}" || model_summary_status=$?
if ((model_summary_status > 1)); then
  exit "${model_summary_status}"
fi

full_summary_status=0
if [[ -f ${evidence}/summary-27/ratios.json && -f ${evidence}/summary-35/ratios.json ]]; then
  python3 "${repo_root}/tools/bench/online_gate_summary.py" \
    --evidence "${evidence}" || full_summary_status=$?
  if ((full_summary_status > 1)); then
    exit "${full_summary_status}"
  fi
else
  echo "model ${model} summary complete; cross-model summary waits for the other model" >&2
fi
if ((model_summary_status != 0 || full_summary_status != 0)); then
  exit 1
fi