From 068bdffc6490e878a897f402724356eda8e44e09 Mon Sep 17 00:00:00 2001 From: Vikash Loomba Date: Wed, 12 Aug 2026 16:24:20 -0700 Subject: [PATCH 1/2] gate(ROCM): M4 -- pinned vLLM-ROCm oracle on gfx1100 + ROCm device-golden lane in the Qwen3-0.6B near-tie gate Issue #41 M4 milestone. The oracle is upstream vLLM AT THE PIN (555967922, runtime 0.23.1rc1.dev1511+g555967922 -- verified == the parity-pin block) built from source as a ROCm wheel for gfx1100 in rocm/vllm-dev:base and committed as the image vllm-rocm-oracle:555967922-gfx1100 (recipe: .agents/specs/rocm-m4-oracle.md). Gate result on this box (4x RX 7900 XTX, ROCm 7.14, enforce_eager, batch=1, flock ~/gpu.lock): oracle K=10 per-prompt greedy deterministic in every cell; our ROCm engine 16/16 prompts PASS -- 11/16 strict token-exact vs base greedy, 5/16 near-tie band, max teacher-forced gap 0.125 nats, 0 forward-divergent, 125/125 assertions, backend proof all 8 Qwen3-dense ops selections>0 and declines==0 on device 5. The p0 France/Italy flip is a literal 0.0000-nat tie in the oracle's own logits (the Metal M3b signature, reproduced on ROCm). Change shape: the parity gate's Metal/Tenstorrent device-golden lane gains the kROCM arm (identical anchor+band logic, ROCm's own oracle-backed golden pair); dgx base goldens byte-untouched; the *_rocm goldens are additive. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: pi:kimi-k3 [pi] --- .agents/NOW.md | 2 +- .agents/coordination.md | 1 + .agents/parity-ledger.md | 1 + .agents/specs/rocm-m4-oracle.md | 74 ++++++++++++++++++ docs/STATUS.md | 7 +- .../qwen3_greedy_0_6b/greedy_dist_rocm.npy | Bin 0 -> 10368 bytes .../qwen3_greedy_0_6b/greedy_ids_rocm.npy | Bin 0 -> 1152 bytes .../neartie_gap_mnats_rocm.npy | Bin 0 -> 1152 bytes .../qwen3_greedy_0_6b/our_ids_rocm.npy | Bin 0 -> 1152 bytes tests/parity/test_qwen3_paged_engine.cpp | 18 +++-- 10 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 .agents/specs/rocm-m4-oracle.md create mode 100644 tests/parity/goldens/qwen3_greedy_0_6b/greedy_dist_rocm.npy create mode 100644 tests/parity/goldens/qwen3_greedy_0_6b/greedy_ids_rocm.npy create mode 100644 tests/parity/goldens/qwen3_greedy_0_6b/neartie_gap_mnats_rocm.npy create mode 100644 tests/parity/goldens/qwen3_greedy_0_6b/our_ids_rocm.npy diff --git a/.agents/NOW.md b/.agents/NOW.md index 4aa86ac55..130ddb7ad 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -1,6 +1,6 @@ # NOW — the one-Read resume surface - + Snapshot, not log. History is git; evidence: [parity ledger](parity-ledger.md), and benchmarks. Budget: 100 lines / 6,000 diff --git a/.agents/coordination.md b/.agents/coordination.md index ef4491c2e..55c720dae 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -1996,6 +1996,7 @@ this claim will meet. The tiled row is speed-gatable on dgx. | `CLAIM-ENG-LOAD-DIRECT-UPLOAD` | `ENG-LOAD-DIRECT-UPLOAD` (engine-matrix, NEW row -> `ACTIVE`) | Claude Code (opus-5) | isolated worktree `.claude/worktrees/agent-a122a5ddc3546bf31`; GB10 `dgx.casa` tree `~/work/load150/src` (Vulkan `build-vk` + CUDA `build-cuda`), every GPU stage under `flock $HOME/gpu.lock`, `local-ai-worker` parked and restored | branch `row/ENG-LOAD-DIRECT-UPLOAD`, base `origin/main` `375a471e` | Issue #150. Owns ONLY: the refcounted safetensors mapping + byte counters (`include/vllm/model_executor/model_loader/safetensors_reader.h`, `src/vllm/model_executor/model_loader/safetensors_reader.cpp`), `BorrowStTensorBytes` + the `AdoptDeviceBytesAsHost` mmap branch + `OwnedTensor::mmap_src` (`include/vllm/model_executor/models/qwen3_5_weights.h`, `src/vllm/model_executor/models/qwen3_5_weights.cpp`), the qualifying call sites in `include/vllm/model_executor/models/dense_weight_loaders.h` and `src/vllm/model_executor/models/qwen3_5_dense_weights.cpp`, the upload counter in `include/vllm/model_executor/models/dense_attn_block.h`, the `VT_LOAD_STATS` phase timing in `src/vllm/entrypoints/model_loader.cpp`, NEW `tests/vllm/test_load_direct_upload.cpp` + its `tests/CMakeLists.txt` line, NEW `.agents/specs/load-direct-upload.md`, the NEW engine-matrix row + its Loading/Total rollup, the `#150` roadmap intake cell, two `docs/ENVIRONMENT.md` rows, this claim, `.agents/NOW.md`, and the `docs/STATUS.md`/`docs/BENCHMARKS.md` one-liners. **NON-COLLISION:** no new kernel, no model registry change, no CMake target change beyond one test line; every non-verbatim loader helper is untouched and the lever is same-binary A/B-able (`VT_LOAD_DIRECT_UPLOAD=0`). MUST NOT touch README, Metal/SACRED/apex/darwin, or any model/kernel source | `ACTIVE` | 2026-08-09 — mechanism test 6/6 (77) and RED under two mutations; GB10 Vulkan gates on the changed tree green (`test_vulkan_backend` 35/35·2650, `test_backend_cross_device` 11/11·132, `test_opt_paged_engine` 6/6 prompts token-exact 96/96, 0 declines, device type 3); GB10 CUDA full `ctest` 383/393 with BOTH SACRED gates PASS and all 10 failures reproduced on a clean `origin/main` build; 27B bf16 load 1.54x warm / 1.61x cold, bytes moved 100.196 -> 81.260 GiB; `local-ai-worker` restored (`--restart=always`, Up). Fresh scoped review OWED | | `CLAIM-ENG-RELEASE-CONTAINERS-W1-W7` | `ENG-RELEASE-CONTAINERS` (`INVENTORIED`->`READY`->`ACTIVE`; spike spec + W1-W5/W7) | Claude Code (opus-5), helper role | isolated worktree `/home/mudler/_git/vllm.cpp-containers`; local Docker only -- NO GPU, NO registry push, NO publication | `row/ENG-RELEASE-CONTAINERS`, base `origin/main` `24306364`, PR [#307](https://github.com/mudler/vllm.cpp/pull/307) | Container spike spec, `docker/Dockerfile` (cpu/vulkan/cuda calling the release scripts), container matrix + checker, image validator, workflow + least-privilege guard, and the SIGTERM fix the boot gate surfaced (#312). Excludes W6 matching-hardware runtime evidence and any GHCR publication | `ACTIVE` | 2026-08-10 -- cpu lane built and gated e2e (config+layout+boot, clean SIGTERM 0.25 s, RED 137 before the fix); mutation suites 31/31 and 22/22; cuda/vulkan gated statically but never built here; nothing published | | `CLAIM-ROCM-GDN-KERNELS` | `BACKEND-ROCM` (stays `ACTIVE` — M3 GDN kernel slice, issue #41) | pi (kimi-k3) | isolated worktree `/home/vikash/vllm.cpp-rocm-gdn-kernels`, base `origin/main` `5812b8b6`; 4x RX 7900 XTX gfx1100 discrete, ROCm 7.14, GPU jobs hold `~/gpu.lock` | `row/ROCM-GDN-KERNELS` | The M3 GDN kernel family unblocking Qwen3.5-0.8B M2 on discrete ROCm. Owns ONLY: NEW `.agents/specs/rocm-gdn-kernels.md`; NEW `src/vt/rocm/rocm_gdn.hip` (10 ops: kGdnStateGather/Scatter 77/78, kCausalConv1dFwd/Update 5/6, kRmsNormGated 9, kSigmoidGateBf16 58, kGdnPrefill/Decode 10/11, kGdnPostConv 65, kAttnQkNormRopeGate 67 — hand-translated from `src/vt/cuda/cuda_gdn.cu`/`cuda_ops.cu` donors); ADDITIVE registration entries in `src/vt/rocm/rocm_ops.hip`; ADDITIVE GDN cases + buffer helpers in `tests/vt/test_backend_cross_device.cpp`; this claim + `.agents/NOW.md`. **NON-COLLISION:** every new file is ROCm-only; the two shared-file edits are additive registration/case blocks that change no existing behavior (unregistered-op backends skip; CUDA/CPU/Vulkan paths byte-identical). Disjoint from #234 (sampling), #317 (gemma4 FP8/WMMA), #273 (near-tie docs). | `ACTIVE` | 2026-08-11 — **ALL 5 FAMILIES LANDED AS STACKED DRAFT PRs + M2 E2E COMPLETE.** #334 (spec + state I/O), #336 (conv), #341 (postconv), #343 (recurrence), #345 (norm-gate/preamble + the SupportsCompressedGdnState capability seam). Qwen3.5-0.8B runs e2e all-native on discrete gfx1100, deterministic; 'The sky is' 8/8 CPU parity; open-prompt divergence characterized as not-state-rounding, GDN-path drift analysis named open. Gates per family: cross-device green (162/162 total), focused 3/3, full-suite zero-delta vs base. NEXT: fresh review + operator gate per PR; then the MoeRouterTopK bf16-logits arm (the named MoE-path blocker) and the GDN-path divergence characterization. | +| `CLAIM-ROCM-M4` | `BACKEND-ROCM` (stays `ACTIVE` — M4 correctness gate on gfx1100, issue #41) | pi (kimi-k3) | isolated worktree `/home/vikash/vllm.cpp-rocm-m4`, base `origin/main` `8b00f79f`; 4x RX 7900 XTX gfx1100 discrete, ROCm 7.14, GPU jobs hold `~/gpu.lock`; the pinned vLLM-ROCm oracle runs in the committed container image `vllm-rocm-oracle:555967922-gfx1100` | `row/ROCM-M4-GATE` | The M4 gate: the pinned vLLM-ROCm oracle on gfx1100 + the ROCm device-golden lane in the Qwen3-0.6B near-tie gate. Owns ONLY: NEW `.agents/specs/rocm-m4-oracle.md` (oracle build/capture/gate recipe); ADDITIVE `kROCM` arm in `tests/parity/test_qwen3_paged_engine.cpp` (the existing Metal/Tenstorrent device-golden lane extended, no CUDA/CPU/Metal/TT behaviour change); NEW goldens `tests/parity/goldens/qwen3_greedy_0_6b/{our_ids_rocm.npy,neartie_gap_mnats_rocm.npy,greedy_ids_rocm.npy,greedy_dist_rocm.npy}` (dgx base byte-untouched); this claim + `.agents/NOW.md` + `parity-ledger` row + `docs/STATUS.md`. **NON-COLLISION:** the gate edit is a three-branch ternary extension already established for two other devices; the goldens are additive files; disjoint from the in-flight kernel PRs (#497/#506/#509/#523) which touch no parity-gate file. | `ACTIVE` | 2026-08-12 — **M4 GATE LANDED GREEN.** Oracle built from source at pin `555967922` in `rocm/vllm-dev:base` (wheel `0.23.1rc1.dev1511+g555967922.rocm723`, `_rocm_C` ABI present; runtime identity string verified == the parity-pin block), K=10 per-prompt greedy capture DETERMINISTIC in every (prompt,pos) cell. Our engine vs the SAME-BOX oracle: **16/16 prompts PASS** (STRICT token-exact 11/16, near-tie-band 5/16, max gap 0.125 nats @ prompt[3] tok1, 0 forward-divergent; 125/125 assertions), BACKEND PROOF kPagedAttention selections 7,168 / 0 declines on device type 5. The known p0 France/Italy flip is a literal 0.0000-nat tie in the oracle's own logits (Metal-lane M3b signature, reproduced on ROCm). NEXT: Qwen3.5-0.8B (GDN) and the 35B keep-quant gates under the same oracle; fresh review + operator gate for this PR. | **CLOSED same-session claim — `CLAIM-TP-SPIKE-287` (task #287, 2026-08-08, records-only, helper `row/SPIKE-TENSOR-PARALLELISM` draft PR #143, base `b38f78a7`).** Not an active-claims row because every referenced row keeps its diff --git a/.agents/parity-ledger.md b/.agents/parity-ledger.md index 68e733335..ce9f5a692 100644 --- a/.agents/parity-ledger.md +++ b/.agents/parity-ledger.md @@ -940,3 +940,4 @@ Columns: | 2026-08-10 (**`FUSION-DENSE-MIGRATE` — the merged-GEMM allowlist's no-blocker half CLOSED**; issue [#299](https://github.com/mudler/vllm.cpp/issues/299), spec `specs/fusion-dense-migrate.md`; `row/FUSION-DENSE-MIGRATE`, base `origin/main` `c70f42b9` (rebased from the `e17e8f8a` pinned at worktree creation, via `688eea12`), isolated worktree `/home/mudler/_git/vllm.cpp-fusion-fold`; CPU-only, `benchmark_binding=false`) | Routes the gate/up of the five plain bf16 dense SwiGLU MLPs that bypassed the MUST-route merged-GEMM seam with no stated blocker — `commandr`, `glm4`, `minicpm`, `minicpm3`, `phi3` — through `layers::UnquantizedMlpGateUpMethod`, and deletes their five `pending FOLD-MIGRATE` entries from `scripts/merged-gemm-consistency-allowlist.txt` (11 → 6). The six survivors all name a blocker that needs the SHARED LAYER extended (f32 activations + rank-1 biases, ggml block-quant vs the explicitly UNQUANTIZED arm, up-front device staging vs `OwnedTensor`/`ResidentWeight` residency, a GeGLU arm on the SwiGLU-only grouped MoE op for `gemma4_moe`, an NVFP4-Marlin-resident arm for `laguna`'s raw-device-pointer w13) and were NOT touched: extending the seam is its own row. The `gemma4_moe` and `laguna` reasons were REWRITTEN in this branch: both previously read as a bare `pending fold`, which is the vague reason this row's own thesis says let the allowlist grow. Direct Unquantized arm, not `MakeMlpGateUpMethod` — no loader for the three `Qwen3DenseMlpWeights` models ever populates `*_fp4`. Shared-layer TUs (`linear.h`/`nvfp4.h`) and every `vt::` op UNTOUCHED. No env flag: the fold is unconditional and bit-exact, so a rollback arm would be dead code. | Upstream expresses all five as ONE `MergedColumnParallelLinear(gate_up_proj, output_sizes=[I]*2)` + `SiluAndMul()` at pin `555967922` (0.26.0.dev0): `commandr.py:91,102-108,116`; `glm4.py:46` → `llama.py:79,92-99,113`; `minicpm.py:193,204-211,219`; `minicpm3.py:186` (inherits `MiniCPMMLP`); `phi3.py:10` → `llama.py:79`. The fold moves us TOWARD that shape; no new upstream code was ported. | **BYTE-EXACT BY CONSTRUCTION, and that construction is what was gated** (a routing change, so the bar is op-sequence identity, never a tolerance). Each replaced body was the seam's own `{ResidentWeight; MatmulBT[2I,H]; SiluAndMul}` with `M` spelled `T`, and every call site passes a `DBuf{T,H}` so `x.shape[0] == T` identically. NEW byte-exact CPU case in `test_linear_method` for the DIRECTLY-constructed arm at BOTH the decode shape (M=1) and a prefill shape (M=4) vs the standalone sequence — RED-first PROVEN (mutating the seam's `SiluAndMul`→`GeluAndMul` fails 2 cases / 11 assertions; GREEN byte-for-byte on restore) → **6 cases / 76 assertions GREEN**. `check-fusion-consistency.py` RC=0 with 0 drift; two NEW mutation cases in `tests/scripts/test_check_fusion_consistency.py` make BOTH regressions RED (re-allowlisting a folded stem; reverting a fold) → **20/20**. MECHANICAL CORRECTION to issue #299: a fully-folded TU has NO hand-call left, so it leaves the detector's DENOMINATOR rather than entering its numerator — the checker reads `15 scanned / 6 routed / 11 allowlisted` → `10 / 6 / 6`, not the `11 routed / 15` the issue predicted; drift is 0 either way, which is what the gate asserts. Full CPU `ctest -j 6` on the final rebased base `60e71a0e`: **369/369, 0 failed**, 1288.75 s. RE-RUN on the MERGED head `4b99cefb` (after merging `origin/main` `5812b8b6` and adding the new coverage binary): **369/370, 1 failed**, 1441.85 s — the single failure is `test_engine_core_proc`, a STARVATION FLAKE and not a regression: serially on the same binary it is **10/10 cases, 93 assertions GREEN**, whereas under `-j 6` at load average ~170 (two other worktrees running their own suites) it failed in 0.06 s on `CHECK(abort_seen)` (`test_engine_core_proc.cpp:345`) having spun to 1089 assertions waiting for the abort. It is one of the four known starvation-prone binaries, loads none of the five folded TUs, and the other three passed in this run. `test_dense_gate_up_seam_forward` passed inside the suite (32.96 s), proving the new binary is registered and wired. An earlier run on the intermediate base read **367/369**, and both numbers are reported rather than the red one deleted: `test_async_llm` and `test_openai_conformance` failed while a SECOND worktree ran its own suite at load average 89-122, and re-run alone on a quiet box (load 12.8) passed in **0.04 s** and **20.76 s** (vs 605 s of `statuses == -1`, the client's 30 s read timeout, under load); `test_openai_conformance` builds a synthetic `Qwen3_5MoeForConditionalGeneration` in-process and loads none of the five folded TUs, so it could not be this row's, and the clean 369/369 settles it. The #274 ASan/UBSan five did not appear in either run — this gate is Release with no sanitizer. **OWED / NOT CLAIMED:** the five paged-engine SACRED gates (`test_{commandr,glm4,minicpm,minicpm3,phi3}_paged_engine`) are checkpoint-gated dgx-only and emit a loud SKIP on a CPU box, so empirical token-exact confirmation is PENDING for the next GPU holder — the same shape A1 carried for Granite/StableLM — and is tracked by [#337](https://github.com/mudler/vllm.cpp/issues/337) so the handle outlives #299. EXECUTED CPU COVERAGE OF THE FOLDED TUs (added on review finding F4, which proved that mutating `phi3`'s `I` to `I - 1` at the call site survived 176 CPU tests): NEW `tests/vllm/models/test_dense_gate_up_seam_forward.cpp` drives the REAL forward of four of the five folded TUs (`commandr`/`glm4`/`minicpm`/`phi3`) over synthetic in-memory weights with no checkpoint and no GPU — **4 cases / 1940 assertions GREEN** — pinning the gate/up split analytically (zeroing the UP half and zeroing `down_proj` both make the MLP contribute exactly zero, so they must be BYTE-IDENTICAL, which holds only at a split of exactly `I`) plus the half order and a vacuity guard. RED-first EXECUTED both ways: `I` → `I - 1` at all four call sites fails all four cases (`phi3` alone fails only its own, so each TU is genuinely reached), and a SHAPE-PRESERVING mutation of `vt::SiluAndMul`'s CPU kernel (`silu * up` → `silu + up`) fails all four on the split assertion itself while `test_linear_method` stays GREEN at 76/76 — a defect class the seam unit test structurally cannot see. Both restored byte-for-byte (md5 verified). This also retires the row's original "no e2e evidence is possible on a CPU box" framing as too strong: ORACLE evidence needs the GPU, SELF-CONSISTENCY evidence does not. `minicpm3` is the one folded TU that harness does not drive (MLA + load-time `kv_b_proj` absorption belong to the DeepSeek-V2 synthetic harness). Also REJECTED as evidence: an object-code A/B — compiling each of the five TUs from main's source and from the folded source with the identical production command and diffing the disassembly gives 3689-6888 differing instruction lines per TU, which is GCC re-allocating registers around a header-inlined method, not a numerical signal in either direction. STATED RATHER THAN HIDDEN: the OP sequence is identical but the ALLOCATION sequence is not — `DBuf` is pooled and returns its block on destruction (`dense_device_glue.h:99`), so the `[T,2I]` gate_up buffer is now released when `Apply` returns instead of at the end of the MLP block and the `[T,H]` output may reuse it. That cannot move a value (`vt::MatmulBT` writes every output element from a fresh f32 accumulator rather than accumulating into `out`) and peak pool usage is unchanged or lower; it is the same shape every earlier fold produced. ALSO REPOINTED: `glm4`/`phi3` sit on the OTHER (glue) allowlist reading `pending FUSION-DENSE-MIGRATE`; closing this row would leave both aimed at closed work, so their reasons now name [#314](https://github.com/mudler/vllm.cpp/issues/314), which owns the glue half — the glue fold itself is NOT done here. | | 2026-08-11 (`ENG-TRAILER-MERGE-ARTIFACTS`; implementation merge `157080c8`, issue [#406](https://github.com/mudler/vllm.cpp/issues/406)) | The trailer gate was rejecting CORRECT commits. `check-commit-trailers.py` reads through `git interpret-trailers --parse`, which treats only the FINAL paragraph as the block, and GitHub appends `Co-authored-by:` as a separate trailing paragraph on squash merge — so a complete block became invisible and 13 of the last 30 commits on `main` failed the check, unnoticed because those runs were cancelled (#274). Fixed by fusing consecutive trailing TRAILER-SHAPED paragraphs before parsing; no rule relaxed, and a prose paragraph still terminates the block. | `test_check_commit_trailers.py` 21/21 — the RED-BEFORE appended-co-author case plus four guards (doubled block, contradictory declarations, no-trailer merge message, prose after the block) green before and after; the five real `main` commits re-verified per commit and the verdict table reproduces. | Closes ONE of five shapes. `f64f2b71` (bot co-author) is a REAL violation the parse had hidden and now correctly fails; `87308dea`, `b8293c88` and `b580452d` stay red by design — merge-method artifacts, not checker defects. A first attempt that also collapsed identical duplicate trailers was REVERTED in full: it relaxed a uniqueness rule an existing test pins. | | 2026-08-12 (**Audio ENCODER TTFT — FA-2 TENSOR CORES for the hd-64 non-causal encoder attention; 5.50x encoder forward / 115.8x kernel, LANDS OPT-IN because it costs precision**; `CLAIM-MM-SPEED-AUDIO-ENC-FA2`, multimodal-speed.md §17, issue [#432](https://github.com/mudler/vllm.cpp/issues/432); worktree `/home/mudler/_git/vllm.cpp-mm-speed` branch `row/MM-SPEED-ENC-FA2` off `dc7a1392`) | New dense hd-64 non-split FA-2 instantiation (`flash_fwd_hdim64_bf16_sm80.cu`, upstream's own `run_mha_fwd_hdim64` via the plain batch entry — both firsts here) + `LaunchDenseFA2Bf16` (b=1, null `cu_seqlens_q`) + additive op `vt::AttentionDenseFa2`, routing the Whisper encoder self-attention to the kernel vLLM itself dispatches (`whisper.py:298-317` -> `flash_attn_varlen_func`). dgx GB10 sm_121a, all three mandatory banners CONFIRMED, `-Werror` 0-warn, GPU under `flock` on a CONTENDED box (3 other agents' jobs queued). **SPEED:** encoder attention 19,278 -> 166.5 us/layer (115.8x, nsys 32 inst both arms, SAME tool); encoder forward 731.7 -> 133.0 ms (5.50x, non-overlapping bands, rep0 dropped) => the ENCODER FORWARD goes from 15.90x to **2.89x** of vLLM's whole TTFT against the PIN `555967922` (TTFT median 46.02 ms, 45.60-46.41, production/graphed, 6 reps rep0 dropped, re-measured 2026-08-12). NOT a TTFT ratio: our projector, merge and prefill are unmeasured (F3). The originally published `~17.1x -> ~3.11x` used the carried-forward 0.25.0 denominator (42.8 ms), which the pin is 7.5% faster than - so the published figure was CONSERVATIVE. Oracle asserted BY COMMIT: the venv's `0.23.1rc1.dev1511+g555967922` version string is a setuptools_scm nearest-ancestor-tag artefact, not a mismatch; `soundfile==0.14.0` had to be installed into `~/venvs/vllm-oracle-next` before the pin could tokenize Voxtral at all (recorded against #375). **CORRECTNESS:** oracle teacher-force (fixture's own stack, asserted live as vLLM 0.25.0 + mistral_common 1.11.5) = 3 divergent, worst 0.1250 nats, over-band 0, **RESULT PASS** — inside the ratified band but the shipping kernel has 0 divergent at gap 0.0. Cause is a HYPOTHESIS, not grounded (corrected 2026-08-12 by the fresh review of PR #439): the stated cause - FA-2 converting P to bf16 before the PV MMA (`flash_fwd_kernel.h:347`) - was REFUTED by mutation M4, which forced exactly that rounding into the shipping scalar kernel, rebuilt clean, and got the SAME token md5 `89923566...`. Five candidate differences remain and none is isolated. **DISPOSITION: default UNCHANGED (byte-exact, `test_voxtral_e2e` 16/16, goldens md5 unchanged); FA-2 opt-in behind `VT_WHISPER_ENC_FA2=1`; ADOPTION = DEVELOPER DECISION.** Not a ceiling, RE-RANKED after M4: (1) attention is now only 5.33 ms of the 133 ms, so the §15.1-deferred device im2col kernel is the top encoder lever; (2) measure our ACTUAL TTFT (projector + merge + prefill); (3) isolate which of the five FA-2/scalar differences flips the tokens, one M4-style mutation at a time; (4) the FA-3-style f32-correction rescale, DEMOTED from #1 because M4 refuted its premise. Repairs landed on the branch: issue [#463](https://github.com/mudler/vllm.cpp/issues/463) for the zero-assertion SUCCESS skip (now `exit 77` + CTest `SKIP_RETURN_CODE`), `tests/vt/test_ops_attention_dense_fa2.cpp` for the op's missing tests, and a `causal` parameter on `LaunchDenseFA2Bf16` that THROWS instead of silently answering non-causally. No mm row advances to DONE. | +| 2026-08-12 (**M4 on gfx1100 — the pinned vLLM-ROCm oracle stood up on the only board that can host it + the ROCm device-golden lane in the Qwen3-0.6B near-tie gate; issue #41; `CLAIM-ROCM-M4`; worktree `/home/vikash/vllm.cpp-rocm-m4` branch `row/ROCM-M4-GATE`, base `8b00f79f`) | The M4 milestone for `BACKEND-ROCM`: build the PINNED vLLM oracle (`555967922`, runtime `0.23.1rc1.dev1511+g555967922` — the parity-pin block's identity, verified) as a ROCm wheel inside `rocm/vllm-dev:base` for `gfx1100` (`vllm-...rocm723-cp312` wheel, 2,696 files incl. `_rocm_C` ABI), commit the image (`vllm-rocm-oracle:555967922-gfx1100`, recipe in `.agents/specs/rocm-m4-oracle.md`), and capture the Qwen3-0.6B goldens FROM THAT ORACLE on this box. The gate test's existing Metal/Tenstorrent device-golden lane gains the `kROCM` arm (`our_ids_rocm.npy` + `neartie_gap_mnats_rocm.npy` + the `*_rocm` greedy/evidence pair); base dgx goldens byte-untouched. | **Oracle = upstream vLLM AT THE PIN ITSELF** — not a port: `scripts/qwen3-oracle-capture.py` + `scripts/qwen3-neartie-gap.py` run against the pinned wheel in the container; the gate lane mirrors the Metal M3b precedent (PR-set around row 632/633) file:line (`test_qwen3_paged_engine.cpp` device-golden branch). | **PASS.** Oracle K=10 per-prompt greedy: deterministic in every (prompt,pos) cell (0 multi-member). Our ROCm engine vs the same-box oracle: **16/16 prompts PASS** — STRICT token-exact 11/16 vs base greedy, 5/16 near-tie-band only, **max teacher-forced gap 0.125 nats** (prompt[3] tok1), 0 forward-divergent, 125/125 assertions; backend proof all 8 Qwen3-dense ops `selections>0 ∧ declines==0` on device 5 (kPagedAttention 7,168 selections; RoPE served via the fused-cache arm, `rope_alt` counted). The 28 token-divergent positions vs oracle greedy are all in-band; the p0 France/Italy flip is a literal 0.0000-nat tie in the oracle's OWN logits (vLLM's teacher-forced argmax on our prefix IS 15344 — the Metal M3b signature reproduced on ROCm). Hardware: 4x RX 7900 XTX gfx1100, ROCm 7.14 host, HIP 7.2.5 userland, `enforce_eager=True`, batch=1, GPU under `flock ~/gpu.lock`. Not a speed change (`benchmark_binding=false`). NEXT: Qwen3.5-0.8B (GDN) + 35B keep-quant gates under the same oracle. | diff --git a/.agents/specs/rocm-m4-oracle.md b/.agents/specs/rocm-m4-oracle.md new file mode 100644 index 000000000..70b08f408 --- /dev/null +++ b/.agents/specs/rocm-m4-oracle.md @@ -0,0 +1,74 @@ +# ROCm M4 — the pinned vLLM-ROCm oracle on gfx1100 and the ROCm near-tie gate lane + +## Goal + +Give `BACKEND-ROCM` (issue #41) the milestone that only this box can host: the M4 +correctness gate — our ROCm paged engine held against a **pinned vLLM-ROCm oracle +running on the same gfx1100 hardware**, not against the dgx (CUDA) capture. + +## The oracle + +- **Pin:** upstream vLLM commit `5559679229bc961848b121ccdeaa8fa5d79bec98` + (runtime identity `0.23.1rc1.dev1511+g555967922`, verified against the + parity-pin block in `.agents/upstream-sync.md`). +- **Base image:** `rocm/vllm-dev:base` (HIP 7.2.5 userland, host ROCm 7.14 + driver). `PYTORCH_ROCM_ARCH=gfx1100` covers the 7900 XTX. +- **Build:** `python3 setup.py bdist_wheel` inside the container after + `apt-get install binutils mold` (the base image ships no linker) and the + pyproject build-system requirements. Wheel: + `vllm-0.23.1rc1.dev1511+g555967922.rocm723-cp312-cp312-linux_x86_64.whl` + (2,696 files; `_C`/`_rocm_C`/`_moe_C` ABI extensions present). +- **Committed image:** `vllm-rocm-oracle:555967922-gfx1100` + (`/home/vikash/oracle/Dockerfile.oracle`), wheel installed with deps. +- **Determinism:** K=10 per-prompt greedy capture on the 16 gate prompts is + **deterministic in every cell** (0 multi-member (prompt,pos) cells) — a + well-posed strict gate on this board, matching the dgx finding for batch=1 + `enforce_eager=True`. + +## The gate lane + +`tests/parity/test_qwen3_paged_engine.cpp` already has device-aware goldens for +Metal and Tenstorrent (`our_ids_.npy` + `neartie_gap_mnats_.npy` under +the shared `qwen3_greedy_0_6b/` dir, with the dgx CUDA pair as base). This spec +adds `kROCM` to that lane, unchanged logic: + +- hard anchor REQUIRE (our deterministic tokens vs the committed ROCm anchor), +- near-tie band ≤ 500 milli-nats (vLLM teacher-forced on OUR exact prefix), +- backend proof (all 8 Qwen3-dense ops `selections>0 ∧ declines==0` on kROCM, + with the fused-RoPE alternative counted as in the existing lane). + +New committed goldens (captured on gfx1100, 4x RX 7900 XTX, ROCm 7.14, +`enforce_eager=True`, batch=1): + +| file | content | +|---|---| +| `our_ids_rocm.npy` | our engine's greedy tokens (16×16) | +| `neartie_gap_mnats_rocm.npy` | oracle teacher-forced gaps in milli-nats (16×16) | +| `greedy_ids_rocm.npy` | the ROCm oracle's own greedy (evidence; the base `greedy_ids.npy` stays the dgx capture) | +| `greedy_dist_rocm.npy` | K=10 run set (evidence) | + +Flow (the three commands, all GPU-locked): +1. oracle capture: `scripts/qwen3-oracle-capture.py --runs 10 --per-prompt` in the + committed container against `/models` (Qwen3-0.6B). +2. bootstrap dump: `VT_DUMP_IDS=1 ./build-hip/tests/test_qwen3_paged_engine` + → `our_ids_rocm.i32`. +3. teacher-forced gaps: `scripts/qwen3-neartie-gap.py --golden-dir ...` in the + container → `our_ids_rocm.npy` + `neartie_gap_mnats_rocm.npy`. + +## Result (Qwen3-0.6B, gfx1100) + +**16/16 prompts PASS** — STRICT token-exact 11/16 vs the base greedy, 5/16 via +the near-tie band, **max gap 0.125 nats** (prompt[3] tok=1), **0 +forward-divergent**; backend proof: kPagedAttention selections 7,168, 0 declines; +125/125 assertions. The 28 token-divergent positions vs the oracle's own greedy +are all within the band; the known p0 France/Italy flip is a literal 0.0000-nat +tie in the oracle's own logits. + +## Boundaries + +- No source/kernel change; the only edited file is the parity gate test. +- The dgx base goldens are byte-untouched; the ROCm additions are purely additive. +- Qwen3-4B (the strict deterministic dense) is not on this box (disk); the 0.6B + near-tie-robust gate is the M4 evidence for the lane. +- The oracle image and `/home/vikash/oracle/` scratch are machine-local, not + committed; this spec is the reproduction recipe. diff --git a/docs/STATUS.md b/docs/STATUS.md index 4693e59e0..99c2e9874 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -476,7 +476,12 @@ recurrences + fused attn preamble; 27B prefill 21.5x, decode [campaign](../.agents/specs/vulkan-full-support.md)), ROCm (W0 community-green on 5 gfx archs; the APU unified-memory fix remains unverified; gfx1200 runs Gemma-3 and Qwen3 all-native, with Gemma-3 strict 48/48 against two vLLM-ROCm -oracles and Qwen3 in a measured near-tie regime; Qwen3.5-0.8B GDN runs all-native +oracles and Qwen3 in a measured near-tie regime; on gfx1100 the M4 gate now +runs against the **pinned vLLM-ROCm oracle built on the same box** +(`555967922` / `0.23.1rc1.dev1511+g555967922`): Qwen3-0.6B **16/16 PASS** +(11/16 strict token-exact, 5/16 near-tie band, max teacher-forced gap 0.125 +nats, 0 forward-divergent; oracle K=10 deterministic in every cell) with the +ROCm device-golden lane in `test_qwen3_paged_engine`; Qwen3.5-0.8B GDN runs all-native but its CPU/ROCm divergence remains open; gfx1201 Gemma-4 FP8 MoE is contributor-measured on 2x R9700 and CPU-link-verified our side; [guide](ROCM.md)), inference-time CPU weight offload (`ENG-WEIGHT-OFFLOAD` diff --git a/tests/parity/goldens/qwen3_greedy_0_6b/greedy_dist_rocm.npy b/tests/parity/goldens/qwen3_greedy_0_6b/greedy_dist_rocm.npy new file mode 100644 index 0000000000000000000000000000000000000000..ed0bf47c4c68b9225479ac3882cee0b840679bd5 GIT binary patch literal 10368 zcmeI2OGuPq6oy|bEltZpGg4#%V=M>K@KQOT6hs?KqQIaFlTJ4*Z6-trHxVvdDxE#op3bDwr$-8x3%QFYM+!sv_&&G7(YiI0 znn?Y}Uq#P$iBKhp(7Q|`?Aas{x=SR&tt}E^%-=`oUg32Cb^bmAu{AWr*3ghOL(@^q zpz9JJhlb=zXd3KeLb-K>ul`QLdg};lty>J{Z7-qKF($0Cj&Nh6M4<5{wIDRdN*Ody ziSI%~&(UO7BJ8!F33T0S-Xq~er9^n_{Sj*FB|_G*BT#!kxpoj}9?*4(k3-X7KNINr zp!<7zQzFoH>A56phKBUi(2yDwn)Xfx-QSX57S5ly?r(%L$Aoa#IzoHI7;wS$g+O|K zXlO70M@ZTZ65G&_*oJ1;ItGbtXsEr_-Xnp=m+C%`O9Wb9<}D=9dP@7p zV3R~3vt(%Q{A7?>C^W|hOAu2lCBmG0HKFOFL|Egob`!pX&x;z-N8t2t!_%K-aBxpC;V%x&*rJ&O;J`&XZL=z5@iz z9+U{*S|kFkS6^Bs!YlikK!8i76^ Sy>K27=>D>PuR%z9j{OT7m`(El literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen3_greedy_0_6b/greedy_ids_rocm.npy b/tests/parity/goldens/qwen3_greedy_0_6b/greedy_ids_rocm.npy new file mode 100644 index 0000000000000000000000000000000000000000..b8b49ca47c010222660f064ced9f6847431ea3c5 GIT binary patch literal 1152 zcmbVLO-NKx7(J6|YEGH>6Djfn<5@n?31_SvPz<7tC6ZuJ#%pHs2mhK85yDM`%NDta zRzi_bCI~^$rlO!ttrDb~T8P`iU9<|%;l9h8R)PZ`@4MeU-_JSso5f25y?s|Bn8h78 zoyn!L?m5Rzj<&m=&crL?uZp5F?@Z20rcup*F(sSmWJ^sqO zT7f8l?tMVdF`zR7JURi)^4qx|sN>hh<;QD6jPe?YDWm0_XXae~+=D3SgCfbv=;eT#A3?Oq$JN8i-MUonJl-Ys=T{>e-&y{q7@c zlo0z=c-jQ}iHmsrD$X*gRG?*(#`h0%fWVw>%Niw z$CxKQPdx{o1%BUKARYKM#6`<{y=tbE-m=L2_BIf}jd0*tescrH;7tDo*(5FaJ}b=>gL zUfX@!D5K`TOrCwW$K)VA?YWk`>S!$cUtR6x66dlz*l`HB!WrDpA+DKqtoGo3TC87U zFqH-O)j=)o=`gXMNJ9EwWFK4OH+7v3{%_(|d>qh!Mz}xZrP%3tb1vMsya#mfQE&5J Rm_hQEo^OJjBzMnhvIARbhW!8l literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen3_greedy_0_6b/neartie_gap_mnats_rocm.npy b/tests/parity/goldens/qwen3_greedy_0_6b/neartie_gap_mnats_rocm.npy new file mode 100644 index 0000000000000000000000000000000000000000..a845e0408a76e8747e6eeb0e66969a5a58a3b67a GIT binary patch literal 1152 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= wXCxM+0{I$-W;zOnW|}$*wF+bcE{0(MYKMUzM!kSP1c;4eV%6cdZRE%T0GK`<4*&oF literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen3_greedy_0_6b/our_ids_rocm.npy b/tests/parity/goldens/qwen3_greedy_0_6b/our_ids_rocm.npy new file mode 100644 index 0000000000000000000000000000000000000000..cff63049206332bc46aef4eb9339071199be10df GIT binary patch literal 1152 zcmbVLK}Zx~6#lkS)7;X~tQ48R7_E)m&{cCmDGD!^h=PeIle?8sX*(i9c!`is9rVyq zDhjfKpi`F;4_@jPEb8VVJO`enlk|Q3^D`?Qgcct2=DqK|_q}i4&R^^A?zs`c6dt*$ zv_G10FFI~=tjUcz?yd36WX5|iGM-7L6~F4;^V5v`cf5%--|ORvm=jMl#GD!DKYt}1 z=Yc4I&V9h8lR$d}*lGu+_{{eGn>%sE|aeCrK$Q#H$A&KYk)n~L%P|yk4bE=xbn4} zf0uRA^UNafHt>7C0;#~ST4$%%EcOBR`Yv!&u0!ze*Q=Izfmb{z6w1n^?G454GV`^7l-Q@y#X(!ETISll1C--}Zt7qJ558kK6 zdYmneWqJ9EpqIGaCiWWv(!YyyY>(g6b)qutQ*a8runner().device().type; const bool metal = run_dev == vt::DeviceType::kMETAL; + const bool rocm = run_dev == vt::DeviceType::kROCM; const bool tenstorrent = run_dev == vt::DeviceType::kTENSTORRENT; - const bool device_golden = metal || tenstorrent; + const bool device_golden = metal || tenstorrent || rocm; // The forward + greedy ops Qwen3-dense dispatches on the DEFAULT // (VT_QWEN3_ROPE_CACHE) path. kRopeCosSinCache + kRopeFromCache are the M3b // additions (build the per-step cos|sin cache, then apply it); the rest are @@ -252,7 +253,7 @@ void RunGate(const std::string& repo_dir, const std::string& golden_subdir, } vt::EnableOpProviderCallStats(true); MESSAGE(label << ": running on device type " << static_cast(run_dev) - << " (2=METAL, 6=TENSTORRENT) — gated against this device's OWN " + << " (2=METAL, 5=ROCM, 6=TENSTORRENT) — gated against this device's OWN " "oracle-backed golden"); } @@ -261,9 +262,13 @@ void RunGate(const std::string& repo_dir, const std::string& golden_subdir, const int32_t* anchor_ids = od; // hard anchor for THIS device const int32_t* gap_ids = gapd; // vLLM teacher-forced gaps for THIS device parity::NpyArray o_dev, gap_dev; // keep device arrays alive for the loop - const char* ids_name = metal ? "our_ids_metal.npy" : "our_ids_tenstorrent.npy"; + const char* ids_name = + metal ? "our_ids_metal.npy" + : (rocm ? "our_ids_rocm.npy" : "our_ids_tenstorrent.npy"); const char* gap_name = - metal ? "neartie_gap_mnats_metal.npy" : "neartie_gap_mnats_tenstorrent.npy"; + metal ? "neartie_gap_mnats_metal.npy" + : (rocm ? "neartie_gap_mnats_rocm.npy" + : "neartie_gap_mnats_tenstorrent.npy"); bool bootstrap_only = false; if (device_golden) { const bool have_dev = fs::exists(gdir / ids_name) && fs::exists(gdir / gap_name); @@ -272,7 +277,7 @@ void RunGate(const std::string& repo_dir, const std::string& golden_subdir, // qwen3-neartie-gap.py then teacher-forces vLLM on that sequence. bootstrap_only = true; MESSAGE(label << ": BOOTSTRAP dump (device golden absent) for " - << (metal ? "Metal" : "Tenstorrent") << "..."); + << (metal ? "Metal" : (rocm ? "ROCm" : "Tenstorrent")) << "..."); } else { REQUIRE_MESSAGE(have_dev, label << ": device oracle golden absent (" << ids_name << " / " @@ -402,7 +407,8 @@ void RunGate(const std::string& repo_dir, const std::string& golden_subdir, if (dump) { const std::string dump_name = tenstorrent ? "our_ids_tenstorrent.i32" - : (metal ? "our_ids_metal.i32" : "our_ids.i32"); + : (metal ? "our_ids_metal.i32" + : (rocm ? "our_ids_rocm.i32" : "our_ids.i32")); const std::string path = (gdir / dump_name).string(); std::FILE* f = std::fopen(path.c_str(), "wb"); if (f != nullptr) { From a139f346a0d1bddbe402944416c49cce1808ae39 Mon Sep 17 00:00:00 2001 From: Vikash Loomba Date: Fri, 14 Aug 2026 00:22:25 -0700 Subject: [PATCH 2/2] fix(ROCM): AttnQkNormRopeGate dispatch keyed on the OUTPUT dtype -- the 0.8B GDN forward divergence, root-caused and fixed Issue #41. The ROCm dispatcher for the fused full-attention preamble keyed its template on the SOURCE dtype (qgate.dtype) instead of the OUTPUT dtype like the CUDA lane (LaunchAttnPreambleOut switches on q_out.dtype). The 0.8B bf16 model runs bf16 QKV-projection output + f32 q/k/gate outs (the f32-attention path; no FA-2 on ROCm), which the src-keyed dispatch mis-launched as all-bf16 -- writing bf16 bits through the f32 out pointers. Evidence chain (gfx1100, flock): CPU==pinned-oracle token-exact / ROCm diverges at tok0; VT_DUMP_ACT per-layer residual dumps: layers 0-2 clean (<=9.6e-3), layer 3 (first full-attn) 2.1e-1; sub-stage dumps: block_out 4.4e-1 with post_input_norm clean; VT_DUMP_ATTN op dumps: fa0_q rms-rel 1.196 with qkv/gate clean; VT_ROCM_ATTN_CPU_REF=1 unchanged. Fix: dispatch on q_out.dtype; bf16-out requires bf16-src (VT_CHECK). Teeth: the cross-device AttnQkNormRopeGate case gains the bf16-src -> f32-out combo at the real 0.8B dims (Hq=8, Hkv=2, Dh=256, rot=64); mutation-proven (fails with the fix reverted, passes with it). Gates on this branch (gfx1100, flock ~/gpu.lock): test_backend_cross_device 19/19 (346 assertions, incl. the new combo); test_qwen3_paged_engine 16/16 unchanged; e2e 0.8B 'The capital of France is' -> ' Paris.\nThe capital of France is' == pinned oracle == our CPU. The M4 0.8B gate with the re-derived GREEN goldens lands in the stacked sibling on row/ROCM-M4-GDN-GATE; full HIP ctest zero-delta vs base measured there. Also: three pre-existing hipcc/clang-only build breaks that block ANY full HIP build on this toolchain: same-line double CAPTURE in test_qwen3_5_gdn_spec_routing.cpp and test_ops_quant_repack.cpp, and -Wunused-const-variable on kGdnLayers (uses compile out on HIP). Debug instrumentation env-gated and off by default (VT_DUMP_ACT / VT_DUMP_ACT_SUB / VT_DUMP_ATTN; deepseek_v4 VT_DUMP_ACT precedent; env-doc-allowlist updated). FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: pi:kimi-k3 [pi] --- .agents/specs/rocm-gdn-08b-fix.md | 98 +++++++++++++++++++ docs/FEATURES.md | 2 +- docs/STATUS.md | 3 +- scripts/env-doc-allowlist.txt | 2 + src/vllm/model_executor/models/qwen3_5.cpp | 67 +++++++++++++ src/vt/rocm/rocm_gdn_fused.hip | 22 ++++- .../test_qwen27n_fp8_tower_paged_engine.cpp | 5 +- .../models/test_qwen3_5_gdn_spec_routing.cpp | 7 +- tests/vt/test_backend_cross_device.cpp | 61 ++++++++++++ tests/vt/test_ops_quant_repack.cpp | 9 +- 10 files changed, 265 insertions(+), 11 deletions(-) create mode 100644 .agents/specs/rocm-gdn-08b-fix.md diff --git a/.agents/specs/rocm-gdn-08b-fix.md b/.agents/specs/rocm-gdn-08b-fix.md new file mode 100644 index 000000000..1078b7eac --- /dev/null +++ b/.agents/specs/rocm-gdn-08b-fix.md @@ -0,0 +1,98 @@ +# ROCm Qwen3.5-0.8B forward-divergence fix — the RED gate's target + +## The problem (oracle-measured) + +The M4 gate (`test_qwen35_paged_engine`, #559) FAILS 13/16 prompts against the +pinned vLLM-ROCm oracle on gfx1100, with **first-token divergences on 6 prompts** +(gaps 0.375–1.188 nats, over the 0.5-nat band) and max gap 14.125 nats deep in a +diverged continuation. This is a REAL forward divergence, not a bf16 near-tie. + +## The evidence (all measurements on gfx1100, GPU-locked) + +1. **The split is ROCm-kernel-specific, not model math.** For prompt + `The capital of France is` (p0): + - pinned oracle (vLLM `555967922`, enforce_eager): argmax **11751 " Paris"** (-1.9957), token 25 ":" at -2.8082 (#4) + - **our CPU backend: matches the oracle exactly** (` Paris.\nThe capital of France is`) + - our ROCm backend: argmax **25 ":"** (-1.6535), 11751 at -3.0910 (#9) + The candidate SETS are the same with similar spread; the rankings shift ~1.1 + nats in opposite directions — a systematic numerical offset in some block + output, not a distribution reshape. +2. **Backend proof is green**: all 15 dispatched GDN/full-attn ops run natively + on device 5 with 0 declines (kPagedAttention 1,536, kGdnDecode 4,320) — the + divergence is numerical, not dispatch. +3. **Not the ROCm paged-attention kernel**: `VT_ROCM_ATTN_CPU_REF=1` (the CPU-ref + attention path) produces the SAME divergent tokens. +4. **GDN ops were cleared in isolation**: the merged cross-device cases + (tests/vt/test_backend_cross_device.cpp) pass at real dims vs the CPU oracle. +5. **The 0.6B dense gate passes 16/16** with the same ROCm attention/GEMM/RMSNorm + kernels — so the diverging op is exercised by 0.8B's GDN hybrid path but not + (or differently) by dense 0.6B. +6. Prior characterization (issue #41, op-level): attention-block localized + (layer-3 attn block 20% gated / 4.4% raw mismatch); GDN ops + op 67 + op 66 + + paged-attention + GEMM all cleared at real dims; conclusion then was + "bf16-softmax amplification" — but (3) refutes the paged-attention softmax as + the sole source, so the amplification must be fed by an earlier drift. + +## The plan (W-plan) + +- **W1 — instrument.** Add `VT_DUMP_ACT`-style per-layer hidden-state dumps to + `qwen3_5.cpp` (the deepseek_v4.cpp precedent), plus a scratch host comparator + (CPU vs ROCm, per-layer max-abs/rel delta at the residual stream) in + `/tmp/vllm-diag` (never committed). +- **W2 — localize.** Find the first layer+op where the ROCm hidden state drifts + past ~1e-3 rel vs CPU on the p0 prefill. Suspects in order: the GDN + prefill/decode recurrence IN CONTEXT (state carry across the chunk boundary), + the fused preamble `kAttnQkNormRopeGate` at 0.8B geometry, the conv1d state + handoff, the GDN output gated RMSNorm. +- **W3 — fix.** The minimal kernel fix at the localized op (portable-math + corrections first; e.g. the reduction-order or accumulation-dtype issue the + in-isolation tests couldn't see). +- **W4 — gate GREEN.** `test_qwen35_paged_engine` 16/16 (strict + band), the + 0.6B gate re-run green, focused + full gates, fresh review, operator gate. + +## RESULT (W1–W4 complete — fix verified; the gate lands GREEN stacked on top) + +**Stack shape (post-review):** this branch carries ONLY the kernel fix + its +teeth + the debug instrumentation. The M4 0.8B gate with the GREEN (fixed-engine, +oracle-re-derived) goldens lands in the stacked sibling commit on +`row/ROCM-M4-GDN-GATE` — so every commit in the stack is individually green on +the gate host, and the gate never exists in a state whose goldens its own tree +cannot reproduce. + +**Root cause:** `AttnQkNormRopeGateKernelRocm` (src/vt/rocm/rocm_gdn_fused.hip) +dispatched its template on the SOURCE dtype (`qgate.dtype`) instead of the +OUTPUT dtype like the CUDA lane (`LaunchAttnPreambleOut` switches on +`q_out.dtype`). The 0.8B bf16 model runs bf16 QKV-projection output + f32 +q/k/gate outs (the f32-attention path — no FA-2 on ROCm), which the src-keyed +dispatch silently mis-launched as all-bf16: the kernel wrote bf16 bits through +the f32 out pointers. The cross-device test covered only f32-src, so the bug +class was invisible in isolation; the in-context logits comparison (W2) showed +`fa0_q` rms-rel 1.196 with `fa0_qkv`/`fa0_gate` clean, localizing it to the +preamble's q/k path. + +**Fix:** dispatch on the output dtype (bf16 src + f32 out is now a first-class +combo); VT_CHECK bf16-out requires bf16-src. + +**Teeth:** the cross-device AttnQkNormRopeGate case gains the bf16-src/f32-out +combo at the real 0.8B dims (Hq=8, Hkv=2, Dh=256, rot=64) — FAILS with the fix +reverted (mutation-proven), passes with it. + +**Gates (gfx1100, flock):** +- `test_qwen35_paged_engine` (the RED anchor): **16/16 PASS — 15/16 strict + token-exact vs the pinned oracle, 1/16 near-tie band, max gap 0.125 nats, + 0 forward-divergent** (was 3/16, 13 divergent, max 14.125 nats) +- `test_qwen3_paged_engine` (0.6B, regression): 16/16 PASS unchanged +- `test_backend_cross_device`: 19/19, 346 assertions +- e2e: `The capital of France is` → ` Paris.\nThe capital of France is`, + matching the pinned oracle AND our CPU backend token-for-token + +**Also fixed (pre-existing main bug, blocked the full HIP build):** +same-line double `CAPTURE` in test_qwen3_5_gdn_spec_routing.cpp:357 (doctest +redefinition that clang/hipcc rejects). + +## Boundaries + +- The RED gate stays RED until W4; no gate-weakening (no band-widening, no + goldens recapture against a "fixed" engine without the oracle re-derivation). +- ROCm-only files + the additive dump hook; no shared-path behavior change. +- Rollback: the dump hook is env-gated (off by default). diff --git a/docs/FEATURES.md b/docs/FEATURES.md index b16d6b38b..29297c1a0 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -245,7 +245,7 @@ both refuse, naming what is missing. | CPU (x86, Arm i8mm; A76 assembly correct/default, llama speed gate open) | ✅ | ◐ | ☐ | ✅ | | Metal (Apple Silicon) | ✅ | ☐ | ☐ | ✅ | | Vulkan | ◐ | ☐ | ☐ | ✅ | -| ROCm | W0 verified on 5 gfx archs; dense and GDN models run all-native. Strict CPU parity is open in the measured near-tie regime (#269) | 44 registered ops including full GDN; ctest-green gfx1151/1103/1100/1201/1200 ([#41](https://github.com/mudler/vllm.cpp/issues/41)). APU managed allocation is unverified. [ROCM.md](ROCM.md) | ✅ | ✅ | +| ROCm | W0 verified on 5 gfx archs; dense and GDN models run all-native; the 0.8B GDN CPU/ROCm divergence is fixed (AttnQkNormRopeGate dispatch; the oracle gate stacks above) | 44 registered ops including full GDN; ctest-green gfx1151/1103/1100/1201/1200 ([#41](https://github.com/mudler/vllm.cpp/issues/41)). APU managed allocation is unverified. [ROCM.md](ROCM.md) | ✅ | ✅ | | XPU / TPU | ☐ | ✅ | ◐ | ☐ | | Tenstorrent Blackhole | ◐ `ACTIVE`, OPT-125m 6/6; Qwen3-0.6B wired; Mistral-7B-v0.3 16/16 on P150 ([spec](../.agents/specs/tenstorrent-mistral.md)). 16x16 rerun and residual-RMS owed ([spec](../.agents/specs/tenstorrent-backend.md)) | ✅ | ☐ | ☐ | diff --git a/docs/STATUS.md b/docs/STATUS.md index 99c2e9874..0bcf9d55c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -482,7 +482,8 @@ runs against the **pinned vLLM-ROCm oracle built on the same box** (11/16 strict token-exact, 5/16 near-tie band, max teacher-forced gap 0.125 nats, 0 forward-divergent; oracle K=10 deterministic in every cell) with the ROCm device-golden lane in `test_qwen3_paged_engine`; Qwen3.5-0.8B GDN runs all-native -but its CPU/ROCm divergence remains open; gfx1201 Gemma-4 FP8 MoE is +and its CPU/ROCm divergence is **fixed** (`AttnQkNormRopeGate` output-dtype dispatch; +oracle gate quantifying it stacks above); gfx1201 Gemma-4 FP8 MoE is contributor-measured on 2x R9700 and CPU-link-verified our side; [guide](ROCM.md)), inference-time CPU weight offload (`ENG-WEIGHT-OFFLOAD` READY, spec only: vLLM's `cpu_offload_gb` UVA arm with dotted-segment diff --git a/scripts/env-doc-allowlist.txt b/scripts/env-doc-allowlist.txt index 36cab9ceb..2db904d96 100644 --- a/scripts/env-doc-allowlist.txt +++ b/scripts/env-doc-allowlist.txt @@ -219,3 +219,5 @@ VT_W4A4_TRUE # selects a kernel or a batching strategy for the expert GEMMs and each # treats a name here as a known kernel-internal knob. A NEW production env # var must be documented in docs/ENVIRONMENT.md OR added here. +VT_DUMP_ACT_SUB +VT_DUMP_ATTN diff --git a/src/vllm/model_executor/models/qwen3_5.cpp b/src/vllm/model_executor/models/qwen3_5.cpp index 30dd2c7ee..72d4f7135 100644 --- a/src/vllm/model_executor/models/qwen3_5.cpp +++ b/src/vllm/model_executor/models/qwen3_5.cpp @@ -5126,6 +5126,34 @@ DBuf FullAttnBlockPaged(Dev d, const FullAttnLayerWeights& w, const HfConfig& cf pa_args.max_seq_len = meta.max_seq_len; vt::PagedAttention(d.q, dattn.t(), qn3, k_cache, v_cache, dblk, dsl, dqsl, pa_args); + // VT_DUMP_ATTN (issue #41, 0.8B ROCm divergence spike W1/W2): dump the + // full-attn block's internals per full-attn-layer call index (0-based), as + // raw little-endian dumps under $VT_DUMP_ATTN/fa_{qkv,q,attn,gate}.bin. + // Inert when unset; the Downloads sync, so never set on a graph path. + static thread_local int64_t dump_fa_idx = -1; + dump_fa_idx++; + if (std::getenv("VT_DUMP_ATTN") != nullptr) { + auto DumpT = [&](const char* stage, const Tensor& t) { + int64_t n = 1; + for (int i = 0; i < t.rank; ++i) n *= t.shape[i]; + const size_t es = vt::SizeOf(t.dtype); + // contiguous check: innermost stride 1 and packed + std::vector raw(static_cast(n) * es); + DBuf tmp(d, t.dtype, {n}); + d.b.Copy(d.q, tmp.ptr(), t.data, raw.size()); + tmp.Download(d, raw.data()); + const std::string path = std::string(std::getenv("VT_DUMP_ATTN")) + + "/fa" + std::to_string(dump_fa_idx) + "_" + + stage + ".bin"; + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f != nullptr) { std::fwrite(raw.data(), 1, raw.size(), f); std::fclose(f); } + }; + DumpT("qkv", qgate); + DumpT("q", qn3); + DumpT("attn", dattn.t()); + DumpT("gate", gatef.t()); + } + // Sigmoid output gate, folded into the o_proj activation quant on the true-W4A4 // path (§5) — see SigmoidGateOProjD. return SigmoidGateOProjD(d, Reshape(dattn.t(), {T, Hq * Dh}), @@ -6544,9 +6572,27 @@ void RunDenseLayerPaged(Dev d, const Qwen3_5DenseLayerWeights& layer, const int64_t H = cfg.hidden_size; const float eps = static_cast(cfg.rms_norm_eps); + // VT_DUMP_ACT layer index: the paged loop's per-layer counter (0-based), used + // by the sub-stage dump below. Declared per-process; only the dump reads it. + static thread_local int64_t dump_layer_idx = -1; + dump_layer_idx++; + const bool dump_sub = std::getenv("VT_DUMP_ACT_SUB") != nullptr; + auto DumpStage = [&](const char* stage, DBuf& buf) { + if (!dump_sub) return; + std::vector raw(static_cast(T) * static_cast(H) * + vt::SizeOf(buf.t().dtype)); + buf.Download(d, raw.data()); + const std::string path = std::string(std::getenv("VT_DUMP_ACT_SUB")) + + "/layer_" + std::to_string(dump_layer_idx) + "_" + + stage + ".bin"; + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f != nullptr) { std::fwrite(raw.data(), 1, raw.size(), f); std::fclose(f); } + }; + Tensor dw_in = ResidentWeight(d, layer.input_layernorm, {H}); DBuf dhn(d, DType::kBF16, {T, H}); vt::RmsNorm(d.q, dhn.t(), hidden.t(), dw_in, vt::RmsNormArgs{eps, true}, &res.t()); + DumpStage("post_input_norm", dhn); DBuf attn = [&] { if (layer.is_linear_attention) { @@ -6559,12 +6605,15 @@ void RunDenseLayerPaged(Dev d, const Qwen3_5DenseLayerWeights& layer, return FullAttnBlockPaged(d, layer.attn, cfg, dhn.t(), sdi, attn_meta, *attn_kv, T); }(); + DumpStage("block_out", attn); Tensor dw_post = ResidentWeight(d, layer.post_attention_layernorm, {H}); DBuf dh2(d, DType::kBF16, {T, H}); vt::RmsNorm(d.q, dh2.t(), attn.t(), dw_post, vt::RmsNormArgs{eps, true}, &res.t()); + DumpStage("post_attn_norm", dh2); hidden = DenseMlpBlock(d, layer.mlp, cfg, dh2.t(), T); + DumpStage("mlp_out", hidden); } // ── Qwen3.5/3.6 MTP head shared preamble (SPEC-MTP I5c). ──────────────────── @@ -7892,6 +7941,24 @@ static DBuf DenseForwardLayers(Dev d, const Tensor& hidden_in, // DFlash DF-AUX-TAPS: capture (hidden+res) at configured boundaries. Inert // (no-op) when aux_out is null — every non-DFlash caller. MaybeCaptureAuxTap(d, l, aux_layer_ids, aux_out, hidden.t(), res.t(), T, H); + // VT_DUMP_ACT (issue #41, ROCm 0.8B forward-divergence fix spike W1): dump + // the residual stream after each layer as raw little-endian bf16 to + // $VT_DUMP_ACT/layer_.bin (inert when unset; a debug hook, never the + // hot path — the Download forces a sync, so capture-graph paths must not + // set the env). + if (std::getenv("VT_DUMP_ACT") != nullptr) { + std::vector raw(static_cast(T) * static_cast(H) * + vt::SizeOf(hidden.t().dtype)); + hidden.Download(d, raw.data()); // Copy + Synchronize + const char* dir = std::getenv("VT_DUMP_ACT"); + const std::string path = + std::string(dir) + "/layer_" + std::to_string(l) + ".bin"; + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f != nullptr) { + std::fwrite(raw.data(), 1, raw.size(), f); + std::fclose(f); + } + } } // Final RMSNorm over the fused stream (res += hidden; norm), then lm_head. diff --git a/src/vt/rocm/rocm_gdn_fused.hip b/src/vt/rocm/rocm_gdn_fused.hip index 554912752..9e5c60f7d 100644 --- a/src/vt/rocm/rocm_gdn_fused.hip +++ b/src/vt/rocm/rocm_gdn_fused.hip @@ -250,16 +250,28 @@ void AttnQkNormRopeGateKernelRocm(Queue& q, Tensor& q_out, Tensor& k_out, Tensor kf.Ptr(), q_norm.Ptr(), k_norm.Ptr(), cos_sin.Ptr(), t, hq, hkv, dh, qgate.stride[0], kf.stride[0], ra.rotary_dim, na.eps, na.gemma); }; - // Validated combos (mirror the CUDA lane): (f32 src, f32 out); (bf16 src, - // bf16 out + bf16 gate); (bf16 src, bf16 out + f32 gate). - if (qgate.dtype == DType::kBF16) { + // Dispatch on the OUTPUT dtype like the CUDA lane (cuda_ops.cu + // LaunchAttnPreambleOut): the caller picks f32 out (token-exact path) or bf16 + // out (FA-2 combos) INDEPENDENTLY of the source dtype — the 0.8B bf16 model + // runs bf16 src + f32 q/k out + f32 gate on the f32-attention path, which the + // old src-keyed dispatch silently mis-launched as all-bf16 (writing bf16 bits + // through the f32 out pointer; issue #41 M4 W2 finding). Validated combos + // (mirror CUDA): (f32 src, f32 out); (bf16 src, f32 out + f32 gate); + // (bf16 src, bf16 out + bf16 gate); (bf16 src, bf16 out + f32 gate). + const bool src_bf16 = qgate.dtype == DType::kBF16; + if (q_out.dtype == DType::kF32) { + if (src_bf16) { + launch(__hip_bfloat16{}, float{}, float{}); + } else { + launch(float{}, float{}, float{}); + } + } else { + VT_CHECK(src_bf16, "rocm attn_qk_norm_rope_gate: bf16 out needs bf16 src"); if (gate_out.dtype == DType::kF32) { launch(__hip_bfloat16{}, __hip_bfloat16{}, float{}); } else { launch(__hip_bfloat16{}, __hip_bfloat16{}, __hip_bfloat16{}); } - } else { - launch(float{}, float{}, float{}); } Check(hipGetLastError(), "attn_qk_norm_rope_gate launch"); } diff --git a/tests/parity/test_qwen27n_fp8_tower_paged_engine.cpp b/tests/parity/test_qwen27n_fp8_tower_paged_engine.cpp index cf5cabcdc..6a619864d 100644 --- a/tests/parity/test_qwen27n_fp8_tower_paged_engine.cpp +++ b/tests/parity/test_qwen27n_fp8_tower_paged_engine.cpp @@ -92,7 +92,10 @@ namespace { // Qwen3.6-27B has 48 GDN (`linear_attention`) layers. One merged FP8 qkvz GEMM // per layer on the default arm; two split GEMMs per layer on the // VT_GDN_MERGED_QKVZ_FP8=0 rollback. -constexpr uint64_t kGdnLayers = 48; +// [[maybe_unused]]: the uses below live behind the CUDA-only fp8-tower arms; +// on a HIP-only build they compile out and the constant would be an +// -Wunused-const-variable error under clang/hipcc. +[[maybe_unused]] constexpr uint64_t kGdnLayers = 48; // Snapshot dir of the FP8-tower 27B, or "" to refuse. Pinned to the revision // its goldens were captured against; a cache holding some other revision of the diff --git a/tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp b/tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp index e0fe74185..9d5cbe19e 100644 --- a/tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp +++ b/tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp @@ -363,7 +363,12 @@ void RunMixedRoutingCase(vt::DeviceType dev, const GdnDims& g, bool bit_exact) { INFO("dims := ", std::string(g.name)); CAPTURE(bad); CAPTURE(first); - if (bad != 0) { CAPTURE(mixed_out[first]); CAPTURE(ref[first]); } + if (bad != 0) { + // One CAPTURE per line: same-line pairs collide on DOCTEST_CAPTURE_ + // (clang/hipcc reject the redefinition; GCC happened not to). + CAPTURE(mixed_out[first]); + CAPTURE(ref[first]); + } CHECK(bad == 0); } else { float maxabs = 0.0f; diff --git a/tests/vt/test_backend_cross_device.cpp b/tests/vt/test_backend_cross_device.cpp index cd3820c63..e31db2fa6 100644 --- a/tests/vt/test_backend_cross_device.cpp +++ b/tests/vt/test_backend_cross_device.cpp @@ -1792,6 +1792,67 @@ TEST_CASE("AttnQkNormRopeGate matches the CPU oracle within NMSE <= 5e-4") { } } + // The in-context production mix (issue #41 M4 W2): the 0.8B bf16 model feeds + // the preamble a BF16 projection output but wants F32 q/k/gate out (the f32 + // attention path — FA-2 is off on ROCm). The ROCm dispatcher once keyed on + // the SOURCE dtype and mis-launched all-bf16, writing bf16 bits through the + // f32 out pointers; this arm pins the (bf16 src -> f32 out) combo at the real + // 0.8B dims so the bug class cannot return silently. + { + const int64_t HQr = 8, HKVr = 2, DHr = 256, ROTr = 64; + const std::vector qg = RandomVec(static_cast(T * HQr * 2 * DHr), 991, -0.5f, 0.5f); + const std::vector kfv = RandomVec(static_cast(T * HKVr * DHr), 992, -0.5f, 0.5f); + const std::vector qnr = RandomVec(static_cast(DHr), 993, 0.2f, 1.0f); + const std::vector knr = RandomVec(static_cast(DHr), 994, 0.2f, 1.0f); + const std::vector csr = RandomVec(static_cast(T * ROTr), 995, -1.0f, 1.0f); + const std::vector qg_bf = Bf16Bits(qg), kf_bf = Bf16Bits(kfv); + vt::RmsNormArgs na3; na3.eps = 1e-6f; na3.gemma = true; + vt::RopeArgs ra3; ra3.rotary_dim = static_cast(ROTr); + // CPU reference: bf16 in (exact upcast inside the op) -> f32 out. + std::vector rq(static_cast(T * HQr * DHr)); + std::vector rk(static_cast(T * HKVr * DHr)); + std::vector rg(static_cast(T * HQr * DHr)); + { + vt::Backend& cpu = vt::GetBackend(DeviceType::kCPU); + Queue cq = cpu.CreateQueue(); + const Device cd{DeviceType::kCPU, 0}; + std::vector a = qg_bf, b = kf_bf; std::vector e = qnr, f = knr, g = csr; + Tensor tqg = Tensor::Contiguous(a.data(), DType::kBF16, cd, {T, HQr * 2 * DHr}); + Tensor tkf = Tensor::Contiguous(b.data(), DType::kBF16, cd, {T, HKVr * DHr}); + Tensor tqn = T1(e.data(), cd, DHr), tkn = T1(f.data(), cd, DHr); + Tensor tcs = T2(g.data(), cd, T, ROTr); + Tensor tqo = Tensor::Contiguous(rq.data(), DType::kF32, cd, {T, HQr, DHr}); + Tensor tko = Tensor::Contiguous(rk.data(), DType::kF32, cd, {T, HKVr, DHr}); + Tensor tgo = Tensor::Contiguous(rg.data(), DType::kF32, cd, {T, HQr, DHr}); + vt::AttnQkNormRopeGate(cq, tqo, tko, tgo, tqg, tkf, tqn, tkn, tcs, na3, ra3); + cpu.DestroyQueue(cq); + } + for (DeviceType dt : RegisteredDevices()) { + if (!OpAvailable(vt::OpId::kAttnQkNormRopeGate, dt)) continue; + CAPTURE(DeviceName(dt)); + vt::Backend& dev = vt::GetBackend(dt); + Queue q = dev.CreateQueue(); + const Device d{dt, 0}; + DevBufBytes dqg(dev, q, qg_bf.size() * 2), dkf(dev, q, kf_bf.size() * 2); + DevBuf dqn(dev, q, DHr), dkn(dev, q, DHr), dcs(dev, q, csr.size()); + DevBuf dqo(dev, q, rq.size()), dko(dev, q, rk.size()), dgo(dev, q, rg.size()); + dqg.Upload(qg_bf.data()); dkf.Upload(kf_bf.data()); + dqn.Upload(qnr); dkn.Upload(knr); dcs.Upload(csr); + Tensor tqg = Tensor::Contiguous(dqg.ptr(), DType::kBF16, d, {T, HQr * 2 * DHr}); + Tensor tkf = Tensor::Contiguous(dkf.ptr(), DType::kBF16, d, {T, HKVr * DHr}); + Tensor tqn = T1(dqn.ptr(), d, DHr), tkn = T1(dkn.ptr(), d, DHr); + Tensor tcs = T2(dcs.ptr(), d, T, ROTr); + Tensor tqo = Tensor::Contiguous(dqo.ptr(), DType::kF32, d, {T, HQr, DHr}); + Tensor tko = Tensor::Contiguous(dko.ptr(), DType::kF32, d, {T, HKVr, DHr}); + Tensor tgo = Tensor::Contiguous(dgo.ptr(), DType::kF32, d, {T, HQr, DHr}); + vt::AttnQkNormRopeGate(q, tqo, tko, tgo, tqg, tkf, tqn, tkn, tcs, na3, ra3); + CHECK(Nmse(rq, dqo.Download()) <= kNmseTol); + CHECK(Nmse(rk, dko.Download()) <= kNmseTol); + CHECK(Nmse(rg, dgo.Download()) <= kNmseTol); + dev.DestroyQueue(q); + } + } + const int64_t HQ = 3, HKV = 2, DH = 32, ROT = 16; const int64_t qg_outer = HQ * 2 * DH + 7, kf_outer = HKV * DH + 5; const std::vector qgate = RandomVec(static_cast(T * qg_outer), 881, -0.5f, 0.5f); diff --git a/tests/vt/test_ops_quant_repack.cpp b/tests/vt/test_ops_quant_repack.cpp index a1bb02b97..f3f80707f 100644 --- a/tests/vt/test_ops_quant_repack.cpp +++ b/tests/vt/test_ops_quant_repack.cpp @@ -201,8 +201,13 @@ TEST_CASE("G7 repacked GEMM matches plain at real model shapes / dtypes") { for (const Shape& s : shapes) { for (vt::DType odt : {vt::DType::kF32, vt::DType::kBF16}) { for (bool strided : {false, true}) { - CAPTURE(s.m); CAPTURE(s.n); CAPTURE(s.k); - CAPTURE(static_cast(odt)); CAPTURE(strided); + // One CAPTURE per line: same-line groups collide on + // DOCTEST_CAPTURE_ (clang/hipcc reject the redefinition). + CAPTURE(s.m); + CAPTURE(s.n); + CAPTURE(s.k); + CAPTURE(static_cast(odt)); + CAPTURE(strided); const int64_t nblocks = s.k / vt::cpu::kQK8_0; const std::vector plain = MakePlainQ8_0(s.n, nblocks, 0x11U); // Optionally over-allocate the activation row stride (a column slice of