From f33b1078cd9c1d6a25c528b5f17c4f41c7cd2123 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 17:23:52 +0100 Subject: [PATCH 01/11] spec(cpu): define A76 BF16 GEMM closure (#284) FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/NOW.md | 2 +- .agents/kernel-matrix.md | 7 +- .agents/roadmap_v1.md | 1 + .agents/specs/cpu-a76-bf16-gemm.md | 225 +++++++++++++++++++++++++++++ docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-agent-record.py | 6 +- 7 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 .agents/specs/cpu-a76-bf16-gemm.md diff --git a/.agents/NOW.md b/.agents/NOW.md index c70370f83..2cb65215a 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -22,7 +22,7 @@ Work: 7 external PRs landed (6 merged + #227 repaired). | Kimi-Linear-48B | 122/128 held; e2e NOT ESTABLISHED | tiktoken-only ckpt: no warm server | | 35B mid-band | **canonical 0.918-0.972x** c1-c32 (@`348c265d`, first c16/c32) | Decode-only window, ONE tool | | Qwen3.5-4B sm_120 | tput **1.0283x** `PENDING`; TTFT/TPOT/E2E 1.085/1.017/1.029x | Profile wave | -| RPi5 A76 CPU | **R5 asm GREEN; llama NOT MET**: 0.461x pf, 0.653x dec | W6: BF16 GEMM | +| RPi5 A76 CPU | **#284 READY**: 0.461x pf, 0.653x dec | Reprofile; C++ BF16 GEMM | | SERVE-ASYNC-MIRROR | **#323 FIXED** (mitigation): graph declines while the mirror is live; 7/7 async gates | Graph read ids at REPLAY | | CPU levers (`QUANT-GGUF-CIQ-GEMM`) | Profile DONE: decode **47% threadpool sync**, prefill **~39% paged attn** | Parakeet encoder; attn dtype hoist | | `SERVE-METRICS` async (#277) | **`/metrics` was DEAD on the shipped server**: AsyncLLM folded nothing. Now live, ctest 366/366 | Config-gated families | diff --git a/.agents/kernel-matrix.md b/.agents/kernel-matrix.md index 7625ff250..41d3110d6 100644 --- a/.agents/kernel-matrix.md +++ b/.agents/kernel-matrix.md @@ -112,6 +112,7 @@ host/sched. Detail: state `KERNEL-FA2-GQA-SWAP-FLIP`. | ID | Item | Upstream | Our code | Tests/evidence | Spike/spec | State | Owner | |---|---|---|---|---|---|---|---| +| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); implementation pending | Same-file Pi floor: 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS; fresh profile + focused BF16 PMU fixture pending under [issue #284](https://github.com/mudler/vllm.cpp/issues/284) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `READY` | - | | `KERNEL-CPU-A76-Q8-DOT` | Cortex-A76 Q8_0 x Q8_0 SDOT, exact-order C++ and scheduled AAPCS64 variants for the Pi decode/prefill keep-quant GEMM | llama.cpp `ggml/src/ggml-cpu/quants.c:400` portable dot and `arch/arm/quants.c:1076-1160` Arm DotProd @ project pin `237ad9b96`; Qwen semantics remain vLLM `555967922` | [ACLE SDOT + HWCAP/MIDR selector](../src/vt/cpu/cpu_quant_dot_sdot.cpp#L85), [AAPCS64 two-block leaf](../src/vt/cpu/cpu_quant_dot_a76.S#L10), [Q8 dispatch](../src/vt/cpu/cpu_quant_dot.cpp#L675), [same-binary variants](../examples/cpu_kernel_bench/main.cpp#L858) | [direct exact variant/edge tests](../tests/vt/test_ops_quant_dot.cpp#L593); QEMU 20/20, 150258 assertions; physical Pi exact checksums/tokens; assembly vs compiler SDOT +3.66% M1/T1, +5.08% M128/T1, +3.69% M128/T4, but −2.43% M1/T4. Same-file llama.cpp floor measured/NOT MET: vllm.cpp 0.461x prefill / 0.653x decode+E2E, 0.758x peak RSS (better), exact-prompt output equal; [assembly evidence](../docs/bench-evidence/rpi5-a76-q8-dot-20260806.md#L40), [competitor evidence](../docs/bench-evidence/rpi5-a76-llamacpp-20260806.md) | [A76 Q8 dot spec](specs/cpu-a76-q8-dot.md) | `GATING` | - | | `KERNEL-ACCEL-PROVIDER-SELECT` | **WHICH implementation of an op runs, when more than one exists on a device** — the selection layer above every kernel family in this matrix. Distinct from `KERNEL-CUDA-DISPATCH-AOT`, which is about which ARCH a CUDA kernel is compiled/selected for; this is about which PROVIDER (ours, a vendor library, MLX, llama.cpp) serves the op at all | no single upstream file: this is the shape vLLM's runtime chain uses everywhere — flashinfer tactic registries, cuBLASLt/CUTLASS per-call heuristics, and torch's backend selection — rather than compile-time pinning | `vt::OpProvider` [op_provider.h](../include/vt/op_provider.h) + [op_provider.cpp](../src/vt/op_provider.cpp); the flat `[OpId][DeviceType]` `void*` table it replaces is gone from [ops.cpp](../src/vt/ops.cpp) with the ~70 op wrappers untouched. Providers registered today: `vt-native` (every backend kernel in the tree, priority 0, unconditional — behaviour preserved exactly), `mlx` (priority 100, Metal `kMatmul`/`kMatmulBT`, build-gated `VLLM_CPP_MLX`, [metal_mlx_provider.mm](../src/vt/metal/metal_mlx_provider.mm)), and — NEW 2026-07-23 (`CLAIM-BACKEND-SEAM-S5-1`, work row `S5`) — **`vt-cpu-ref` (priority −1000, the portable reference tier)**: the CPU kernel installed LAZILY as a negative-priority fallback on a UNIFIED-MEMORY device's first `GetOp` miss, mirroring `custom_op.py:138 forward_native`, so a partial backend runs an op it lacks natively instead of throwing. Native always wins (priority); gated on `Backend::UnifiedMemory()` (a discrete GPU never gets it — a CPU kernel on true device memory is corruption); observable via `GetReferenceTierHits()` + a one-time loud stderr line | [test_op_provider.cpp](../tests/vt/test_op_provider.cpp) 11 cases / 47 assertions — deterministic selection under REVERSED registration order, name tie-break, duplicate rejection, capability predicate, caps re-resolution, decline-and-fall-back, stats, runtime disable; [test_metal_backend.cpp](../tests/vt/test_metal_backend.cpp) 9 cases / 108 assertions on the M4 with MLX ON, including MLX-vs-MSL-vs-CPU NMSE per op at real shapes and an end-to-end DECLINE; **[test_reference_tier.cpp](../tests/vt/test_reference_tier.cpp) (S5): discrete-device refusal + unified-device zero-native-kernel fallback correctness + native-wins + observability, hardware-free via a fake backend on `kXPU`.** Linux CPU 156/156; dgx regression set ALL UNCHANGED — anchor `tests/vt/test_op_provider.cpp:64` | [Metal/MLX reuse study §6](specs/metal-mlx-reuse-study.md); [accelerator-seam-audit §10](specs/accelerator-seam-audit.md); [drop-in kernel ABI](specs/dropin-kernel-abi.md) (the complementary ARGUMENT half) | `ACTIVE` — mechanism landed and gated with THREE provider kinds (`vt-native`, `mlx`, `vt-cpu-ref`); the CUDA/CPU/Vulkan vendor provider rows it was designed for are not yet populated (so the row is deliberately left open) | `CLAIM-BACKEND-ACCEL-PROVIDER-1` | | `KERNEL-CUDA-DISPATCH-AOT` | CUDA runtime, streams, graphs, per-arch dispatch, and generated/AOT artifact selection | platform selection `vllm/platforms/cuda.py:205-493`; gencode handling `CMakeLists.txt:201-220`; JIT packages `cmake/external_projects/triton_kernels.cmake:1-28` | [cuda_backend.cu:20](../src/vt/cuda/cuda_backend.cu#L20), [CMakeLists.txt:37](../CMakeLists.txt#L37), [TritonAOT.cmake:57](../cmake/TritonAOT.cmake#L57) | [CUDA backend tests](../tests/vt/test_cuda_backend.cpp#L31); the Triton GDN AOT tree is now vendored per-arch — `sm_121a` (RUNTIME-VERIFIED) plus `sm_80/86/89/90a/100a` (DERIVED+BUILD-VERIFIED, `CLAIM-TRITON-AOT-PER-ARCH`, 2026-07-28), each a real per-target cubin (`cuobjdump sm=80/86/89/90/100`); the CUDA fast-path arch fan-out (non-GDN) remains as evidenced in `backend-matrix.md` | [inventory](specs/kernel-family-inventory.md); [per-arch Triton AOT](specs/triton-aot-per-arch.md) | `ANCHOR-BACKFILL` | - | @@ -211,9 +212,9 @@ were removed while their specs and same-tool traces remain. Lifecycle stays ## Count invariants -- This table has exactly 35 practical kernel-family rows. -- Baseline lifecycle counts are 8 `ANCHOR-BACKFILL`, 0 `READY`, 4 `PARTIAL`, - 9 `ACTIVE`, 0 `GATING`, 1 `DONE`, and 13 `INVENTORIED`. +- This table has exactly 52 practical kernel-family rows. +- Lifecycle counts are 11 `ANCHOR-BACKFILL`, 3 `READY`, 5 `PARTIAL`, + 7 `ACTIVE`, 1 `GATING`, 4 `DONE`, 12 `INVENTORIED`, and 9 `SPIKE`. - The distinct `KERNEL-CPU-A76-Q8-DOT` compiler/assembly family (`GATING`, PR #79) was added on 2026-08-06; `scripts/check-agent-record.py` pins the total row count. diff --git a/.agents/roadmap_v1.md b/.agents/roadmap_v1.md index 583ca77e0..f68ad4b85 100644 --- a/.agents/roadmap_v1.md +++ b/.agents/roadmap_v1.md @@ -54,6 +54,7 @@ issue is not yet placed. Keyed record: update in place, never append. | [#296](https://github.com/mudler/vllm.cpp/issues/296) | — | Two limitations recorded when #285 landed: a stale TTL comment, and a publish-NAME pin `os.rename` escapes (spec `specs/operator-record.md`, "Follow-up") | bug | | [#285](https://github.com/mudler/vllm.cpp/issues/285) | — | The operator lock refuses a second coordinator; it should only RECORD who is working where (spec `specs/operator-record.md`) | bug | | [#294](https://github.com/mudler/vllm.cpp/issues/294) | `SERVE-ASYNC-LLM` | `test_async_llm`: reusing an aborted request id races the core abort — 26% failure rate under contention, on main | bug | +| [#284](https://github.com/mudler/vllm.cpp/issues/284) | `KERNEL-GEMM-CPU-ELEM-A76` | Close the Raspberry Pi 5 Cortex-A76 BF16 GEMM gap against llama.cpp | perf | | [#277](https://github.com/mudler/vllm.cpp/issues/277) | `SERVE-METRICS` | `/metrics` is dead on the production server: AsyncLLM never folds IterationStats into the logger | bug | | [#241](https://github.com/mudler/vllm.cpp/issues/241) | `ROAD-V1-H3` | MiniMax-H3: support the PRUNED (AdaLN timestep-curve) checkpoint variants | feature | | [#250](https://github.com/mudler/vllm.cpp/issues/250) | — | `a5b52047` reached main without a task branch, and `check-role-discipline` cannot be waived | bug | diff --git a/.agents/specs/cpu-a76-bf16-gemm.md b/.agents/specs/cpu-a76-bf16-gemm.md new file mode 100644 index 000000000..c919ddfb9 --- /dev/null +++ b/.agents/specs/cpu-a76-bf16-gemm.md @@ -0,0 +1,225 @@ +# Cortex-A76 BF16 elementwise GEMM: close the Raspberry Pi 5 llama.cpp gap + +**Row:** `KERNEL-GEMM-CPU-ELEM-A76` · **issue:** +[#284](https://github.com/mudler/vllm.cpp/issues/284) · **state:** `READY` · +**parent:** `KERNEL-GEMM-CPU-ELEM` · **target:** Raspberry Pi 5 Cortex-A76 +r4p1, four cores, AArch64 NEON + DotProd, no i8mm · **comparison source:** +official llama.cpp b9892 at `ee445f93d8a0a5033a46d1960e901ef5caec9a41`. + +## Scope + +The goal is to make the production C++/NEON CPU path for the same-file +Qwen3.5-2B Q8_K_XL workload faster than llama.cpp on the Raspberry Pi 5. +Prefill, decode and output-equivalent end-to-end throughput must all exceed +llama.cpp; latency must be lower and the existing peak-RSS win must not regress. + +In scope: + +- a fresh clean profile of both engines on the identical binding workload; +- a focused BF16 elementwise GEMM microbenchmark in + `examples/cpu_kernel_bench/main.cpp` with production shapes, threads, PMU + counters, cache state and same-binary variant control; +- Cortex-A76-specific C++/ACLE scheduling for the `[N,K]` BF16 path in + `src/vt/cpu/cpu_matmul_elem.cpp`, including tile shape, activation-row + sharing, load/convert/transpose ordering, prefetch distance, tail handling + and caller thread partition; +- a separate A76 translation unit and runtime MIDR selection if Cortex-A76 + compile flags improve code without weakening the generic AArch64 build; +- recursive measurement at microkernel, enclosing `kMatmulBT`, prefill/decode + phase and full-model levels after every retained candidate. + +Out of scope: + +- CUDA, Vulkan, model-forward, loader, quantization-format or C ABI changes; +- weakening the byte-exact elementwise GEMM contract, changing sampling or + regenerating goldens; +- compiling on the Pi, installing vLLM there, or timing under QEMU; +- handwritten assembly until the explicit assembly gate below is satisfied; +- treating the already-faster peak RSS result as permission to trade memory + for speed without measuring the new peak. + +## Upstream chain + +The behavioral oracle remains pinned vLLM `555967922`; llama.cpp is the speed +competitor for this CPU vehicle. The anchors below were verified in a fresh +official b9892 clone at exact commit `ee445f93d`; reconstruct that commit rather +than relying on a mutable local llama.cpp checkout. + +| Component | Exact competitor anchor | What it establishes | +|---|---|---| +| BF16 dot | `ggml/src/ggml-cpu/vec.cpp:139-260` `ggml_vec_dot_bf16` | BF16 widening, independent accumulators and the scalar fallback | +| Chunk worker | `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` `ggml_compute_forward_mul_mat_one_chunk` | 16x16 outer blocking, row/column traversal and `vec_dot` call shape | +| Type dispatch | `ggml/src/ggml-cpu/ggml-cpu.c:390` | BF16 type selects `ggml_vec_dot_bf16` | +| SIMD vocabulary | `ggml/src/ggml-cpu/simd-mappings.h` | AArch64 load/convert/mul/add spellings used by the competitor build | + +Before calling any competitor mechanism decisive, record its resolved build +flags and disassembly on the Pi artifact. Source inspection proposes a lever; +matched profiles and counters establish whether it ran. + +## Our baseline and measured gap + +Binding evidence is +[`docs/bench-evidence/rpi5-a76-llamacpp-20260806.md`](../../docs/bench-evidence/rpi5-a76-llamacpp-20260806.md). +Both engines use the same Qwen3.5-2B Q8_K_XL GGUF bytes, four A76 cores, +17 input tokens, 64 output tokens and three clean unthrottled repetitions. + +| Axis | vllm.cpp | llama.cpp | Ratio / gap | +|---|---:|---:|---:| +| Prefill | 12.81 tok/s | 27.77 tok/s | **0.461x; 2.17x behind** | +| Decode | 2.55 tok/s | 3.91 tok/s | **0.653x; 1.53x behind** | +| Output-equivalent E2E | 2.46 tok/s | 3.77 tok/s | **0.653x; 1.53x behind** | +| E2E latency | 26,018.39 ms | 16,998.49 ms | **1.531x slower** | +| Peak RSS | 2.841 GiB | 3.747 GiB | **0.758x; 24.2% better** | + +The prior clean model profile attributes 57.76% of wall time to BF16 +`Bt16Neon`, 20.10% to the quantized Q8 dot, 6.45% to thread-ready/synchronism +and 4.87% to `F16ToF32`. That makes BF16 elementwise GEMM the largest measured +lever and prefill the furthest-behind binding metric. The attribution must be +reproduced from the fresh branch before changing the kernel. + +Current local anchors: + +- `src/vt/cpu/cpu_matmul_elem.cpp:152-186` — `Bt16Neon`, 16 output columns, + four vector accumulators, four loads + a 4x4 transpose per output group; +- `src/vt/cpu/cpu_matmul_elem.cpp:194-232` — `BtM4Neon`, four activation rows + sharing weight loads and transposes; +- `src/vt/cpu/cpu_matmul_elem.cpp:559-593` — Arm runtime tier selection; +- `src/vt/cpu/cpu_ops.cpp:106-276` — typed chunk worker and partitioning; +- `tests/vt/test_ops_matmul_elem.cpp` — byte-identity oracle across dtypes, + layouts, shapes and thread counts; +- `examples/cpu_kernel_bench/main.cpp` — existing PMU-capable quant benchmark, + which does not yet expose elementwise BF16 GEMM. + +## Port map + +| Competitor / reference | Local surface | Adaptation | +|---|---|---| +| llama.cpp `ggml_vec_dot_bf16` | `src/vt/cpu/cpu_matmul_elem.cpp` A76 BF16 microkernel | preserve its load-latency-hiding intent, but retain lane-per-output and strict K order instead of horizontal reduction | +| llama.cpp 16x16 `mul_mat` chunk worker | `src/vt/cpu/cpu_ops.cpp` typed elementwise chunk worker | measure its current 16-column and M-block partition before changing ownership or tile shape | +| llama.cpp AArch64 build flags / resolved disassembly | optional additive A76 translation unit + runtime MIDR selector | compile locally under QEMU; execute and count only on the Pi | +| llama.cpp benchmark and `perf` evidence | `examples/cpu_kernel_bench/main.cpp` + indexed Pi evidence | identical shapes, affinity and counters; QEMU timings are void | + +## Design and candidate order + +The correctness-preserving invariant is one SIMD lane per output column and +strictly sequential accumulation over K. Separate `vmulq_f32` then +`vaddq_f32` stays mandatory: FMA or K-reassociation is not an optimization for +this row because it changes the existing result contract. + +Candidates are evaluated one at a time in this order: + +1. **Measurement seam.** Add `--op matmul-bt-elem --dtype bf16` fixtures for + real Qwen shapes and M in `{1,17,128}`, threads in `{1,4}`, hot/cold cache, + plus named same-binary variants. Capture wall time, cycles, instructions, + IPC, L1D/L2/refill events and migration/throttle state. +2. **Generated-code audit.** Compile the current NEON function for generic + AArch64 and `-mcpu=cortex-a76`; compare instruction schedule, spills, + register pressure, load-use distance, transpose sequence and unrolling. +3. **C++ schedule.** Interleave independent group loads/converts/transposes + with arithmetic from prior groups; test group-major versus K-major order, + explicit prefetch distances and MR in `{1,2,3,4}`. Preserve exact K order + per lane and reject any spill-driven or model-level regression. +4. **Caller partition.** Measure output-tile ownership and false sharing at M=1 + and prefill M. Change chunking only if the enclosing-op counters demonstrate + a synchronism or cache-locality defect. +5. **Optional packed view.** Consider a load-time/cacheable BF16 tile view only + if counters prove transpose/load waste remains binding and its full model + peak RSS stays no worse than llama.cpp and no worse than the accepted local + baseline without an explicit result disposition. + +Each candidate has a production-default arm and an exact same-binary rollback. +Rejected candidates are recorded with the smallest recursive level that +falsified them; they are not stacked into a mixed experiment. + +## Assembly gate + +Handwritten AArch64 assembly is forbidden while any binding C++ axis remains +at or below llama.cpp. It becomes eligible only after all of these are true: + +1. the retained C++/NEON default is byte-exact at unit and model level; +2. vllm.cpp exceeds llama.cpp on prefill, decode and output-equivalent E2E + throughput, has lower TTFT/E2E latency, and retains its memory floor; +3. the C++ result reproduces on an idle Pi in at least three interleaved + same-binary repetitions with sub-3% spread; +4. disassembly plus PMU counters identify a remaining compiler scheduling, + register-allocation or instruction-selection defect that source-level C++ + experiments failed to remove. + +If assembly becomes eligible, it is a new work item within this same issue: +first capture the compiler body, write a byte-exact direct kernel mutation +test, implement the smallest AAPCS64 leaf, and compare compiler/assembly in the +same binary. A microbenchmark-only win never justifies selecting assembly. + +## Tests to port and extend + +| Source / guarantee | Local gate | +|---|---| +| llama.cpp `tests/test-backend-ops.cpp` BF16 MUL_MAT coverage | extend `tests/vt/test_ops_matmul_elem.cpp` only for any new A76 dispatch/tail/tile seam; retain `memcmp` byte identity | +| Existing scalar reference and full 16-bit conversion sweep | all old cases unchanged under `ref`, `portable`, `neon` and any new A76 tier | +| Variant selection | CLI/mechanism test proves each named benchmark arm selects the intended function; mutation of the selector must fail | +| Model correctness | the established 64-token Pi stream is byte-identical across rollback/default and matches the recorded x86/portable tokens | + +## Gates + +1. **Build/test locally, never on the Pi.** Use the repository-family buildx + recipe (`pf-arm`, Ubuntu 24.04, GCC 13.3, `linux/arm64`) to build Release + CPU-only artifacts under QEMU. Run the focused unit and CLI tests under QEMU. +2. **Direct correctness.** `test_ops_matmul_elem` passes byte-for-byte under + every selectable tier. A deliberate wrong selector or altered accumulation + order must turn the intended test red, then the exact source bytes are + restored. +3. **Physical execution.** Copy only built artifacts and indexed evidence into + a disposable directory on `rich@rpi5fan.lan`. Record SHA-256, CPU/MIDR, + affinity, governor, frequency, temperature and `vcgencmd get_throttled` + before/after. QEMU timing is void. +4. **Recursive performance.** Retain a change only when the focused kernel, + enclosing `kMatmulBT`, phase-level prefill/decode and full model tell a + coherent story. A lower-level win with an enclosing regression is rejected. +5. **Competitor floor.** Rebuild the exact llama.cpp b9892 artifact with the + same QEMU toolchain and rerun it on the same Pi/file/workload. Success is + strictly `>1.0x` throughput and `<1.0x` latency/memory ratio on every + applicable binding axis; equality remains open. +6. **Regression.** The native CPU gate and project record gates pass. The + kernel change must be inert on x86 and on non-A76 Arm selection. + +## Dependencies + +- `KERNEL-GEMM-CPU-ELEM` and `KERNEL-CPU-A76-Q8-DOT` are landed prerequisites. +- The exact GGUF is already present on the Pi; no new model download is owed. +- Docker buildx/QEMU is the compilation environment. The Pi is execution and + PMU hardware only. +- Physical-Pi deployment is user-authorized for this disposable machine; do + not change persistent services or system configuration. + +## Work breakdown + +| W | Deliverable | State | +|---|---|---| +| W0 | Refresh both-engine binding baseline/profile; validate artifact hashes and idle-state evidence | pending | +| W1 | Add elementwise BF16 microbench/PMU fixtures and selection mutation proof | pending | +| W2 | Audit current compiler output and evaluate C++/ACLE schedule/tile/prefetch candidates | pending | +| W3 | Recursively gate retained C++ default through full model vs llama.cpp | pending | +| W4 | Assembly eligibility decision from the explicit gate; implement only if eligible | blocked by W3 | + +## Risks and stop conditions + +- Qwen's mixed GGUF means an op-level BF16 gain may be Amdahl-limited. Stop a + candidate when the enclosing phase falsifies it and re-profile before moving + to a different family. +- Prefetch can improve an isolated hot loop while increasing shared-cache + pressure. Both hot/cold microbench arms and full-model peak RSS are binding. +- A76 has 32 architectural SIMD registers, but MR=4 plus transient transpose + vectors can still spill after compiler scheduling. Disassembly is evidence, + not a post-hoc explanation. +- If the fresh profile no longer names BF16 GEMM as the largest actionable + lever, stop before code, update this spec with the measured ranking, and + create/claim the correct row rather than forcing this hypothesis. +- Network, SSH, PMU permission, thermal throttling or unavailable artifacts + leave the corresponding gate `PENDING`; they never turn into assumed success. + +## Outcome + +Spec accepted after the inventory checker correctly rejected the new 52nd +kernel row while pinned to 51. The count ratchet moved only after the issue, +row and structured spec existed. Implementation and binding measurement remain +pending. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index d77737f02..2583b67e8 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -248,7 +248,7 @@ host mirror is freed once the device Marlin resident is built. | 64-token Qwen model gate | Byte-identical across x86, portable, SDOT and assembly arms; asm vs SDOT median TTFT -1.55%, TPOT neutral, E2E -0.13%; vs portable TTFT -33.40%, E2E -2.67%. Cortex-A76+DotProd selects assembly by default | | Same-file llama.cpp floor (pp17/tg64) | **NOT MET on speed**: prefill 12.81 vs 27.77 tok/s (0.461x), decode 2.55 vs 3.91 (0.653x), E2E 26,018.39 vs 16,998.49 ms ([competitor evidence](bench-evidence/rpi5-a76-llamacpp-20260806.md)) | | Peak RSS | **2.841 vs 3.747 GiB, 24.2% less**; 3 clean unthrottled reps; same-text 64-token greedy output byte-identical after trailing-newline normalization | -| `PENDING` | Pi concurrency; BF16 GEMM / speed closure (the 2.17x prefill, 1.53x decode gap profiling is W6) | +| `READY` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Fresh both-engine profile, then C++/NEON BF16 GEMM speed closure; assembly remains blocked until C++ exceeds llama.cpp on every speed axis | Same GGUF file both arms, `dgx.casa` GB10 aarch64 (20 cores), idle, 3 reps, llama.cpp `237ad9b96` built fresh on the same host. diff --git a/docs/STATUS.md b/docs/STATUS.md index 12b6844cf..8ea2aa18a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -110,7 +110,7 @@ token-for-token correctness against the pinned oracle. | GGUF loading (F32/F16/BF16/Q4_0/Q8_0/Q2_K/Q3_K/Q4_K/Q5_K/Q6_K/IQ2_XXS/IQ3_XXS/IQ2_S/MXFP4/NVFP4) | Supported; compute-in-quant (keep-quant) on CPU AND now CUDA for the six K-block encodings PLUS Q2_K/IQ2_XXS/IQ3_XXS (DeepSeek-V4 W8, 2026-07-29 - the FIRST CUDA keep-quant GGUF k-quant GEMM `KERNEL-QUANT-CIQ-GEMM-CUDA`, MMVQ-style dequant-in-kernel, GB10-gated 92401/92401 vs the CPU oracle, so a CUDA runner keeps blocks compressed and dots them on the GPU instead of the ARM cores); **NVFP4 now COMPUTES IN FP4 on CUDA for the dense-MLP and full-attention projections (2026-07-29, `CLAIM-GGUF-NVFP4-COMPUTE`), no longer materialize-only** | Weights in six block encodings stay compressed from file to matmul on CPU (no BF16 expansion). NVFP4 (ggml type 40) DEQUANTIZES, including the per-tensor (per-expert) `.scale` sidecar the container keeps outside the blocks; gated BIT-EXACT against the compressed-tensors NVFP4 path on real Qwen3.6-27B bytes from both containers. **It no longer expands to bf16 on CUDA:** an NVFP4 matmul/expert weight is REPACKED at load into the same (`weight_packed [N,K/2]`, `weight_scale [N,K/16]`) operand pair the compressed-tensors path produces - a pure byte permutation, gated BYTE-IDENTICAL against that container - and the existing `vt::MatmulNvfp4*` kernels run on it, so no new kernel exists and no numerics are re-derived. Covers the dense MLP + full-attention q/k/v/o and the MoE shared/routed experts; the GDN `in_proj_*` family and `ssm_out` still expand (the V-head reorder rewrites their layout) and a CPU build still expands everything - the documented `part` subset. **MEASURED GB10 (2026-07-29), same-binary A/B, 2 reps/arm:** peak RSS **50.8 -> 25.7 GiB**, load-and-generate **1:58 -> 0:41**; the 256 moved projections cost 35 840 MiB expanded vs 10 080 fp4-resident (3.56x). **The safetensors-sibling divergence CLOSES:** the fp4 arm is token-IDENTICAL over 24 greedy tokens where the same binary's bf16 arm diverges at index 4. REPORTED, not gated: the containers are not the same model - the GGUF NVFP4-quantizes 192 GDN `in_proj` tensors the safetensors keeps BF16 (mean rel. weight error ~0.18) and their activation global scales differ, so identity is not guaranteed and a cross-container throughput arm is invalid. SACRED gates unmoved: `test_qwen27_paged_engine` 235/235, `test_qwen36_paged_engine` 315/315. **The MoE (35B) stacked-expert arm is HARDWARE-GATED too (2026-07-29)**: the real 35B A3B NVFP4 GGUF loads and generates through the fp4 path, its 120 routed-expert stacks x 256 experts repack to the modelopt safetensors' operands with ZERO differing bytes over 840 sampled (tensor, expert) slabs, and all 840 per-expert `.scale[e]` are bit-identical to that expert's `weight_scale_2` - the scale INDEXING, mutation-proved against a `scales[0]`-for-all and an expert-0-slab-for-all mutant. Same-binary A/B: peak RSS 68.5 -> 22.7 GiB (3.01x), load-and-generate 1:51.9 -> 0:28.8, tokens IDENTICAL (the 35B routed experts run the W4A16 grouped GEMM in both arms). Recorded OPEN: this case's 24-token greedy stream is NOT run-to-run stable (1 of 3 `use_a16` and 1 of 4 reference runs differed), so the binding results are the weight-level byte identity and the residency audit, not token-exactness; `test_qwen36_paged_engine` is token-exact at ITS engine params, so the instability belongs to this case's configuration and attributing it is owed work. It also FIXED a latent defect the MoE arm made reachable: the two fp4 fused MoE blocks issued the router GEMM assuming the safetensors `[K,N]` gate layout and threw `matmul: inner dims mismatch` on the GGUF's `[N,K]`; `MoeRouterLogits` now branches on `nk` (inert for safetensors, SACRED unmoved). **Q2_K (id 10) + IQ2_XXS (id 16) DEQUANTIZE (2026-07-29, `CLAIM-DSV4-GGUF-LOADER`):** the ~2-bit types the single-Spark `DeepSeek-V4-Flash-GGUF UD-IQ2_XXS`/`UD-Q2_K_XL` vehicles use, ported 1:1 from llama.cpp `ggml-quants.c` (`iq2xxs_grid` codebook + signs; Q2_K nibble sub-scale/min), unit-gated on hand-derived bytes (`test_gguf_dequant` 15/15). Dequant-only (no vec_dot -> expand-bf16). A V4-GGUF model cannot RUN yet: the name map (tensor-manifest-blocked) + the V4 forward (W3-W8) remain. **Multi-shard split GGUF READING (2026-08-03, `CLAIM-GGUF-SPLIT-SHARDS`):** `GgufFile::Open` now transparently stitches llama.cpp `gguf-split` shards (`...-00001-of-00003.gguf`) — every shard mmap'd, tensor tables merged, KV metadata from shard `00001`, sibling mappings kept alive by the primary so keep-quant mmap-borrows stay valid across shards (`OwnsSpan` is shard-aware); `VT_GGUF_NO_SPLIT=1` opts out; unit-gated (`test_gguf` split-merge / no-split / count-mismatch cases, 33/33 local). This unblocks the real 3-shard `unsloth/DeepSeek-V4-Flash-0731 UD-IQ2_M` (~91 GiB), whose layout is the NATIVE `deepseek4` arch — per-block `ffn_gate_tid2eid` hash tables (hash layers 0/1/2) + `hc_*` MHC + DSA compressor/indexer are all PRESENT (name-map 1328/1328), `vocab_size` derives from `token_embd` — NOT a standard llama.cpp conversion, so no loader-layout change is owed. It now loads THROUGH 1324/1328 tensors; the sole remaining gap is 4 routed-expert slabs quantized with IQ2_S (id 22, ×2) + MXFP4 (id 39, ×2) — encodings we have GGUF block traits for but no keep-quant vec_dot, so they hit the expand→dequant path which lacks them. Expanding those 4 expert tensors to bf16 would add ~17 GiB (~106 GiB total → OOM-reboot risk), so the memory-safe fix is an IQ2_S+MXFP4 keep-quant kernel (CPU dequant dispatch + `iq2s_grid` + CUDA `DotSuperblock`), spec'd as the next brick **IQ2_S (id 22) + MXFP4 (id 39) DEQUANTIZE + KEEP-QUANT on CPU (2026-08-03, `CLAIM-DSV4-UDIQ2M-QUANT`, off-GPU):** the extra per-tensor "dynamic" encodings the `unsloth/DeepSeek-V4-Flash-GGUF UD-IQ2_M` checkpoint mixes into its last routed-expert slabs (IQ2_S `ffn_gate/up` dotting Q8_K, MXFP4 `ffn_down` dotting Q8_0) — ported 1:1 from llama.cpp `ggml-quants.c` @ 237ad9b96 (`iq2s_grid` 1024-entry codebook + DIRECT sign bytes; MXFP4 `kvalues_mxfp4` + `e8m0_to_fp32_half` micro-scaling, distinct from the compressed-tensors `E8M0ToF32` NVFP4 path). CPU dequant + keep-quant `vec_dot`, unit-gated on hand-derived golden bytes (`test_gguf_dequant` 17/17), an INDEPENDENT f64 dequant-then-dot + GEMM NMSE (`test_ops_quant_dot` 19/19), and keep-quant routing (`test_gguf_keep_quant` 37/37) — all CPU-green, so UD-IQ2_M's four previously-`unsupported ggml type 22/39` slabs now load COMPRESSED (no ~17 GiB bf16 expansion that OOM-reboots the box). CUDA: the IQ2_S device `DotSuperblock` is wired into the Q8_K grouped-MoE GEMM and now **CUDA-BUILT + LINKED on GB10 (sm_121a, CUDA 13.0, `-Werror`, 2026-08-03 integration)** — it compiles clean and the merged binary links; MXFP4's device dot (`DotMXFP4`) is written but NOT wired (Q8_0-activation needs a separate 32-block GEMM) so it is marked `[[maybe_unused]]` to keep the ready math without tripping nvcc #177-D, and on GPU MXFP4 CPU-fallbacks like Q4_0/Q8_0. The V4-GGUF forward + a real UD-IQ2_M GPU load/coherence run are owed | | AWQ / GPTQ quantization | W0 spike + W1 CPU INT4 dequant primitive; not yet loadable end to end | INT4 unpack+dequant-to-bf16 for BOTH community formats, mirroring vLLM 1:1 (AWQ reverse-order `awq_triton.py`; GPTQ `qdq_4.cuh` with zero_offset v1/v2 + act-order g_idx). Unit-gated RED-first (hand-computed known bytes + double-precision roundtrip). NOT wired to a loader, no GPU Marlin compute, no model run yet: config recognizer (W2), Marlin GPU GEMM riding the vendored NVFP4 Marlin (W4), CPU e2e (W3), GPTQ 8/2/3-bit (W5) and MoE (W6) are named next bricks. See [.agents/specs/awq-gptq-quant.md](../.agents/specs/awq-gptq-quant.md) | | MXFP4 (compressed-tensors `mxfp4-pack-quantized`) | Compute PROVEN (#38); GQA-swap ON (#49); decode-graph+gate_up FUSION default-ON. `VT_MARLIN_DENSE` DEFAULT-ON (`KERNEL-MARLIN-DENSE-EXEC`): dense marlin 48-CTA byte-faithful (32B 0.000, 263/263), binding beats #51 every axis (c1 1.020, c8 0.969, mem 2.63x). **`QUANT-CT-MXFP4-FINAL-STACK` TERMINAL — both last levers exhausted: num_splits cap `VT_FA2_NSPLITS_CAP` gated-OFF (c1-only, self-corrects@c8; 32B strict char-identical); glue folds via `vt::FusedChain`; `FLASH-AUDIT` #68: c8 flash gap +12.5us/call is occupancy/L2-bound; `-use_fast_math` TRIED, REGRESSES flash (168.8→189.8), rejected. c1 1.020x PASS, c2-c8 0.962-0.969.** state.md | Shared with DeepSeek-V4-Flash + Kimi-K3 MXFP4 paths. CPU E8M0 dequant 5/5·1142. GPU W4A4 + MoE-expert e2e later | -| CPU backend vs llama.cpp | 20-core at floor; RPi5/A76 below floor `GATING` | Pi: AAPCS64 beats SDOT 3.66-5.08%; llama.cpp 2.17x pf / 1.53x dec faster (0.461x/0.653x); RSS -24.2%; 64-tok byte-exact. BF16 GEMM open | +| CPU backend vs llama.cpp | 20-core at floor; RPi/A76 `GATING`; BF16 `READY` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Pi: 0.461x prefill / 0.653x decode; RSS 24.2% lower. C++/NEON before assembly. | | Paged KV cache + prefix caching | Supported | Block-paged full attention, hybrid full-attention + GDN state groups, automatic prefix caching (APC) on by default for dense models (cache-ON gated end to end: token-identical output, cache hits, faster TTFT) | | fp8 KV cache (`cache_dtype=fp8`) | In progress (W1 CPU brick), not yet usable end-to-end | HIGH-priority memory/throughput lever (halves the KV footprint). W0 spike + W1 CPU brick landed (`KV-FP8` ACTIVE): fp8-e4m3 K/V STORE (`Quantize(hp/scale)`) + the paged-attention READ dequant (`Dequant(fp8)*scale`) + the `cache_dtype` config parse, all CPU-gated RED-first (`test_ops_fp8_kv_cache` 8/8·511; a wrong store direction fails 3/480). Storage is 1-byte fp8 (`DType::kI8`) + a `Fp8KVCacheDataType` interpretation enum, per-tensor k/v scales (mirroring vLLM `BaseKVCacheMethod`). The CUDA store + fp8 paged-attention read (the GPU memory-halving path, DGX-blocked), the runner/spec integration (half-sized KV blocks + checkpoint-scale threading + `--kv-cache-dtype`/`--calculate-kv-scales`), fp8_e5m2 and per-head scales are named W2-W5 in [.agents/specs/fp8-kv-cache.md](../.agents/specs/fp8-kv-cache.md). No model can run with an fp8 KV cache yet | | Prefix-cache matching unit (`--prefix-match-unit`) | Partial (resolver landed, config/scheduler wiring pending) | 0.26-new knob setting the finest token boundary a prefix-cache hit can land on (the `hash_block_size`). W1: `resolve_kv_cache_block_sizes` ported 1:1 (hybrid `hash_block_size = prefix_match_unit if set else gcd(group block sizes)`; single-group inert; back-off on no-consumer / mamba-non-align; throws on non-divisible), CPU unit-gated RED-first (default gcd vs `=16`). Pending: the config/CLI/ABI field (W2), scheduler threading of a finer-than-block matching unit (W3, needs the KV-block-pool align path), and the benchmark (W4). Default path byte-identical (dense single-group models ignore it). | diff --git a/scripts/check-agent-record.py b/scripts/check-agent-record.py index 96769c865..f4e700c01 100644 --- a/scripts/check-agent-record.py +++ b/scripts/check-agent-record.py @@ -174,7 +174,11 @@ # proves the portable dot is reached at 20.10% of Qwen3.5-2B user cycles; # the row owns exact-order C++ SDOT vs scheduled AAPCS64, independent of # the broad CPU-backend row. - "KERNEL": (AGENTS / "kernel-matrix.md", 51), + # 52 since 2026-08-10 (issue #284): +`KERNEL-GEMM-CPU-ELEM-A76`, the + # Raspberry Pi 5 Cortex-A76 BF16 specialization of the elementwise GEMM. + # The C++/NEON row is separately gateable from the broad portable family; + # assembly is explicitly blocked until its C++ path beats llama.cpp E2E. + "KERNEL": (AGENTS / "kernel-matrix.md", 52), # 56 since 2026-07-22: +`BACKEND-ACCEL-PROVIDER` (the acceleration-provider seam # itself, which is a cross-backend platform concern rather than a platform). # 57 since 2026-07-22: +`BACKEND-SEAM-AUDIT` (the accelerator-seam AUDIT — does From 237b33c0d99eb016119a44072c1de2d8c929944d Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 17:25:38 +0100 Subject: [PATCH 02/11] chore(cpu): activate A76 BF16 GEMM row (#284) FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/NOW.md | 2 +- .agents/coordination.md | 1 + .agents/kernel-matrix.md | 6 +++--- .agents/specs/cpu-a76-bf16-gemm.md | 2 +- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.agents/NOW.md b/.agents/NOW.md index 2cb65215a..f4b6dc4c4 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -22,7 +22,7 @@ Work: 7 external PRs landed (6 merged + #227 repaired). | Kimi-Linear-48B | 122/128 held; e2e NOT ESTABLISHED | tiktoken-only ckpt: no warm server | | 35B mid-band | **canonical 0.918-0.972x** c1-c32 (@`348c265d`, first c16/c32) | Decode-only window, ONE tool | | Qwen3.5-4B sm_120 | tput **1.0283x** `PENDING`; TTFT/TPOT/E2E 1.085/1.017/1.029x | Profile wave | -| RPi5 A76 CPU | **#284 READY**: 0.461x pf, 0.653x dec | Reprofile; C++ BF16 GEMM | +| RPi5 A76 CPU | **#284 ACTIVE**: 0.461x pf, 0.653x dec | Reprofile; C++ BF16 GEMM | | SERVE-ASYNC-MIRROR | **#323 FIXED** (mitigation): graph declines while the mirror is live; 7/7 async gates | Graph read ids at REPLAY | | CPU levers (`QUANT-GGUF-CIQ-GEMM`) | Profile DONE: decode **47% threadpool sync**, prefill **~39% paged attn** | Parakeet encoder; attn dtype hoist | | `SERVE-METRICS` async (#277) | **`/metrics` was DEAD on the shipped server**: AsyncLLM folded nothing. Now live, ctest 366/366 | Config-gated families | diff --git a/.agents/coordination.md b/.agents/coordination.md index b95b29347..e9a1db063 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -1635,6 +1635,7 @@ commit order. | Claim | Row IDs | Agent | Worktree / remote dir | Branch | Owned scope | State | Last update | |---|---|---|---|---|---|---|---| +| `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | `KERNEL-GEMM-CPU-ELEM-A76` (`ACTIVE`) | Codex (gpt-5.6-sol), fresh helper implementer | isolated local worktree `/tmp/vllm-a76-bf16-impl`; build/test under buildx/QEMU only; execute/PMU on user-authorized disposable `rich@rpi5fan.lan` | `helper/KERNEL-GEMM-CPU-ELEM-A76-W0-W2`, base spec commit `999e1221e` plus this claim commit | Issue #284 W0-W2 only: refresh both-engine baseline/profile; add the BF16 elementwise benchmark/selection mutation proof; audit disassembly and retain only byte-exact C++/NEON changes with recursive measurements. Owns the new spec/row and scoped edits to `examples/cpu_kernel_bench/main.cpp`, its CLI test/CMake only if needed, `src/vt/cpu/cpu_matmul_elem*`, `src/vt/cpu/cpu_ops.cpp` only if profiling proves partitioning, focused tests, and indexed Pi evidence/record projections. Excludes assembly, model/loader/CUDA/Vulkan/C ABI changes, building on the Pi, service changes and PR/push. | `ACTIVE` | 2026-08-10 — spec committed; helper worktree creation and fresh W0 measurement are next | | `CLAIM-SAMPLE-PROMPT-LOGPROBS-W1` | `SAMPLE-PROMPT-LOGPROBS` (`ACTIVE`; W1 runner source only) | Claude Code (claude-opus-5) | `/home/mudler/_git/vllm.cpp-prompt-logprobs`; CPU-only `build-gate` (`-DVLLM_CPP_CUDA=OFF` Release) — NO GPU, NO download, claims no benchmark device | `row/SAMPLE-PROMPT-LOGPROBS`, opened on `origin/main` `bd6b3936`, rebased onto `8a6704a2` for the 2026-08-10 review repair and onto `e63d11d3` to land, issue #223 | The runner-side prompt-logits source ported from `_get_prompt_logprobs_dict`. Owns ONLY: the `prompt_logprob_rows`/`prompt_logprob_indices` block in `prepare_inputs.{h,cpp}`, `collect_prompt_logprobs` + `in_progress_prompt_logprobs_` in `runner.{h,cpp}`, `Sampler::compute_prompt_logprobs` in `sampler.{h,cpp}`, `num_prompt_logprobs` in `input_batch.{h,cpp}`, section 9 of `tests/vllm/v1/test_llm_engine.cpp`, this claim, the row cells, `.agents/NOW.md`, `.agents/roadmap_v1.md` issue rows, one `docs/STATUS.md` sentence, and the `prompt_logprobs` paragraph in `docs/USAGE.md`. **NON-COLLISION:** NO kernel, NO vt op, NO ABI, NO CMake, NO model file — the full-logits route reuses the existing `VT_LOGITS_GATHER=0` path rather than widening any model's gather contract. EXCLUDED: the OpenAI `echo` serialization (W2) and issue #231, both split out. | `ACTIVE` | 2026-08-10 — W1 landed then REPAIRED after a fresh review returned `VERDICT: FAIL`. Review found a REAL bug: the inertness guard keyed on `prompt_logprob_indices` while the full-logits assertion keyed on `prompt_logprob_rows`, so a zero-row final chunk (the exact-prefill edge, `:5668-5673`) beside another multi-token request threw `VT_CHECK` out of `engine.step()` and killed the whole batch. Fixed by moving the check inside the `num_rows > 0` slice; regression §8(h), RED-first (the throw). Second finding: the on-vs-off inertness case could not see a change to the SHARED route (forcing full logits on every step left it 17/17·346 green), so §8(g) now asserts the route DECISION through a const runner seam and fails under that exact mutation. Third finding recorded, NOT fixed and NOT claimed: the full-logits route hands the sampler a host pointer wearing the device label — sound on unified memory, unverified on CUDA; spec risk 4 + a `PENDING` CUDA smoke gate, and `docs/USAGE.md` narrowed to say so. Three latent lifetime divergences recorded as risks 5-7. Gates on the rebased tree (`e63d11d3`): clean CPU Release build 0 warnings under `-Werror`, `test_llm_engine` **21/21 · 384**, full `ctest -j 6` **365/366** with `test_openai_conformance` (parallel starvation — connection `-1`s) passing serially 1/1 in 0.39 s. Row stays `ACTIVE`: W2 `echo` serialization is the named residual, and the CUDA smoke gate is `PENDING`. | | `CLAIM-MOONCAKE-STORE` | `KV-MOONCAKE-STORE` (new, `SPIKE`) | Claude Code (opus-5), helper role | isolated worktree `/home/mudler/_git/vllm.cpp-mooncake`; records-only — NO build, NO GPU, NO download, NO external dependency installed | `row/KV-MOONCAKE-STORE`, base `origin/main` `848d4a87` | The `MooncakeStoreConnector` spike, records-only in this commit: NEW [`.agents/specs/mooncake-store-connector.md`](specs/mooncake-store-connector.md), the NEW `KV-MOONCAKE-STORE` engine-matrix row + section/total counters, the `ENGINE_ROWS` 146→147 bump in `scripts/check-agent-record.py` with its justification comment, a PROSE-ONLY Mooncake-disposition correction inside the `KV-CONNECTORS` row (both NOT-SCHEDULED sentences; that row's `ANCHOR-BACKFILL` state, tier, anchors, evidence, spec link and owner are UNCHANGED, so it is not claimed here), the `ROAD-V1-D4` portfolio note + canonical-table link, the roadmap issue-table row for [#287](https://github.com/mudler/vllm.cpp/issues/287), and this claim. **NON-COLLISION:** touches NO `src/`, `include/`, `tests/`, `examples/` or CMake path; the only script touched is the record checker's own row-count constant. | `ACTIVE` | 2026-08-10 — spec committed. The `KV-CONNECTORS` spike's blanket "Mooncake NOT SCHEDULED" conflated TWO connectors: `MooncakeConnector` (P2P prefill/decode over the Transfer Engine — two nodes, fabric, proxy) KEEPS that verdict; `MooncakeStoreConnector` (shared KV object store, the LMCache analogue) is reopened because (a) Mooncake is NATIVE C++ — `mooncake::Client` in `client_service.h`, and the `MooncakeDistributedStore` vLLM imports is a pybind wrapper over it — so we LINK instead of reimplementing a wire (the inverse of the LMCache cost shape), and (b) its single-node `protocol: "tcp"` + `mooncake_master` config is gateable on one box with NO RDMA NIC. The landed W5 `KVConnector` seam needs NO change. NEXT: W0, the go/no-go link spike (build Mooncake from source, pin the revision, drive `mooncake::Client` from a standalone C++ TU against a local master over TCP) — a genuine stop point per spec §S1. Speed is recorded as an OPEN axis: the RDMA/GPUDirect path that motivates the connector is unmeasurable for want of a fabric on any box we own. PENDING developer authority: the Mooncake source build, push/PR. | | `CLAIM-MUSE-GLIMMER-SPEC` | `MODEL-MM-muse-glimmer-muse-glimmer-for-conditional-generation` (`SPIKE`) | Claude Code (opus-5), helper role | isolated worktree `/home/mudler/_git/vllm.cpp-muse-glimmer`; CPU-only, records+spec+W0 scaffold — NO GPU, weights on the shared NAS | `row/MODEL-MUSE-GLIMMER`, base `origin/main`; PR [#279](https://github.com/mudler/vllm.cpp/pull/279) OPEN | The Muse Glimmer port. Spec `.agents/specs/muse-glimmer.md`, the `MODEL-MM-muse-glimmer-*` matrix row + checklist + rollup, issue-table entry for [#268](https://github.com/mudler/vllm.cpp/issues/268), porting-inventory §9 deviation 16 (the OFF-PIN anchor), `check-agent-record.py` MODEL count, the W0 scaffold (registry + config parse + weight name map + refuse-by-name forward, gate 11/11), `CHECKPOINT_ROOT` in `.env.example` + `.agents/environment.md`. Those helpers ALL LANDED onto the row branch: W1 the 52-layer text forward, W3 the 50-layer perception encoder, W4 the mm wiring (tower to adapter to projection to `perception_emb_norm` to the masked scatter; released-30B accounting 1436/1436), W7 the ATEM reasoning + tool parsers. The branch is no longer "records + spec + W0 scaffold" and its forward no longer refuses. `CLAIM-MUSE-GLIMMER-FIX` (`row/MODEL-MUSE-GLIMMER-FIX`) then closed the PR #279 review findings: three COVERAGE HOLES that mutation testing found green (`perception_emb_norm` untested, the whole non-FusedChain fallback arm unexercised, RoPE theta ungated at a realistic magnitude), the stale "OPEN FINDING" that had disarmed the real-weights accounting assertion, and the false claim in both parser headers that the seam has no `skip_special_tokens` (it does; channel scoping is an OPEN GAP at server defaults, spec §6.7). WHAT IS STILL NOT ESTABLISHED, and must not be implied: the full-depth 52-layer arm never ran (evidence is reduced depth 4/52, 5 prefill argmax positions, no generated tokens), multi-step decode is untested, the perception encoder has no reference check, and nothing has run through the server. **Anchored to the UNMERGED [vllm#51655](https://github.com/vllm-project/vllm/pull/51655) head `075d645af`** on explicit developer direction 2026-08-10: Muse Glimmer exists neither at the pin `555967922` nor on vLLM `main`. **No speed axis is claimable** until #51655 merges and the pin advances — the pinned oracle cannot load the model. | `ACTIVE` | 2026-08-10 | diff --git a/.agents/kernel-matrix.md b/.agents/kernel-matrix.md index 41d3110d6..9929c017f 100644 --- a/.agents/kernel-matrix.md +++ b/.agents/kernel-matrix.md @@ -112,7 +112,7 @@ host/sched. Detail: state `KERNEL-FA2-GQA-SWAP-FLIP`. | ID | Item | Upstream | Our code | Tests/evidence | Spike/spec | State | Owner | |---|---|---|---|---|---|---|---| -| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); implementation pending | Same-file Pi floor: 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS; fresh profile + focused BF16 PMU fixture pending under [issue #284](https://github.com/mudler/vllm.cpp/issues/284) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `READY` | - | +| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); implementation pending | Same-file Pi floor: 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS; fresh profile + focused BF16 PMU fixture pending under [issue #284](https://github.com/mudler/vllm.cpp/issues/284) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `ACTIVE` | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | | `KERNEL-CPU-A76-Q8-DOT` | Cortex-A76 Q8_0 x Q8_0 SDOT, exact-order C++ and scheduled AAPCS64 variants for the Pi decode/prefill keep-quant GEMM | llama.cpp `ggml/src/ggml-cpu/quants.c:400` portable dot and `arch/arm/quants.c:1076-1160` Arm DotProd @ project pin `237ad9b96`; Qwen semantics remain vLLM `555967922` | [ACLE SDOT + HWCAP/MIDR selector](../src/vt/cpu/cpu_quant_dot_sdot.cpp#L85), [AAPCS64 two-block leaf](../src/vt/cpu/cpu_quant_dot_a76.S#L10), [Q8 dispatch](../src/vt/cpu/cpu_quant_dot.cpp#L675), [same-binary variants](../examples/cpu_kernel_bench/main.cpp#L858) | [direct exact variant/edge tests](../tests/vt/test_ops_quant_dot.cpp#L593); QEMU 20/20, 150258 assertions; physical Pi exact checksums/tokens; assembly vs compiler SDOT +3.66% M1/T1, +5.08% M128/T1, +3.69% M128/T4, but −2.43% M1/T4. Same-file llama.cpp floor measured/NOT MET: vllm.cpp 0.461x prefill / 0.653x decode+E2E, 0.758x peak RSS (better), exact-prompt output equal; [assembly evidence](../docs/bench-evidence/rpi5-a76-q8-dot-20260806.md#L40), [competitor evidence](../docs/bench-evidence/rpi5-a76-llamacpp-20260806.md) | [A76 Q8 dot spec](specs/cpu-a76-q8-dot.md) | `GATING` | - | | `KERNEL-ACCEL-PROVIDER-SELECT` | **WHICH implementation of an op runs, when more than one exists on a device** — the selection layer above every kernel family in this matrix. Distinct from `KERNEL-CUDA-DISPATCH-AOT`, which is about which ARCH a CUDA kernel is compiled/selected for; this is about which PROVIDER (ours, a vendor library, MLX, llama.cpp) serves the op at all | no single upstream file: this is the shape vLLM's runtime chain uses everywhere — flashinfer tactic registries, cuBLASLt/CUTLASS per-call heuristics, and torch's backend selection — rather than compile-time pinning | `vt::OpProvider` [op_provider.h](../include/vt/op_provider.h) + [op_provider.cpp](../src/vt/op_provider.cpp); the flat `[OpId][DeviceType]` `void*` table it replaces is gone from [ops.cpp](../src/vt/ops.cpp) with the ~70 op wrappers untouched. Providers registered today: `vt-native` (every backend kernel in the tree, priority 0, unconditional — behaviour preserved exactly), `mlx` (priority 100, Metal `kMatmul`/`kMatmulBT`, build-gated `VLLM_CPP_MLX`, [metal_mlx_provider.mm](../src/vt/metal/metal_mlx_provider.mm)), and — NEW 2026-07-23 (`CLAIM-BACKEND-SEAM-S5-1`, work row `S5`) — **`vt-cpu-ref` (priority −1000, the portable reference tier)**: the CPU kernel installed LAZILY as a negative-priority fallback on a UNIFIED-MEMORY device's first `GetOp` miss, mirroring `custom_op.py:138 forward_native`, so a partial backend runs an op it lacks natively instead of throwing. Native always wins (priority); gated on `Backend::UnifiedMemory()` (a discrete GPU never gets it — a CPU kernel on true device memory is corruption); observable via `GetReferenceTierHits()` + a one-time loud stderr line | [test_op_provider.cpp](../tests/vt/test_op_provider.cpp) 11 cases / 47 assertions — deterministic selection under REVERSED registration order, name tie-break, duplicate rejection, capability predicate, caps re-resolution, decline-and-fall-back, stats, runtime disable; [test_metal_backend.cpp](../tests/vt/test_metal_backend.cpp) 9 cases / 108 assertions on the M4 with MLX ON, including MLX-vs-MSL-vs-CPU NMSE per op at real shapes and an end-to-end DECLINE; **[test_reference_tier.cpp](../tests/vt/test_reference_tier.cpp) (S5): discrete-device refusal + unified-device zero-native-kernel fallback correctness + native-wins + observability, hardware-free via a fake backend on `kXPU`.** Linux CPU 156/156; dgx regression set ALL UNCHANGED — anchor `tests/vt/test_op_provider.cpp:64` | [Metal/MLX reuse study §6](specs/metal-mlx-reuse-study.md); [accelerator-seam-audit §10](specs/accelerator-seam-audit.md); [drop-in kernel ABI](specs/dropin-kernel-abi.md) (the complementary ARGUMENT half) | `ACTIVE` — mechanism landed and gated with THREE provider kinds (`vt-native`, `mlx`, `vt-cpu-ref`); the CUDA/CPU/Vulkan vendor provider rows it was designed for are not yet populated (so the row is deliberately left open) | `CLAIM-BACKEND-ACCEL-PROVIDER-1` | | `KERNEL-CUDA-DISPATCH-AOT` | CUDA runtime, streams, graphs, per-arch dispatch, and generated/AOT artifact selection | platform selection `vllm/platforms/cuda.py:205-493`; gencode handling `CMakeLists.txt:201-220`; JIT packages `cmake/external_projects/triton_kernels.cmake:1-28` | [cuda_backend.cu:20](../src/vt/cuda/cuda_backend.cu#L20), [CMakeLists.txt:37](../CMakeLists.txt#L37), [TritonAOT.cmake:57](../cmake/TritonAOT.cmake#L57) | [CUDA backend tests](../tests/vt/test_cuda_backend.cpp#L31); the Triton GDN AOT tree is now vendored per-arch — `sm_121a` (RUNTIME-VERIFIED) plus `sm_80/86/89/90a/100a` (DERIVED+BUILD-VERIFIED, `CLAIM-TRITON-AOT-PER-ARCH`, 2026-07-28), each a real per-target cubin (`cuobjdump sm=80/86/89/90/100`); the CUDA fast-path arch fan-out (non-GDN) remains as evidenced in `backend-matrix.md` | [inventory](specs/kernel-family-inventory.md); [per-arch Triton AOT](specs/triton-aot-per-arch.md) | `ANCHOR-BACKFILL` | - | @@ -213,8 +213,8 @@ were removed while their specs and same-tool traces remain. Lifecycle stays ## Count invariants - This table has exactly 52 practical kernel-family rows. -- Lifecycle counts are 11 `ANCHOR-BACKFILL`, 3 `READY`, 5 `PARTIAL`, - 7 `ACTIVE`, 1 `GATING`, 4 `DONE`, 12 `INVENTORIED`, and 9 `SPIKE`. +- Lifecycle counts are 11 `ANCHOR-BACKFILL`, 2 `READY`, 5 `PARTIAL`, + 8 `ACTIVE`, 1 `GATING`, 4 `DONE`, 12 `INVENTORIED`, and 9 `SPIKE`. - The distinct `KERNEL-CPU-A76-Q8-DOT` compiler/assembly family (`GATING`, PR #79) was added on 2026-08-06; `scripts/check-agent-record.py` pins the total row count. diff --git a/.agents/specs/cpu-a76-bf16-gemm.md b/.agents/specs/cpu-a76-bf16-gemm.md index c919ddfb9..3aec5ba15 100644 --- a/.agents/specs/cpu-a76-bf16-gemm.md +++ b/.agents/specs/cpu-a76-bf16-gemm.md @@ -1,7 +1,7 @@ # Cortex-A76 BF16 elementwise GEMM: close the Raspberry Pi 5 llama.cpp gap **Row:** `KERNEL-GEMM-CPU-ELEM-A76` · **issue:** -[#284](https://github.com/mudler/vllm.cpp/issues/284) · **state:** `READY` · +[#284](https://github.com/mudler/vllm.cpp/issues/284) · **state:** `ACTIVE` · **parent:** `KERNEL-GEMM-CPU-ELEM` · **target:** Raspberry Pi 5 Cortex-A76 r4p1, four cores, AArch64 NEON + DotProd, no i8mm · **comparison source:** official llama.cpp b9892 at `ee445f93d8a0a5033a46d1960e901ef5caec9a41`. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 2583b67e8..0d2bb4d40 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -248,7 +248,7 @@ host mirror is freed once the device Marlin resident is built. | 64-token Qwen model gate | Byte-identical across x86, portable, SDOT and assembly arms; asm vs SDOT median TTFT -1.55%, TPOT neutral, E2E -0.13%; vs portable TTFT -33.40%, E2E -2.67%. Cortex-A76+DotProd selects assembly by default | | Same-file llama.cpp floor (pp17/tg64) | **NOT MET on speed**: prefill 12.81 vs 27.77 tok/s (0.461x), decode 2.55 vs 3.91 (0.653x), E2E 26,018.39 vs 16,998.49 ms ([competitor evidence](bench-evidence/rpi5-a76-llamacpp-20260806.md)) | | Peak RSS | **2.841 vs 3.747 GiB, 24.2% less**; 3 clean unthrottled reps; same-text 64-token greedy output byte-identical after trailing-newline normalization | -| `READY` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Fresh both-engine profile, then C++/NEON BF16 GEMM speed closure; assembly remains blocked until C++ exceeds llama.cpp on every speed axis | +| `ACTIVE` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Fresh both-engine profile, then C++/NEON BF16 GEMM speed closure; assembly remains blocked until C++ exceeds llama.cpp on every speed axis | Same GGUF file both arms, `dgx.casa` GB10 aarch64 (20 cores), idle, 3 reps, llama.cpp `237ad9b96` built fresh on the same host. diff --git a/docs/STATUS.md b/docs/STATUS.md index 8ea2aa18a..8410065f3 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -110,7 +110,7 @@ token-for-token correctness against the pinned oracle. | GGUF loading (F32/F16/BF16/Q4_0/Q8_0/Q2_K/Q3_K/Q4_K/Q5_K/Q6_K/IQ2_XXS/IQ3_XXS/IQ2_S/MXFP4/NVFP4) | Supported; compute-in-quant (keep-quant) on CPU AND now CUDA for the six K-block encodings PLUS Q2_K/IQ2_XXS/IQ3_XXS (DeepSeek-V4 W8, 2026-07-29 - the FIRST CUDA keep-quant GGUF k-quant GEMM `KERNEL-QUANT-CIQ-GEMM-CUDA`, MMVQ-style dequant-in-kernel, GB10-gated 92401/92401 vs the CPU oracle, so a CUDA runner keeps blocks compressed and dots them on the GPU instead of the ARM cores); **NVFP4 now COMPUTES IN FP4 on CUDA for the dense-MLP and full-attention projections (2026-07-29, `CLAIM-GGUF-NVFP4-COMPUTE`), no longer materialize-only** | Weights in six block encodings stay compressed from file to matmul on CPU (no BF16 expansion). NVFP4 (ggml type 40) DEQUANTIZES, including the per-tensor (per-expert) `.scale` sidecar the container keeps outside the blocks; gated BIT-EXACT against the compressed-tensors NVFP4 path on real Qwen3.6-27B bytes from both containers. **It no longer expands to bf16 on CUDA:** an NVFP4 matmul/expert weight is REPACKED at load into the same (`weight_packed [N,K/2]`, `weight_scale [N,K/16]`) operand pair the compressed-tensors path produces - a pure byte permutation, gated BYTE-IDENTICAL against that container - and the existing `vt::MatmulNvfp4*` kernels run on it, so no new kernel exists and no numerics are re-derived. Covers the dense MLP + full-attention q/k/v/o and the MoE shared/routed experts; the GDN `in_proj_*` family and `ssm_out` still expand (the V-head reorder rewrites their layout) and a CPU build still expands everything - the documented `part` subset. **MEASURED GB10 (2026-07-29), same-binary A/B, 2 reps/arm:** peak RSS **50.8 -> 25.7 GiB**, load-and-generate **1:58 -> 0:41**; the 256 moved projections cost 35 840 MiB expanded vs 10 080 fp4-resident (3.56x). **The safetensors-sibling divergence CLOSES:** the fp4 arm is token-IDENTICAL over 24 greedy tokens where the same binary's bf16 arm diverges at index 4. REPORTED, not gated: the containers are not the same model - the GGUF NVFP4-quantizes 192 GDN `in_proj` tensors the safetensors keeps BF16 (mean rel. weight error ~0.18) and their activation global scales differ, so identity is not guaranteed and a cross-container throughput arm is invalid. SACRED gates unmoved: `test_qwen27_paged_engine` 235/235, `test_qwen36_paged_engine` 315/315. **The MoE (35B) stacked-expert arm is HARDWARE-GATED too (2026-07-29)**: the real 35B A3B NVFP4 GGUF loads and generates through the fp4 path, its 120 routed-expert stacks x 256 experts repack to the modelopt safetensors' operands with ZERO differing bytes over 840 sampled (tensor, expert) slabs, and all 840 per-expert `.scale[e]` are bit-identical to that expert's `weight_scale_2` - the scale INDEXING, mutation-proved against a `scales[0]`-for-all and an expert-0-slab-for-all mutant. Same-binary A/B: peak RSS 68.5 -> 22.7 GiB (3.01x), load-and-generate 1:51.9 -> 0:28.8, tokens IDENTICAL (the 35B routed experts run the W4A16 grouped GEMM in both arms). Recorded OPEN: this case's 24-token greedy stream is NOT run-to-run stable (1 of 3 `use_a16` and 1 of 4 reference runs differed), so the binding results are the weight-level byte identity and the residency audit, not token-exactness; `test_qwen36_paged_engine` is token-exact at ITS engine params, so the instability belongs to this case's configuration and attributing it is owed work. It also FIXED a latent defect the MoE arm made reachable: the two fp4 fused MoE blocks issued the router GEMM assuming the safetensors `[K,N]` gate layout and threw `matmul: inner dims mismatch` on the GGUF's `[N,K]`; `MoeRouterLogits` now branches on `nk` (inert for safetensors, SACRED unmoved). **Q2_K (id 10) + IQ2_XXS (id 16) DEQUANTIZE (2026-07-29, `CLAIM-DSV4-GGUF-LOADER`):** the ~2-bit types the single-Spark `DeepSeek-V4-Flash-GGUF UD-IQ2_XXS`/`UD-Q2_K_XL` vehicles use, ported 1:1 from llama.cpp `ggml-quants.c` (`iq2xxs_grid` codebook + signs; Q2_K nibble sub-scale/min), unit-gated on hand-derived bytes (`test_gguf_dequant` 15/15). Dequant-only (no vec_dot -> expand-bf16). A V4-GGUF model cannot RUN yet: the name map (tensor-manifest-blocked) + the V4 forward (W3-W8) remain. **Multi-shard split GGUF READING (2026-08-03, `CLAIM-GGUF-SPLIT-SHARDS`):** `GgufFile::Open` now transparently stitches llama.cpp `gguf-split` shards (`...-00001-of-00003.gguf`) — every shard mmap'd, tensor tables merged, KV metadata from shard `00001`, sibling mappings kept alive by the primary so keep-quant mmap-borrows stay valid across shards (`OwnsSpan` is shard-aware); `VT_GGUF_NO_SPLIT=1` opts out; unit-gated (`test_gguf` split-merge / no-split / count-mismatch cases, 33/33 local). This unblocks the real 3-shard `unsloth/DeepSeek-V4-Flash-0731 UD-IQ2_M` (~91 GiB), whose layout is the NATIVE `deepseek4` arch — per-block `ffn_gate_tid2eid` hash tables (hash layers 0/1/2) + `hc_*` MHC + DSA compressor/indexer are all PRESENT (name-map 1328/1328), `vocab_size` derives from `token_embd` — NOT a standard llama.cpp conversion, so no loader-layout change is owed. It now loads THROUGH 1324/1328 tensors; the sole remaining gap is 4 routed-expert slabs quantized with IQ2_S (id 22, ×2) + MXFP4 (id 39, ×2) — encodings we have GGUF block traits for but no keep-quant vec_dot, so they hit the expand→dequant path which lacks them. Expanding those 4 expert tensors to bf16 would add ~17 GiB (~106 GiB total → OOM-reboot risk), so the memory-safe fix is an IQ2_S+MXFP4 keep-quant kernel (CPU dequant dispatch + `iq2s_grid` + CUDA `DotSuperblock`), spec'd as the next brick **IQ2_S (id 22) + MXFP4 (id 39) DEQUANTIZE + KEEP-QUANT on CPU (2026-08-03, `CLAIM-DSV4-UDIQ2M-QUANT`, off-GPU):** the extra per-tensor "dynamic" encodings the `unsloth/DeepSeek-V4-Flash-GGUF UD-IQ2_M` checkpoint mixes into its last routed-expert slabs (IQ2_S `ffn_gate/up` dotting Q8_K, MXFP4 `ffn_down` dotting Q8_0) — ported 1:1 from llama.cpp `ggml-quants.c` @ 237ad9b96 (`iq2s_grid` 1024-entry codebook + DIRECT sign bytes; MXFP4 `kvalues_mxfp4` + `e8m0_to_fp32_half` micro-scaling, distinct from the compressed-tensors `E8M0ToF32` NVFP4 path). CPU dequant + keep-quant `vec_dot`, unit-gated on hand-derived golden bytes (`test_gguf_dequant` 17/17), an INDEPENDENT f64 dequant-then-dot + GEMM NMSE (`test_ops_quant_dot` 19/19), and keep-quant routing (`test_gguf_keep_quant` 37/37) — all CPU-green, so UD-IQ2_M's four previously-`unsupported ggml type 22/39` slabs now load COMPRESSED (no ~17 GiB bf16 expansion that OOM-reboots the box). CUDA: the IQ2_S device `DotSuperblock` is wired into the Q8_K grouped-MoE GEMM and now **CUDA-BUILT + LINKED on GB10 (sm_121a, CUDA 13.0, `-Werror`, 2026-08-03 integration)** — it compiles clean and the merged binary links; MXFP4's device dot (`DotMXFP4`) is written but NOT wired (Q8_0-activation needs a separate 32-block GEMM) so it is marked `[[maybe_unused]]` to keep the ready math without tripping nvcc #177-D, and on GPU MXFP4 CPU-fallbacks like Q4_0/Q8_0. The V4-GGUF forward + a real UD-IQ2_M GPU load/coherence run are owed | | AWQ / GPTQ quantization | W0 spike + W1 CPU INT4 dequant primitive; not yet loadable end to end | INT4 unpack+dequant-to-bf16 for BOTH community formats, mirroring vLLM 1:1 (AWQ reverse-order `awq_triton.py`; GPTQ `qdq_4.cuh` with zero_offset v1/v2 + act-order g_idx). Unit-gated RED-first (hand-computed known bytes + double-precision roundtrip). NOT wired to a loader, no GPU Marlin compute, no model run yet: config recognizer (W2), Marlin GPU GEMM riding the vendored NVFP4 Marlin (W4), CPU e2e (W3), GPTQ 8/2/3-bit (W5) and MoE (W6) are named next bricks. See [.agents/specs/awq-gptq-quant.md](../.agents/specs/awq-gptq-quant.md) | | MXFP4 (compressed-tensors `mxfp4-pack-quantized`) | Compute PROVEN (#38); GQA-swap ON (#49); decode-graph+gate_up FUSION default-ON. `VT_MARLIN_DENSE` DEFAULT-ON (`KERNEL-MARLIN-DENSE-EXEC`): dense marlin 48-CTA byte-faithful (32B 0.000, 263/263), binding beats #51 every axis (c1 1.020, c8 0.969, mem 2.63x). **`QUANT-CT-MXFP4-FINAL-STACK` TERMINAL — both last levers exhausted: num_splits cap `VT_FA2_NSPLITS_CAP` gated-OFF (c1-only, self-corrects@c8; 32B strict char-identical); glue folds via `vt::FusedChain`; `FLASH-AUDIT` #68: c8 flash gap +12.5us/call is occupancy/L2-bound; `-use_fast_math` TRIED, REGRESSES flash (168.8→189.8), rejected. c1 1.020x PASS, c2-c8 0.962-0.969.** state.md | Shared with DeepSeek-V4-Flash + Kimi-K3 MXFP4 paths. CPU E8M0 dequant 5/5·1142. GPU W4A4 + MoE-expert e2e later | -| CPU backend vs llama.cpp | 20-core at floor; RPi/A76 `GATING`; BF16 `READY` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Pi: 0.461x prefill / 0.653x decode; RSS 24.2% lower. C++/NEON before assembly. | +| CPU backend vs llama.cpp | 20-core at floor; Pi/A76 `GATING`; BF16 `ACTIVE` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Pi: 0.461x prefill / 0.653x decode; RSS 24.2% lower. C++/NEON before assembly. | | Paged KV cache + prefix caching | Supported | Block-paged full attention, hybrid full-attention + GDN state groups, automatic prefix caching (APC) on by default for dense models (cache-ON gated end to end: token-identical output, cache hits, faster TTFT) | | fp8 KV cache (`cache_dtype=fp8`) | In progress (W1 CPU brick), not yet usable end-to-end | HIGH-priority memory/throughput lever (halves the KV footprint). W0 spike + W1 CPU brick landed (`KV-FP8` ACTIVE): fp8-e4m3 K/V STORE (`Quantize(hp/scale)`) + the paged-attention READ dequant (`Dequant(fp8)*scale`) + the `cache_dtype` config parse, all CPU-gated RED-first (`test_ops_fp8_kv_cache` 8/8·511; a wrong store direction fails 3/480). Storage is 1-byte fp8 (`DType::kI8`) + a `Fp8KVCacheDataType` interpretation enum, per-tensor k/v scales (mirroring vLLM `BaseKVCacheMethod`). The CUDA store + fp8 paged-attention read (the GPU memory-halving path, DGX-blocked), the runner/spec integration (half-sized KV blocks + checkpoint-scale threading + `--kv-cache-dtype`/`--calculate-kv-scales`), fp8_e5m2 and per-head scales are named W2-W5 in [.agents/specs/fp8-kv-cache.md](../.agents/specs/fp8-kv-cache.md). No model can run with an fp8 KV cache yet | | Prefix-cache matching unit (`--prefix-match-unit`) | Partial (resolver landed, config/scheduler wiring pending) | 0.26-new knob setting the finest token boundary a prefix-cache hit can land on (the `hash_block_size`). W1: `resolve_kv_cache_block_sizes` ported 1:1 (hybrid `hash_block_size = prefix_match_unit if set else gcd(group block sizes)`; single-group inert; back-off on no-consumer / mamba-non-align; throws on non-divisible), CPU unit-gated RED-first (default gcd vs `=16`). Pending: the config/CLI/ABI field (W2), scheduler threading of a finer-than-block matching unit (W3, needs the KV-block-pool align path), and the benchmark (W4). Default path byte-identical (dense single-group models ignore it). | From 3260b765162adf734d25b8db69b29861cb979163 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 18:32:13 +0100 Subject: [PATCH 03/11] spec(cpu): define benchmark thread-budget control (#293) FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/NOW.md | 2 +- .agents/roadmap_v1.md | 1 + .agents/specs/cpu-bench-thread-budget.md | 173 +++++++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 .agents/specs/cpu-bench-thread-budget.md diff --git a/.agents/NOW.md b/.agents/NOW.md index f4b6dc4c4..68dba5245 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -22,7 +22,7 @@ Work: 7 external PRs landed (6 merged + #227 repaired). | Kimi-Linear-48B | 122/128 held; e2e NOT ESTABLISHED | tiktoken-only ckpt: no warm server | | 35B mid-band | **canonical 0.918-0.972x** c1-c32 (@`348c265d`, first c16/c32) | Decode-only window, ONE tool | | Qwen3.5-4B sm_120 | tput **1.0283x** `PENDING`; TTFT/TPOT/E2E 1.085/1.017/1.029x | Profile wave | -| RPi5 A76 CPU | **#284 ACTIVE**: 0.461x pf, 0.653x dec | Reprofile; C++ BF16 GEMM | +| RPi5 A76 | **#293 READY**: T4 contention | Blocking control; then F16 | | SERVE-ASYNC-MIRROR | **#323 FIXED** (mitigation): graph declines while the mirror is live; 7/7 async gates | Graph read ids at REPLAY | | CPU levers (`QUANT-GGUF-CIQ-GEMM`) | Profile DONE: decode **47% threadpool sync**, prefill **~39% paged attn** | Parakeet encoder; attn dtype hoist | | `SERVE-METRICS` async (#277) | **`/metrics` was DEAD on the shipped server**: AsyncLLM folded nothing. Now live, ctest 366/366 | Config-gated families | diff --git a/.agents/roadmap_v1.md b/.agents/roadmap_v1.md index f68ad4b85..46f0d396b 100644 --- a/.agents/roadmap_v1.md +++ b/.agents/roadmap_v1.md @@ -39,6 +39,7 @@ issue is not yet placed. Keyed record: update in place, never append. | [#269](https://github.com/mudler/vllm.cpp/issues/269) | `BACKEND-ROCM` | ROCm gfx1200 (RX 9060 XT): Qwen3-0.6B produces wrong greedy output despite all-native execution — embedding gather cleared | bug | | [#305](https://github.com/mudler/vllm.cpp/issues/305) | `KERNEL-SSM-MAMBA` | GDN causal-conv: the `conv_state` initial-state read races the final-state write across blocks (`VT_CONV_REG` + exact chunks, both default ON) | bug | | [#206](https://github.com/mudler/vllm.cpp/issues/206) | `KERNEL-SSM-MAMBA` | RTX 5070 Ti: close Qwen3.5-4B TTFT, TPOT and VRAM gaps vs vLLM — owns the sm_120 post-conv token tile and the K=4 causal-conv arm (PR #155) | feature | +| [#293](https://github.com/mudler/vllm.cpp/issues/293) | `SERVE-CLI-BENCH` | Stop benchmark AsyncLLM polling from stealing a CPU worker core | perf | | [#268](https://github.com/mudler/vllm.cpp/issues/268) | `MODEL-MM-muse-glimmer-muse-glimmer-for-conditional-generation` | Muse Glimmer (Meta, 30B agentic multimodal): text tower, perception encoder, DFlash drafter | feature | | [#329](https://github.com/mudler/vllm.cpp/issues/329) | `MODEL-MM-muse-glimmer-muse-glimmer-for-conditional-generation` | Muse Glimmer GGUF k-quants: text arm lands; mmproj blocked by a converter axis drop | feature | | [#333](https://github.com/mudler/vllm.cpp/issues/333) | `MODEL-MM-muse-glimmer-muse-glimmer-for-conditional-generation` | Muse Glimmer speed: no number on any axis; benchmark vs llama.cpp, HF, and ourselves | perf | diff --git a/.agents/specs/cpu-bench-thread-budget.md b/.agents/specs/cpu-bench-thread-budget.md new file mode 100644 index 000000000..2d6d5d891 --- /dev/null +++ b/.agents/specs/cpu-bench-thread-budget.md @@ -0,0 +1,173 @@ +# CPU benchmark frontend wait + thread budget + +**Row:** `SERVE-CLI-BENCH` · **issue:** +[#293](https://github.com/mudler/vllm.cpp/issues/293) · **leaf state:** `READY` · +**trigger:** Raspberry Pi 5 W0 refresh for `KERNEL-GEMM-CPU-ELEM-A76` at +`3520b7f7f` · **target:** current-source causal control first, C++ only. + +## Scope + +Determine whether the closed-loop benchmark frontend steals CPU time from the +engine on core-count-constrained hosts, then remove only the proven overhead. +The first binding vehicle is Qwen3.5-2B Q8_K_XL on the four-core Raspberry Pi 5. + +In scope: + +- a same-current-source control that distinguishes benchmark polling from the + engine, output-handler and compute-worker thread budget; +- an event-driven "any request has output" wait for the deterministic + multi-request benchmark loop if polling is proven causal; +- explicit, auditable benchmark output-wait selection and same-binary rollback; +- a CPU compute-thread budget change only if a blocking frontend control shows + that non-frontend engine threads still require a reserved core; +- focused synthetic tests, QEMU AArch64 build/test, Pi PMU/profile and recursive + full-model measurement. + +Out of scope: + +- kernel arithmetic, model/loader, CUDA/Vulkan, C ABI or service changes; +- assembly; +- changing request order, DELTA coalescing, metric definitions, prompts, + sampling or correctness thresholds; +- compiling on the Pi or treating the old-binary comparison as causal proof. + +## Upstream chain + +Pinned vLLM `555967922` uses one `asyncio.Event` per +`RequestOutputCollector`: `output_processor.py:45-96`. `put()` sets the event; +`get()` waits; `get_nowait()` is attempted once before awaiting. Its +`AsyncLLM.generate` loop at `async_llm.py:525-587` drains an already-ready +value without a task switch, otherwise suspends at `await q.get()`. + +Our collector mirrors that per-request behavior with +`std::condition_variable` in `src/vllm/v1/engine/output_processor.cpp:24-70`. +Single-request `AsyncLLM::generate` already does one nowait attempt then blocks +(`src/vllm/v1/engine/async_llm.cpp:182-240`). The original C++ benchmark needs +deterministic multi-request admission, so `examples/bench/bench_core.h:535-580` +scans all collectors and calls `std::this_thread::yield()` when none is ready. +That final policy is original harness code, not an upstream mirror. + +The CPU pool itself mirrors llama.cpp: worker 0 is the dispatching caller and +only workers 1..N-1 are spawned (`cpu_threadpool.cpp:74-87`). Therefore a pool +width of four does use four compute threads, but AsyncLLM and its benchmark +consumer remain additional runnable threads. + +## Our baseline + +The #284 W0 diagnostic at current source `0e4a1c13d` found: + +| Arm | Result | Interpretation | +|---|---|---| +| current benchmark, CPU threads=4 | 5.844 prefill / 1.368 per-stream decode tok/s; 48.70 s E2E; exact tokens | diagnostic only; TTFT spread 15.268% | +| same current binary, CPU threads=3 | 2.53 per-stream decode tok/s; 26.96 s E2E; exact tokens | proves a runnable-thread/core-budget interaction, not which frontend thread causes it | +| old driver at threads=4 | 26.03 s E2E | historical control only; 141 relevant files differ, so not a one-variable arm | +| current matched profile | Barrier 42.21%, reached F16 `Bt16Neon` 33.09%, Q8 12.05% | reranks the BF16 kernel hypothesis; does not isolate polling | + +Raw evidence is on the Pi under +`~/vllm-cpp-assembly/evidence/a76-bf16-20260810/`. The old/new comparison may +suggest frontend contention, but only a current-source output-wait control can +establish it. + +## Port map + +| Upstream / reference | Local surface | Adaptation | +|---|---|---| +| `RequestOutputCollector.ready = asyncio.Event()` and `await ready.wait()` | existing collector `std::condition_variable` | retain per-request semantics unchanged | +| `get_nowait() or await get()` per generator | `AsyncLLM::get_output_nowait/get_output` | direct current-source blocking control at concurrency 1 | +| asyncio task scheduler wakes whichever request event completes | additive AsyncLLM output epoch + condition, or equivalent any-ready wait | one shared wake only for the deterministic C++ multi-request harness; no spin/yield loop | +| vLLM/llama CPU thread-count behavior | `VLLM_CPP_CPU_THREADS` and `Threadpool::Global()` | alter default budget only if event-driven T4 still loses to T3 and matched profiles attribute the residual to noncompute host threads | + +## Design and discriminator + +1. Add an auditable benchmark output-wait variant. `poll` is the exact current + scan+yield loop. `blocking-c1` is legal only at concurrency 1 and blocks on + the sole active collector after the same initial nowait attempt. Both run in + one binary with identical requests and metrics. +2. Interleave `poll`/`blocking-c1` on the Pi at CPU threads 4 and 3. If + blocking-c1 restores T4 while T3 stays neutral, polling is causal. +3. Only after that positive discriminator, implement `event`: a monotonic + output epoch plus condition wait notified after AsyncLLM publishes outputs. + The benchmark snapshots the epoch before scanning and waits only if the + epoch is unchanged, preventing the scan-to-wait lost-wakeup race. The wait + must also wake on engine death/shutdown. +4. If blocking-c1 is neutral, reject the polling hypothesis. Profile + output-handler/core/compute threads separately; test a reserved-core budget + without changing frontend policy. Do not implement the event seam merely + because it is aesthetically preferable. +5. Promote a default only when full-model T4 improves credibly, T3 and x86 do + not regress, token IDs are identical, and TTFT/ITL measurement remains + faithful. Keep the old arm as an explicit rollback. + +## Tests to port + +There is no upstream multi-request C++ benchmark test. Port the load-bearing +upstream event semantics and extend the local original-harness tests: + +- collector wait returns only after `put`, propagates errors and never loses a + wake between nowait and wait; +- `blocking-c1` rejects concurrency greater than one by name; +- `event` completes multiple requests in deterministic submission order, + preserves every DELTA/token and does not busy-loop while no output exists; +- engine death wakes the wait and throws the existing `EngineDeadError`; +- mutate/remove the epoch increment or notification and prove the bounded test + fails for the intended timeout/unfinished-output reason; +- `poll`, `blocking-c1` and `event` produce byte-identical token-ID fixtures. + +## Gates + +1. RED-first focused test/mutation, then the minimum current-source control. +2. `test_bench`, `test_async_llm`, `test_output_processor`, + `test_engine_core_proc` and threadpool tests all green; bounded timeouts must + diagnose a missing wake rather than hang. +3. Build AArch64 Release locally through `docker/Dockerfile.arm64` using + buildx/QEMU; run focused tests under QEMU. QEMU timings are void. +4. Deploy hash-identified artifacts to the user-authorized disposable Pi. + Record CPU affinity, governor, frequency, temperature, throttle, contenders + and PMU lost-sample count for each interleaved arm. +5. Correctness: exact token IDs across every same-binary arm and against the + accepted current baseline. +6. Performance: three or more interleaved clean reps, sub-3% spread for a + binding claim. First close the current-main regression; then resume the + same-file llama.cpp floor. No micro/harness-only win earns model credit. +7. Native non-A76 CPU gate and all record/ready gates pass. + +## Dependencies + +- `SERVE-CLI-BENCH`, `SERVE-ASYNC-LLM`, `ENG-CORE-BUSY-LOOP` and + `QUANT-GGUF-CPU-THREADPOOL` are existing prerequisites. +- Issue #284 W0 commit `3520b7f7f` supplies the diagnostic, after its review + findings narrow the causal wording. +- Official llama.cpp b9892 `ee445f93d` is the speed denominator; pinned vLLM + supplies collector behavior. +- The model already exists on the Pi. No download or service change is owed. + +## Work breakdown + +| W | Deliverable | State | +|---|---|---| +| C0 | Correct #284 evidence wording and register #293 | ready | +| C1 | Same-current-source `poll` vs `blocking-c1` control + RED mutation | pending | +| C2 | If C1 positive, general event/epoch wait + deterministic multi-request tests | blocked by C1 | +| C3 | QEMU gate and interleaved Pi T4/T3 causal profile | blocked by C1 | +| C4 | Retain event wait or reject it; test reserved-core budget only if C1 is negative/insufficient | blocked by C3 | +| C5 | Full model versus llama.cpp; return to the measured F16/kernel ranking | blocked by C4 | + +## Risks/decisions + +- Blocking an arbitrary request at concurrency greater than one can delay an + already-ready peer and distort ITL. That is why `blocking-c1` is a causal + control, not the general implementation. +- A global condition without an epoch has a lost-wakeup race. The mutation gate + must cover the scan-to-wait boundary. +- Yielding is not sleeping; on four runnable compute threads it may consume a + timeslice despite making scheduler progress. This remains a hypothesis until + C1. +- Reducing compute workers can improve an oversubscribed benchmark while + lowering a properly isolated kernel ceiling. Thread-budget and wait-policy + candidates remain separate experiments. +- Any remaining F16 GEMM optimization belongs to the A76 elementwise row only + after this harness/control issue is resolved and a fresh profile ranks it. + +## Outcome + +Pending C1 current-source discriminator. From b900f3c33c3753fd2348c465b5174e649c871737 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 20:54:34 +0100 Subject: [PATCH 04/11] docs(cpu): record corrected A76 W0 reranking (#284) Transplant the reviewed Pi profile onto current upstream without treating the historical binary or T4/T3 arm as a one-variable polling control. Record the prompt-content mismatch and link the separately tracked #293 discriminator. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/coordination.md | 2 +- .agents/kernel-matrix.md | 2 +- .agents/specs/cpu-a76-bf16-gemm.md | 43 +++- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- .../rpi5-a76-bf16-w0-20260810.md | 204 ++++++++++++++++++ .../rpi5-a76-llamacpp-20260806.md | 5 +- scripts/check-public-doc-tables.py | 6 +- 8 files changed, 254 insertions(+), 12 deletions(-) create mode 100644 docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md diff --git a/.agents/coordination.md b/.agents/coordination.md index e9a1db063..3bdaecc9a 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -1635,7 +1635,7 @@ commit order. | Claim | Row IDs | Agent | Worktree / remote dir | Branch | Owned scope | State | Last update | |---|---|---|---|---|---|---|---| -| `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | `KERNEL-GEMM-CPU-ELEM-A76` (`ACTIVE`) | Codex (gpt-5.6-sol), fresh helper implementer | isolated local worktree `/tmp/vllm-a76-bf16-impl`; build/test under buildx/QEMU only; execute/PMU on user-authorized disposable `rich@rpi5fan.lan` | `helper/KERNEL-GEMM-CPU-ELEM-A76-W0-W2`, base spec commit `999e1221e` plus this claim commit | Issue #284 W0-W2 only: refresh both-engine baseline/profile; add the BF16 elementwise benchmark/selection mutation proof; audit disassembly and retain only byte-exact C++/NEON changes with recursive measurements. Owns the new spec/row and scoped edits to `examples/cpu_kernel_bench/main.cpp`, its CLI test/CMake only if needed, `src/vt/cpu/cpu_matmul_elem*`, `src/vt/cpu/cpu_ops.cpp` only if profiling proves partitioning, focused tests, and indexed Pi evidence/record projections. Excludes assembly, model/loader/CUDA/Vulkan/C ABI changes, building on the Pi, service changes and PR/push. | `ACTIVE` | 2026-08-10 — spec committed; helper worktree creation and fresh W0 measurement are next | +| `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | `KERNEL-GEMM-CPU-ELEM-A76` (`ACTIVE`) | Codex (gpt-5.6-sol), fresh helper implementer | isolated local worktree `/tmp/vllm-a76-bf16-impl`; build/test under buildx/QEMU only; execute/PMU on user-authorized disposable `rich@rpi5fan.lan` | `helper/KERNEL-GEMM-CPU-ELEM-A76-W0-W2`, base spec commit `999e1221e` plus this claim commit | Issue #284 W0-W2 only: refresh both-engine baseline/profile; add the BF16 elementwise benchmark/selection mutation proof; audit disassembly and retain only byte-exact C++/NEON changes with recursive measurements. Owns the new spec/row and scoped edits to `examples/cpu_kernel_bench/main.cpp`, its CLI test/CMake only if needed, `src/vt/cpu/cpu_matmul_elem*`, `src/vt/cpu/cpu_ops.cpp` only if profiling proves partitioning, focused tests, and indexed Pi evidence/record projections. Excludes assembly, model/loader/CUDA/Vulkan/C ABI changes, building on the Pi, service changes and PR/push. | `DONE` | 2026-08-10 — W0 complete and reranked before code: current AsyncLLM T4/T3 proves a runnable-thread/core-budget interaction but does not isolate polling; the old binary is historical only. W1-W4 await the #293 discriminator and fresh profile; no kernel/fixture change retained; row stays `ACTIVE`. | | `CLAIM-SAMPLE-PROMPT-LOGPROBS-W1` | `SAMPLE-PROMPT-LOGPROBS` (`ACTIVE`; W1 runner source only) | Claude Code (claude-opus-5) | `/home/mudler/_git/vllm.cpp-prompt-logprobs`; CPU-only `build-gate` (`-DVLLM_CPP_CUDA=OFF` Release) — NO GPU, NO download, claims no benchmark device | `row/SAMPLE-PROMPT-LOGPROBS`, opened on `origin/main` `bd6b3936`, rebased onto `8a6704a2` for the 2026-08-10 review repair and onto `e63d11d3` to land, issue #223 | The runner-side prompt-logits source ported from `_get_prompt_logprobs_dict`. Owns ONLY: the `prompt_logprob_rows`/`prompt_logprob_indices` block in `prepare_inputs.{h,cpp}`, `collect_prompt_logprobs` + `in_progress_prompt_logprobs_` in `runner.{h,cpp}`, `Sampler::compute_prompt_logprobs` in `sampler.{h,cpp}`, `num_prompt_logprobs` in `input_batch.{h,cpp}`, section 9 of `tests/vllm/v1/test_llm_engine.cpp`, this claim, the row cells, `.agents/NOW.md`, `.agents/roadmap_v1.md` issue rows, one `docs/STATUS.md` sentence, and the `prompt_logprobs` paragraph in `docs/USAGE.md`. **NON-COLLISION:** NO kernel, NO vt op, NO ABI, NO CMake, NO model file — the full-logits route reuses the existing `VT_LOGITS_GATHER=0` path rather than widening any model's gather contract. EXCLUDED: the OpenAI `echo` serialization (W2) and issue #231, both split out. | `ACTIVE` | 2026-08-10 — W1 landed then REPAIRED after a fresh review returned `VERDICT: FAIL`. Review found a REAL bug: the inertness guard keyed on `prompt_logprob_indices` while the full-logits assertion keyed on `prompt_logprob_rows`, so a zero-row final chunk (the exact-prefill edge, `:5668-5673`) beside another multi-token request threw `VT_CHECK` out of `engine.step()` and killed the whole batch. Fixed by moving the check inside the `num_rows > 0` slice; regression §8(h), RED-first (the throw). Second finding: the on-vs-off inertness case could not see a change to the SHARED route (forcing full logits on every step left it 17/17·346 green), so §8(g) now asserts the route DECISION through a const runner seam and fails under that exact mutation. Third finding recorded, NOT fixed and NOT claimed: the full-logits route hands the sampler a host pointer wearing the device label — sound on unified memory, unverified on CUDA; spec risk 4 + a `PENDING` CUDA smoke gate, and `docs/USAGE.md` narrowed to say so. Three latent lifetime divergences recorded as risks 5-7. Gates on the rebased tree (`e63d11d3`): clean CPU Release build 0 warnings under `-Werror`, `test_llm_engine` **21/21 · 384**, full `ctest -j 6` **365/366** with `test_openai_conformance` (parallel starvation — connection `-1`s) passing serially 1/1 in 0.39 s. Row stays `ACTIVE`: W2 `echo` serialization is the named residual, and the CUDA smoke gate is `PENDING`. | | `CLAIM-MOONCAKE-STORE` | `KV-MOONCAKE-STORE` (new, `SPIKE`) | Claude Code (opus-5), helper role | isolated worktree `/home/mudler/_git/vllm.cpp-mooncake`; records-only — NO build, NO GPU, NO download, NO external dependency installed | `row/KV-MOONCAKE-STORE`, base `origin/main` `848d4a87` | The `MooncakeStoreConnector` spike, records-only in this commit: NEW [`.agents/specs/mooncake-store-connector.md`](specs/mooncake-store-connector.md), the NEW `KV-MOONCAKE-STORE` engine-matrix row + section/total counters, the `ENGINE_ROWS` 146→147 bump in `scripts/check-agent-record.py` with its justification comment, a PROSE-ONLY Mooncake-disposition correction inside the `KV-CONNECTORS` row (both NOT-SCHEDULED sentences; that row's `ANCHOR-BACKFILL` state, tier, anchors, evidence, spec link and owner are UNCHANGED, so it is not claimed here), the `ROAD-V1-D4` portfolio note + canonical-table link, the roadmap issue-table row for [#287](https://github.com/mudler/vllm.cpp/issues/287), and this claim. **NON-COLLISION:** touches NO `src/`, `include/`, `tests/`, `examples/` or CMake path; the only script touched is the record checker's own row-count constant. | `ACTIVE` | 2026-08-10 — spec committed. The `KV-CONNECTORS` spike's blanket "Mooncake NOT SCHEDULED" conflated TWO connectors: `MooncakeConnector` (P2P prefill/decode over the Transfer Engine — two nodes, fabric, proxy) KEEPS that verdict; `MooncakeStoreConnector` (shared KV object store, the LMCache analogue) is reopened because (a) Mooncake is NATIVE C++ — `mooncake::Client` in `client_service.h`, and the `MooncakeDistributedStore` vLLM imports is a pybind wrapper over it — so we LINK instead of reimplementing a wire (the inverse of the LMCache cost shape), and (b) its single-node `protocol: "tcp"` + `mooncake_master` config is gateable on one box with NO RDMA NIC. The landed W5 `KVConnector` seam needs NO change. NEXT: W0, the go/no-go link spike (build Mooncake from source, pin the revision, drive `mooncake::Client` from a standalone C++ TU against a local master over TCP) — a genuine stop point per spec §S1. Speed is recorded as an OPEN axis: the RDMA/GPUDirect path that motivates the connector is unmeasurable for want of a fabric on any box we own. PENDING developer authority: the Mooncake source build, push/PR. | | `CLAIM-MUSE-GLIMMER-SPEC` | `MODEL-MM-muse-glimmer-muse-glimmer-for-conditional-generation` (`SPIKE`) | Claude Code (opus-5), helper role | isolated worktree `/home/mudler/_git/vllm.cpp-muse-glimmer`; CPU-only, records+spec+W0 scaffold — NO GPU, weights on the shared NAS | `row/MODEL-MUSE-GLIMMER`, base `origin/main`; PR [#279](https://github.com/mudler/vllm.cpp/pull/279) OPEN | The Muse Glimmer port. Spec `.agents/specs/muse-glimmer.md`, the `MODEL-MM-muse-glimmer-*` matrix row + checklist + rollup, issue-table entry for [#268](https://github.com/mudler/vllm.cpp/issues/268), porting-inventory §9 deviation 16 (the OFF-PIN anchor), `check-agent-record.py` MODEL count, the W0 scaffold (registry + config parse + weight name map + refuse-by-name forward, gate 11/11), `CHECKPOINT_ROOT` in `.env.example` + `.agents/environment.md`. Those helpers ALL LANDED onto the row branch: W1 the 52-layer text forward, W3 the 50-layer perception encoder, W4 the mm wiring (tower to adapter to projection to `perception_emb_norm` to the masked scatter; released-30B accounting 1436/1436), W7 the ATEM reasoning + tool parsers. The branch is no longer "records + spec + W0 scaffold" and its forward no longer refuses. `CLAIM-MUSE-GLIMMER-FIX` (`row/MODEL-MUSE-GLIMMER-FIX`) then closed the PR #279 review findings: three COVERAGE HOLES that mutation testing found green (`perception_emb_norm` untested, the whole non-FusedChain fallback arm unexercised, RoPE theta ungated at a realistic magnitude), the stale "OPEN FINDING" that had disarmed the real-weights accounting assertion, and the false claim in both parser headers that the seam has no `skip_special_tokens` (it does; channel scoping is an OPEN GAP at server defaults, spec §6.7). WHAT IS STILL NOT ESTABLISHED, and must not be implied: the full-depth 52-layer arm never ran (evidence is reduced depth 4/52, 5 prefill argmax positions, no generated tokens), multi-step decode is untested, the perception encoder has no reference check, and nothing has run through the server. **Anchored to the UNMERGED [vllm#51655](https://github.com/vllm-project/vllm/pull/51655) head `075d645af`** on explicit developer direction 2026-08-10: Muse Glimmer exists neither at the pin `555967922` nor on vLLM `main`. **No speed axis is claimable** until #51655 merges and the pin advances — the pinned oracle cannot load the model. | `ACTIVE` | 2026-08-10 | diff --git a/.agents/kernel-matrix.md b/.agents/kernel-matrix.md index 9929c017f..65c0a9fae 100644 --- a/.agents/kernel-matrix.md +++ b/.agents/kernel-matrix.md @@ -112,7 +112,7 @@ host/sched. Detail: state `KERNEL-FA2-GQA-SWAP-FLIP`. | ID | Item | Upstream | Our code | Tests/evidence | Spike/spec | State | Owner | |---|---|---|---|---|---|---|---| -| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); implementation pending | Same-file Pi floor: 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS; fresh profile + focused BF16 PMU fixture pending under [issue #284](https://github.com/mudler/vllm.cpp/issues/284) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `ACTIVE` | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | +| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); no issue-specific implementation retained | Accepted same-file Pi floor remains 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS. Fresh W0 diagnostic reranked before implementation: current AsyncLLM T4/T3 shows a runnable-thread/core-budget interaction without attributing polling; summed cycles rank barrier 42.21%, reached **F16** `Bt16Neon` 33.09%, Q8 12.05%. T3 restores decode but trades prefill; current TTFT spread is non-binding. W1-W4 await the [`SERVE-CLI-BENCH` #293 control](https://github.com/mudler/vllm.cpp/issues/293), its [spec](specs/cpu-bench-thread-budget.md), and a fresh profile ([W0 evidence](../docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md), [issue #284](https://github.com/mudler/vllm.cpp/issues/284)) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `ACTIVE` | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | | `KERNEL-CPU-A76-Q8-DOT` | Cortex-A76 Q8_0 x Q8_0 SDOT, exact-order C++ and scheduled AAPCS64 variants for the Pi decode/prefill keep-quant GEMM | llama.cpp `ggml/src/ggml-cpu/quants.c:400` portable dot and `arch/arm/quants.c:1076-1160` Arm DotProd @ project pin `237ad9b96`; Qwen semantics remain vLLM `555967922` | [ACLE SDOT + HWCAP/MIDR selector](../src/vt/cpu/cpu_quant_dot_sdot.cpp#L85), [AAPCS64 two-block leaf](../src/vt/cpu/cpu_quant_dot_a76.S#L10), [Q8 dispatch](../src/vt/cpu/cpu_quant_dot.cpp#L675), [same-binary variants](../examples/cpu_kernel_bench/main.cpp#L858) | [direct exact variant/edge tests](../tests/vt/test_ops_quant_dot.cpp#L593); QEMU 20/20, 150258 assertions; physical Pi exact checksums/tokens; assembly vs compiler SDOT +3.66% M1/T1, +5.08% M128/T1, +3.69% M128/T4, but −2.43% M1/T4. Same-file llama.cpp floor measured/NOT MET: vllm.cpp 0.461x prefill / 0.653x decode+E2E, 0.758x peak RSS (better), exact-prompt output equal; [assembly evidence](../docs/bench-evidence/rpi5-a76-q8-dot-20260806.md#L40), [competitor evidence](../docs/bench-evidence/rpi5-a76-llamacpp-20260806.md) | [A76 Q8 dot spec](specs/cpu-a76-q8-dot.md) | `GATING` | - | | `KERNEL-ACCEL-PROVIDER-SELECT` | **WHICH implementation of an op runs, when more than one exists on a device** — the selection layer above every kernel family in this matrix. Distinct from `KERNEL-CUDA-DISPATCH-AOT`, which is about which ARCH a CUDA kernel is compiled/selected for; this is about which PROVIDER (ours, a vendor library, MLX, llama.cpp) serves the op at all | no single upstream file: this is the shape vLLM's runtime chain uses everywhere — flashinfer tactic registries, cuBLASLt/CUTLASS per-call heuristics, and torch's backend selection — rather than compile-time pinning | `vt::OpProvider` [op_provider.h](../include/vt/op_provider.h) + [op_provider.cpp](../src/vt/op_provider.cpp); the flat `[OpId][DeviceType]` `void*` table it replaces is gone from [ops.cpp](../src/vt/ops.cpp) with the ~70 op wrappers untouched. Providers registered today: `vt-native` (every backend kernel in the tree, priority 0, unconditional — behaviour preserved exactly), `mlx` (priority 100, Metal `kMatmul`/`kMatmulBT`, build-gated `VLLM_CPP_MLX`, [metal_mlx_provider.mm](../src/vt/metal/metal_mlx_provider.mm)), and — NEW 2026-07-23 (`CLAIM-BACKEND-SEAM-S5-1`, work row `S5`) — **`vt-cpu-ref` (priority −1000, the portable reference tier)**: the CPU kernel installed LAZILY as a negative-priority fallback on a UNIFIED-MEMORY device's first `GetOp` miss, mirroring `custom_op.py:138 forward_native`, so a partial backend runs an op it lacks natively instead of throwing. Native always wins (priority); gated on `Backend::UnifiedMemory()` (a discrete GPU never gets it — a CPU kernel on true device memory is corruption); observable via `GetReferenceTierHits()` + a one-time loud stderr line | [test_op_provider.cpp](../tests/vt/test_op_provider.cpp) 11 cases / 47 assertions — deterministic selection under REVERSED registration order, name tie-break, duplicate rejection, capability predicate, caps re-resolution, decline-and-fall-back, stats, runtime disable; [test_metal_backend.cpp](../tests/vt/test_metal_backend.cpp) 9 cases / 108 assertions on the M4 with MLX ON, including MLX-vs-MSL-vs-CPU NMSE per op at real shapes and an end-to-end DECLINE; **[test_reference_tier.cpp](../tests/vt/test_reference_tier.cpp) (S5): discrete-device refusal + unified-device zero-native-kernel fallback correctness + native-wins + observability, hardware-free via a fake backend on `kXPU`.** Linux CPU 156/156; dgx regression set ALL UNCHANGED — anchor `tests/vt/test_op_provider.cpp:64` | [Metal/MLX reuse study §6](specs/metal-mlx-reuse-study.md); [accelerator-seam-audit §10](specs/accelerator-seam-audit.md); [drop-in kernel ABI](specs/dropin-kernel-abi.md) (the complementary ARGUMENT half) | `ACTIVE` — mechanism landed and gated with THREE provider kinds (`vt-native`, `mlx`, `vt-cpu-ref`); the CUDA/CPU/Vulkan vendor provider rows it was designed for are not yet populated (so the row is deliberately left open) | `CLAIM-BACKEND-ACCEL-PROVIDER-1` | | `KERNEL-CUDA-DISPATCH-AOT` | CUDA runtime, streams, graphs, per-arch dispatch, and generated/AOT artifact selection | platform selection `vllm/platforms/cuda.py:205-493`; gencode handling `CMakeLists.txt:201-220`; JIT packages `cmake/external_projects/triton_kernels.cmake:1-28` | [cuda_backend.cu:20](../src/vt/cuda/cuda_backend.cu#L20), [CMakeLists.txt:37](../CMakeLists.txt#L37), [TritonAOT.cmake:57](../cmake/TritonAOT.cmake#L57) | [CUDA backend tests](../tests/vt/test_cuda_backend.cpp#L31); the Triton GDN AOT tree is now vendored per-arch — `sm_121a` (RUNTIME-VERIFIED) plus `sm_80/86/89/90a/100a` (DERIVED+BUILD-VERIFIED, `CLAIM-TRITON-AOT-PER-ARCH`, 2026-07-28), each a real per-target cubin (`cuobjdump sm=80/86/89/90/100`); the CUDA fast-path arch fan-out (non-GDN) remains as evidenced in `backend-matrix.md` | [inventory](specs/kernel-family-inventory.md); [per-arch Triton AOT](specs/triton-aot-per-arch.md) | `ANCHOR-BACKFILL` | - | diff --git a/.agents/specs/cpu-a76-bf16-gemm.md b/.agents/specs/cpu-a76-bf16-gemm.md index 3aec5ba15..484d40257 100644 --- a/.agents/specs/cpu-a76-bf16-gemm.md +++ b/.agents/specs/cpu-a76-bf16-gemm.md @@ -195,11 +195,11 @@ same binary. A microbenchmark-only win never justifies selecting assembly. | W | Deliverable | State | |---|---|---| -| W0 | Refresh both-engine binding baseline/profile; validate artifact hashes and idle-state evidence | pending | -| W1 | Add elementwise BF16 microbench/PMU fixtures and selection mutation proof | pending | -| W2 | Audit current compiler output and evaluate C++/ACLE schedule/tile/prefetch candidates | pending | -| W3 | Recursively gate retained C++ default through full model vs llama.cpp | pending | -| W4 | Assembly eligibility decision from the explicit gate; implement only if eligible | blocked by W3 | +| W0 | Refresh both-engine binding baseline/profile; validate artifact hashes and idle-state evidence | **complete; reranked the work** | +| W1 | Add elementwise BF16 microbench/PMU fixtures and selection mutation proof | **blocked by W0 reranking** | +| W2 | Audit current compiler output and evaluate C++/ACLE schedule/tile/prefetch candidates | **blocked by W0 reranking** | +| W3 | Recursively gate retained C++ default through full model vs llama.cpp | **blocked by W0 reranking** | +| W4 | Assembly eligibility decision from the explicit gate; implement only if eligible | **blocked by W0 and W3** | ## Risks and stop conditions @@ -223,3 +223,36 @@ Spec accepted after the inventory checker correctly rejected the new 52nd kernel row while pinned to 51. The count ratchet moved only after the issue, row and structured spec existed. Implementation and binding measurement remain pending. + +### W0 fresh-profile result, 2026-08-10 + +W0 triggered this spec's stop condition before any benchmark-fixture or kernel +edit. The exact current `0e4a1c13d` AArch64 artifact passes its QEMU build gate +and emits the established 64-token SHA, but the production `AsyncLLM` benchmark +frontend now competes with a four-thread CPU pool for the Pi's four cores. The +matched profile ranks summed user cycles as `Threadpool::Barrier` 42.21%, +`Bt16Neon` 33.09% and Q8 A76 assembly 12.05%. Thus both binding +premises changed: synchronization/resource ownership ranks before the kernel, +and the reached 16-bit specialization is F16 rather than BF16. + +The old synchronous `9044c2a7d` artifact still runs in 26,028.76 ms, but it is +historical context rather than a causal frontend control because 141 relevant +files differ. Current AsyncLLM with four compute threads takes a 48,698.67 ms +median; the same current binary with three compute threads recovers to +26,957.58 ms and 2.53 decode tok/s. It does not restore prefill, because that +arm has only three compute workers. This proves a runnable-thread/core-budget +interaction, not which frontend or engine thread causes it, and is not evidence +for shipping a three-thread default. + +The fresh current-main TTFT spread is 15.3%, so its 0.209x prefill / 0.350x +decode diagnostic is not promoted over the accepted 2026-08-06 floor. The +official b9892 denominator reproduced at 27.96 prefill and 3.907 decode tok/s. +Full commands, samples, artifact hashes and raw evidence hashes are in the +[W0 reranking evidence](../../docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md). + +Disposition: the row remains `ACTIVE`, but W1-W4 are blocked until the +[`SERVE-CLI-BENCH` control in issue #293](https://github.com/mudler/vllm.cpp/issues/293) +and its [committed spec](cpu-bench-thread-budget.md) resolve or reject the +frontend-polling hypothesis, followed by a new four-core profile that ranks the +reached elementwise kernel. The BF16 fixture, C++ schedule candidates and +assembly gate were not attempted. Assembly remains forbidden. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 0d2bb4d40..d6b4aabce 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -248,7 +248,7 @@ host mirror is freed once the device Marlin resident is built. | 64-token Qwen model gate | Byte-identical across x86, portable, SDOT and assembly arms; asm vs SDOT median TTFT -1.55%, TPOT neutral, E2E -0.13%; vs portable TTFT -33.40%, E2E -2.67%. Cortex-A76+DotProd selects assembly by default | | Same-file llama.cpp floor (pp17/tg64) | **NOT MET on speed**: prefill 12.81 vs 27.77 tok/s (0.461x), decode 2.55 vs 3.91 (0.653x), E2E 26,018.39 vs 16,998.49 ms ([competitor evidence](bench-evidence/rpi5-a76-llamacpp-20260806.md)) | | Peak RSS | **2.841 vs 3.747 GiB, 24.2% less**; 3 clean unthrottled reps; same-text 64-token greedy output byte-identical after trailing-newline normalization | -| `ACTIVE` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Fresh both-engine profile, then C++/NEON BF16 GEMM speed closure; assembly remains blocked until C++ exceeds llama.cpp on every speed axis | +| #284 W0 | Barrier first; F16 next; #293 tests polling. | Same GGUF file both arms, `dgx.casa` GB10 aarch64 (20 cores), idle, 3 reps, llama.cpp `237ad9b96` built fresh on the same host. diff --git a/docs/STATUS.md b/docs/STATUS.md index 8410065f3..38bf71a72 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -110,7 +110,7 @@ token-for-token correctness against the pinned oracle. | GGUF loading (F32/F16/BF16/Q4_0/Q8_0/Q2_K/Q3_K/Q4_K/Q5_K/Q6_K/IQ2_XXS/IQ3_XXS/IQ2_S/MXFP4/NVFP4) | Supported; compute-in-quant (keep-quant) on CPU AND now CUDA for the six K-block encodings PLUS Q2_K/IQ2_XXS/IQ3_XXS (DeepSeek-V4 W8, 2026-07-29 - the FIRST CUDA keep-quant GGUF k-quant GEMM `KERNEL-QUANT-CIQ-GEMM-CUDA`, MMVQ-style dequant-in-kernel, GB10-gated 92401/92401 vs the CPU oracle, so a CUDA runner keeps blocks compressed and dots them on the GPU instead of the ARM cores); **NVFP4 now COMPUTES IN FP4 on CUDA for the dense-MLP and full-attention projections (2026-07-29, `CLAIM-GGUF-NVFP4-COMPUTE`), no longer materialize-only** | Weights in six block encodings stay compressed from file to matmul on CPU (no BF16 expansion). NVFP4 (ggml type 40) DEQUANTIZES, including the per-tensor (per-expert) `.scale` sidecar the container keeps outside the blocks; gated BIT-EXACT against the compressed-tensors NVFP4 path on real Qwen3.6-27B bytes from both containers. **It no longer expands to bf16 on CUDA:** an NVFP4 matmul/expert weight is REPACKED at load into the same (`weight_packed [N,K/2]`, `weight_scale [N,K/16]`) operand pair the compressed-tensors path produces - a pure byte permutation, gated BYTE-IDENTICAL against that container - and the existing `vt::MatmulNvfp4*` kernels run on it, so no new kernel exists and no numerics are re-derived. Covers the dense MLP + full-attention q/k/v/o and the MoE shared/routed experts; the GDN `in_proj_*` family and `ssm_out` still expand (the V-head reorder rewrites their layout) and a CPU build still expands everything - the documented `part` subset. **MEASURED GB10 (2026-07-29), same-binary A/B, 2 reps/arm:** peak RSS **50.8 -> 25.7 GiB**, load-and-generate **1:58 -> 0:41**; the 256 moved projections cost 35 840 MiB expanded vs 10 080 fp4-resident (3.56x). **The safetensors-sibling divergence CLOSES:** the fp4 arm is token-IDENTICAL over 24 greedy tokens where the same binary's bf16 arm diverges at index 4. REPORTED, not gated: the containers are not the same model - the GGUF NVFP4-quantizes 192 GDN `in_proj` tensors the safetensors keeps BF16 (mean rel. weight error ~0.18) and their activation global scales differ, so identity is not guaranteed and a cross-container throughput arm is invalid. SACRED gates unmoved: `test_qwen27_paged_engine` 235/235, `test_qwen36_paged_engine` 315/315. **The MoE (35B) stacked-expert arm is HARDWARE-GATED too (2026-07-29)**: the real 35B A3B NVFP4 GGUF loads and generates through the fp4 path, its 120 routed-expert stacks x 256 experts repack to the modelopt safetensors' operands with ZERO differing bytes over 840 sampled (tensor, expert) slabs, and all 840 per-expert `.scale[e]` are bit-identical to that expert's `weight_scale_2` - the scale INDEXING, mutation-proved against a `scales[0]`-for-all and an expert-0-slab-for-all mutant. Same-binary A/B: peak RSS 68.5 -> 22.7 GiB (3.01x), load-and-generate 1:51.9 -> 0:28.8, tokens IDENTICAL (the 35B routed experts run the W4A16 grouped GEMM in both arms). Recorded OPEN: this case's 24-token greedy stream is NOT run-to-run stable (1 of 3 `use_a16` and 1 of 4 reference runs differed), so the binding results are the weight-level byte identity and the residency audit, not token-exactness; `test_qwen36_paged_engine` is token-exact at ITS engine params, so the instability belongs to this case's configuration and attributing it is owed work. It also FIXED a latent defect the MoE arm made reachable: the two fp4 fused MoE blocks issued the router GEMM assuming the safetensors `[K,N]` gate layout and threw `matmul: inner dims mismatch` on the GGUF's `[N,K]`; `MoeRouterLogits` now branches on `nk` (inert for safetensors, SACRED unmoved). **Q2_K (id 10) + IQ2_XXS (id 16) DEQUANTIZE (2026-07-29, `CLAIM-DSV4-GGUF-LOADER`):** the ~2-bit types the single-Spark `DeepSeek-V4-Flash-GGUF UD-IQ2_XXS`/`UD-Q2_K_XL` vehicles use, ported 1:1 from llama.cpp `ggml-quants.c` (`iq2xxs_grid` codebook + signs; Q2_K nibble sub-scale/min), unit-gated on hand-derived bytes (`test_gguf_dequant` 15/15). Dequant-only (no vec_dot -> expand-bf16). A V4-GGUF model cannot RUN yet: the name map (tensor-manifest-blocked) + the V4 forward (W3-W8) remain. **Multi-shard split GGUF READING (2026-08-03, `CLAIM-GGUF-SPLIT-SHARDS`):** `GgufFile::Open` now transparently stitches llama.cpp `gguf-split` shards (`...-00001-of-00003.gguf`) — every shard mmap'd, tensor tables merged, KV metadata from shard `00001`, sibling mappings kept alive by the primary so keep-quant mmap-borrows stay valid across shards (`OwnsSpan` is shard-aware); `VT_GGUF_NO_SPLIT=1` opts out; unit-gated (`test_gguf` split-merge / no-split / count-mismatch cases, 33/33 local). This unblocks the real 3-shard `unsloth/DeepSeek-V4-Flash-0731 UD-IQ2_M` (~91 GiB), whose layout is the NATIVE `deepseek4` arch — per-block `ffn_gate_tid2eid` hash tables (hash layers 0/1/2) + `hc_*` MHC + DSA compressor/indexer are all PRESENT (name-map 1328/1328), `vocab_size` derives from `token_embd` — NOT a standard llama.cpp conversion, so no loader-layout change is owed. It now loads THROUGH 1324/1328 tensors; the sole remaining gap is 4 routed-expert slabs quantized with IQ2_S (id 22, ×2) + MXFP4 (id 39, ×2) — encodings we have GGUF block traits for but no keep-quant vec_dot, so they hit the expand→dequant path which lacks them. Expanding those 4 expert tensors to bf16 would add ~17 GiB (~106 GiB total → OOM-reboot risk), so the memory-safe fix is an IQ2_S+MXFP4 keep-quant kernel (CPU dequant dispatch + `iq2s_grid` + CUDA `DotSuperblock`), spec'd as the next brick **IQ2_S (id 22) + MXFP4 (id 39) DEQUANTIZE + KEEP-QUANT on CPU (2026-08-03, `CLAIM-DSV4-UDIQ2M-QUANT`, off-GPU):** the extra per-tensor "dynamic" encodings the `unsloth/DeepSeek-V4-Flash-GGUF UD-IQ2_M` checkpoint mixes into its last routed-expert slabs (IQ2_S `ffn_gate/up` dotting Q8_K, MXFP4 `ffn_down` dotting Q8_0) — ported 1:1 from llama.cpp `ggml-quants.c` @ 237ad9b96 (`iq2s_grid` 1024-entry codebook + DIRECT sign bytes; MXFP4 `kvalues_mxfp4` + `e8m0_to_fp32_half` micro-scaling, distinct from the compressed-tensors `E8M0ToF32` NVFP4 path). CPU dequant + keep-quant `vec_dot`, unit-gated on hand-derived golden bytes (`test_gguf_dequant` 17/17), an INDEPENDENT f64 dequant-then-dot + GEMM NMSE (`test_ops_quant_dot` 19/19), and keep-quant routing (`test_gguf_keep_quant` 37/37) — all CPU-green, so UD-IQ2_M's four previously-`unsupported ggml type 22/39` slabs now load COMPRESSED (no ~17 GiB bf16 expansion that OOM-reboots the box). CUDA: the IQ2_S device `DotSuperblock` is wired into the Q8_K grouped-MoE GEMM and now **CUDA-BUILT + LINKED on GB10 (sm_121a, CUDA 13.0, `-Werror`, 2026-08-03 integration)** — it compiles clean and the merged binary links; MXFP4's device dot (`DotMXFP4`) is written but NOT wired (Q8_0-activation needs a separate 32-block GEMM) so it is marked `[[maybe_unused]]` to keep the ready math without tripping nvcc #177-D, and on GPU MXFP4 CPU-fallbacks like Q4_0/Q8_0. The V4-GGUF forward + a real UD-IQ2_M GPU load/coherence run are owed | | AWQ / GPTQ quantization | W0 spike + W1 CPU INT4 dequant primitive; not yet loadable end to end | INT4 unpack+dequant-to-bf16 for BOTH community formats, mirroring vLLM 1:1 (AWQ reverse-order `awq_triton.py`; GPTQ `qdq_4.cuh` with zero_offset v1/v2 + act-order g_idx). Unit-gated RED-first (hand-computed known bytes + double-precision roundtrip). NOT wired to a loader, no GPU Marlin compute, no model run yet: config recognizer (W2), Marlin GPU GEMM riding the vendored NVFP4 Marlin (W4), CPU e2e (W3), GPTQ 8/2/3-bit (W5) and MoE (W6) are named next bricks. See [.agents/specs/awq-gptq-quant.md](../.agents/specs/awq-gptq-quant.md) | | MXFP4 (compressed-tensors `mxfp4-pack-quantized`) | Compute PROVEN (#38); GQA-swap ON (#49); decode-graph+gate_up FUSION default-ON. `VT_MARLIN_DENSE` DEFAULT-ON (`KERNEL-MARLIN-DENSE-EXEC`): dense marlin 48-CTA byte-faithful (32B 0.000, 263/263), binding beats #51 every axis (c1 1.020, c8 0.969, mem 2.63x). **`QUANT-CT-MXFP4-FINAL-STACK` TERMINAL — both last levers exhausted: num_splits cap `VT_FA2_NSPLITS_CAP` gated-OFF (c1-only, self-corrects@c8; 32B strict char-identical); glue folds via `vt::FusedChain`; `FLASH-AUDIT` #68: c8 flash gap +12.5us/call is occupancy/L2-bound; `-use_fast_math` TRIED, REGRESSES flash (168.8→189.8), rejected. c1 1.020x PASS, c2-c8 0.962-0.969.** state.md | Shared with DeepSeek-V4-Flash + Kimi-K3 MXFP4 paths. CPU E8M0 dequant 5/5·1142. GPU W4A4 + MoE-expert e2e later | -| CPU backend vs llama.cpp | 20-core at floor; Pi/A76 `GATING`; BF16 `ACTIVE` ([#284](https://github.com/mudler/vllm.cpp/issues/284)) | Pi: 0.461x prefill / 0.653x decode; RSS 24.2% lower. C++/NEON before assembly. | +| CPU backend vs llama.cpp | Pi/A76 `GATING`; #284 W0 | Floor unchanged: 0.461x/0.653x. T4/T3 proves only thread-budget sensitivity; #293 tests polling. | | Paged KV cache + prefix caching | Supported | Block-paged full attention, hybrid full-attention + GDN state groups, automatic prefix caching (APC) on by default for dense models (cache-ON gated end to end: token-identical output, cache hits, faster TTFT) | | fp8 KV cache (`cache_dtype=fp8`) | In progress (W1 CPU brick), not yet usable end-to-end | HIGH-priority memory/throughput lever (halves the KV footprint). W0 spike + W1 CPU brick landed (`KV-FP8` ACTIVE): fp8-e4m3 K/V STORE (`Quantize(hp/scale)`) + the paged-attention READ dequant (`Dequant(fp8)*scale`) + the `cache_dtype` config parse, all CPU-gated RED-first (`test_ops_fp8_kv_cache` 8/8·511; a wrong store direction fails 3/480). Storage is 1-byte fp8 (`DType::kI8`) + a `Fp8KVCacheDataType` interpretation enum, per-tensor k/v scales (mirroring vLLM `BaseKVCacheMethod`). The CUDA store + fp8 paged-attention read (the GPU memory-halving path, DGX-blocked), the runner/spec integration (half-sized KV blocks + checkpoint-scale threading + `--kv-cache-dtype`/`--calculate-kv-scales`), fp8_e5m2 and per-head scales are named W2-W5 in [.agents/specs/fp8-kv-cache.md](../.agents/specs/fp8-kv-cache.md). No model can run with an fp8 KV cache yet | | Prefix-cache matching unit (`--prefix-match-unit`) | Partial (resolver landed, config/scheduler wiring pending) | 0.26-new knob setting the finest token boundary a prefix-cache hit can land on (the `hash_block_size`). W1: `resolve_kv_cache_block_sizes` ported 1:1 (hybrid `hash_block_size = prefix_match_unit if set else gcd(group block sizes)`; single-group inert; back-off on no-consumer / mamba-non-align; throws on non-divisible), CPU unit-gated RED-first (default gcd vs `=16`). Pending: the config/CLI/ABI field (W2), scheduler threading of a finer-than-block matching unit (W3, needs the KV-block-pool align path), and the benchmark (W4). Default path byte-identical (dense single-group models ignore it). | diff --git a/docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md b/docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md new file mode 100644 index 000000000..492fb84df --- /dev/null +++ b/docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md @@ -0,0 +1,204 @@ +# Raspberry Pi 5 A76 elementwise GEMM W0 reranking, 2026-08-10 + +This is the required fresh-profile stop gate for +`KERNEL-GEMM-CPU-ELEM-A76` ([issue #284](https://github.com/mudler/vllm.cpp/issues/284)). +It does **not** establish a replacement binding floor. Current main changed the +benchmark frontend after the 2026-08-06 comparison, and the current T4/T3 +measurements expose a runnable-thread/core-budget interaction on this four-core +target. They do not attribute that interaction to polling. The profile therefore +reranks the work before the planned BF16 fixture or kernel experiment. + +## Result + +The planned BF16 optimization is blocked before W1-W4: + +1. current main's production `AsyncLLM` benchmark frontend busy-polls + `get_output_nowait()` and yields when no output is ready + (`examples/bench/bench_core.h:535-580`); +2. `VLLM_CPP_CPU_THREADS=4` simultaneously uses four compute workers on the + four-core Pi while the frontend/output threads are also runnable; +3. a matched `perf record -e cycles:u` profile attributes 42.21% of summed + user-space cycles to `Threadpool::Barrier`, ahead of the elementwise kernel; +4. the hot elementwise specialization is + `Bt16Neon` at 33.09%, not the BF16 specialization assumed by + the row; Q8 A76 assembly is third at 12.05%; and +5. a same-current-binary T4-to-T3 diagnostic recovers decode from 1.37 to 2.53 + tok/s per stream and E2E from 48.70 to 26.96 seconds. This proves sensitivity + to runnable-thread/core budget, not that benchmark polling is the cause. + +`perf` sums cycles across threads, so the 42.21% barrier figure is not by itself +a wall-time fraction. The T4/T3 current-binary pair establishes the thread-budget +interaction while leaving its responsible thread unresolved. The old binary is +historical context only: 141 relevant files differ, so it cannot be treated as a +one-variable frontend control. The polling hypothesis is tracked by +[`SERVE-CLI-BENCH` issue #293](https://github.com/mudler/vllm.cpp/issues/293) +and its [committed control spec](../../.agents/specs/cpu-bench-thread-budget.md). +A fresh profile after that discriminator must decide which elementwise dtype or +kernel ranks next. No kernel, benchmark-fixture or assembly change was made in +this checkpoint. + +## Fresh measurements + +All runs used the same GGUF bytes, 17 actual input tokens, 64 output tokens, +greedy sampling, cores 0-3 and an idle, unthrottled Pi. Prompt content is not +matched: `llama-bench` b9892 constructs random vocabulary token IDs, whereas +`vllm-bench` constructs repeated-word text. This makes the cross-engine values +diagnostic shape-matched measurements, not a same-prompt binding comparison. +The current vllm.cpp values are medians of three fresh processes. llama.cpp +reports three in-process samples after its warmups. + +| Axis | current vllm.cpp, AsyncLLM/T4 | llama.cpp b9892 | current / llama | +|---|---:|---:|---:| +| Prefill | 5.84 tok/s | 27.96 tok/s | 0.209x | +| Per-stream decode | 1.37 tok/s | 3.907 tok/s | 0.350x | +| Output-equivalent E2E | 1.314 tok/s | 3.767 tok/s | 0.349x | +| E2E latency | 48,698.67 ms | 16,986.93 ms | 2.867x latency | + +These are diagnostic, not a new accepted floor. Current-main TTFT ranged from +2,630.58 to 3,074.71 ms (15.3% about the median), failing the sub-3% binding +spread. TPOT was 724.19-737.27 ms and E2E 48,678.93-49,357.11 ms. Every current +run and profile emitted the established token file SHA-256 +`0ec98eabb23e4148d540fcf79a2fe61678fb90fe462cdf28134af7a42fe6a826`. + +The official llama.cpp b9892 denominator reproduced the previous evidence: +pp17 `27.9565 +/- 0.1826` tok/s, tg64 `3.906916 +/- 0.003335` tok/s and combined +E2E `16,986.93 +/- 43.76` ms. Its profile ranks `ggml_vec_dot_f16` 51.58%, +`ggml_gemv_q8_0_4x4_q8_0` 31.55% and `ggml_gemm_q8_0_4x4_q8_0` 3.05%. + +## Diagnostic controls + +The previous `9044c2a7d` artifact was rerun immediately on the same host. It +still uses the pre-`3f35356e0` synchronous `LLMEngine` benchmark driver and +reproduced 1,328.30 ms TTFT, 392.07 ms TPOT and 26,028.76 ms E2E. This is a +useful historical check against gross host drift, but 141 relevant files differ +between it and current source; it cannot isolate the frontend as the source of +the current binary's approximately twofold loss. + +| Arm | Compute threads | TTFT | TPOT | Per-stream decode | E2E | +|---|---:|---:|---:|---:|---:| +| old `9044c2a7d`, synchronous driver | 4 | 1,328.30 ms | 392.07 ms | 2.55 tok/s | 26,028.76 ms | +| current `0e4a1c13d`, AsyncLLM | 4 | 2,908.80 ms median | 730.93 ms median | 1.37 tok/s | 48,698.67 ms median | +| current `0e4a1c13d`, AsyncLLM | **3** | 2,089.78 ms | 394.73 ms | **2.53 tok/s** | **26,957.58 ms** | + +The T3 arm is diagnostic rather than a proposed default: reducing the compute +pool restores decode but only reaches 8.13 prefill tok/s. This is consistent +with runnable-thread pressure, but it does not identify polling, the output +handler or another engine thread as the cause. Issue #293 therefore starts with +a same-current-source blocking-wait discriminator before considering a general +event seam or a thread-budget default. Merely selecting three compute threads +would trade prefill away. + +## Provenance and build gate + +- Pi: Raspberry Pi 5, four Cortex-A76 r4p1 cores, 2.4 GHz under load, DotProd, + no i8mm, 8 GiB RAM, kernel `6.18.34+rpt-rpi-2712`. +- Current source: `0e4a1c13d7674705b8c72fd0d623b8a11f1fb991`. +- Current `vllm-bench`: SHA-256 + `a94af4e85b0b7ec8064c81539e363933eeb8859eba9562049755bfda04d6bdfc`. +- Old control `vllm-bench`: SHA-256 + `9eb57cf3760eaade9dcef03dda1648556577c44199369ad38bf42083efbc70a9`. +- llama.cpp: official b9892 `ee445f93d8a0a5033a46d1960e901ef5caec9a41`, + binary SHA-256 + `d9d93d8b38d0d8faa676f7d48f1a8fcbbc235f3a5697fc2ac787422d61783d52`. +- GGUF file: 2,834,940,160 filesystem bytes, SHA-256 + `a53988df91157d78acaf3c95e22db179d13f6236061bdb86576494dc99b1bc3b`. + llama.cpp reports 2,823,978,240 logical tensor bytes; the older evidence + accidentally labelled that logical value as the file size. +- Local AArch64 build: Ubuntu 24.04/GCC 13.3, `docker buildx` builder `pf-arm`, + `linux/arm64`, Release, CPU-only. QEMU timing is void. The build ran + `test_ops_quant_dot`: 23/23 cases and 150,350/150,350 assertions, plus the + A76 assembly benchmark smoke. +- Every accepted diagnostic leg ended at 2.4 GHz with + `vcgencmd get_throttled=0x0`; temperatures stayed below 67 C. + +Build and deployment: + +```sh +docker buildx build --builder pf-arm --platform linux/arm64 \ + -f docker/Dockerfile.arm64 --target export \ + --output type=local,dest=/tmp/vllm-arm64-0e4a1c13d . + +scp /tmp/vllm-arm64-0e4a1c13d/vllm-{bench,cpu-kernel-bench} \ + rich@rpi5fan.lan:vllm-cpp-assembly/bin/a76-bf16-0e4a1c13d/ +``` + +Measurement and diagnostic-control commands, run from the evidence directory: + +```sh +A76_MODEL=/home/rich/vllm-cpp-assembly/models/Qwen3.5-2B-UD-Q8_K_XL.gguf +A76_CURRENT=/home/rich/vllm-cpp-assembly/bin/a76-bf16-0e4a1c13d/vllm-bench +A76_OLD=/home/rich/vllm-cpp-assembly/bin/final/vllm-bench +A76_LLAMA=/home/rich/vllm-cpp-assembly/bin/llama-b9892/llama-bench + +for A76_REP in 1 2 3; do + taskset -c 0-3 env VLLM_CPP_CPU_THREADS=4 VT_CPU_Q8_DOT=auto \ + "$A76_CURRENT" --model "$A76_MODEL" --num-prompts 1 \ + --input-len 16 --output-len 64 --concurrency 1 --seed 0 \ + --temperature 0 \ + --output-token-ids "vllm-default-${A76_REP}-tokens.json" \ + >"vllm-default-${A76_REP}.out" 2>"vllm-default-${A76_REP}.err" +done + +taskset -c 0-3 "$A76_LLAMA" -m "$A76_MODEL" -p 17 -n 64 \ + -pg 17,64 -t 4 -r 3 -o jsonl --progress \ + >llama-baseline.jsonl 2>llama-baseline.err + +taskset -c 0-3 env VLLM_CPP_CPU_THREADS=4 VT_CPU_Q8_DOT=auto \ + "$A76_OLD" --model "$A76_MODEL" --num-prompts 1 \ + --input-len 16 --output-len 64 --concurrency 1 --seed 0 \ + --temperature 0 --output-token-ids vllm-old-control-tokens.json \ + >vllm-old-control.out 2>vllm-old-control.err + +taskset -c 0-3 env VLLM_CPP_CPU_THREADS=3 VT_CPU_Q8_DOT=auto \ + "$A76_CURRENT" --model "$A76_MODEL" --num-prompts 1 \ + --input-len 16 --output-len 64 --concurrency 1 --seed 0 \ + --temperature 0 --output-token-ids vllm-new-t3-tokens.json \ + >vllm-new-t3.out 2>vllm-new-t3.err +``` + +Matched current profile: + +```sh +taskset -c 0-3 perf record -q -e cycles:u -F 199 \ + -o vllm-default-cycles.data -- \ + env VLLM_CPP_CPU_THREADS=4 VT_CPU_Q8_DOT=auto "$A76_CURRENT" \ + --model "$A76_MODEL" --num-prompts 1 \ + --input-len 16 --output-len 64 --concurrency 1 --seed 0 \ + --temperature 0 --output-token-ids vllm-default-profile-tokens.json \ + >vllm-default-profile.out 2>vllm-default-profile.err +perf report -i vllm-default-cycles.data --stdio --no-children \ + --percent-limit 0.1 >vllm-default-cycles-report.txt + +taskset -c 0-3 perf record -q -e cycles:u -F 199 \ + -o llama-cycles.data -- "$A76_LLAMA" -m "$A76_MODEL" \ + -p 17 -n 64 -pg 17,64 -t 4 -r 1 -o jsonl --progress \ + >llama-profile.jsonl 2>llama-profile.err +perf report -i llama-cycles.data --stdio --no-children --percent-limit 0.1 \ + >llama-cycles-report.txt +``` + +The before/after snapshots record wall time, load, `lscpu`, hashes, all four +governors and frequencies, temperature, throttle state and the highest-CPU +processes. Before the two binding arms, load was 0.09/0.03/0.01 and +0.00/0.03/0.16, governors were `ondemand`, every core reported 2.4 GHz, +temperatures were 50.5/51.0 C and throttle state was `0x0`. The only non-kernel +process above 0.1% CPU was `tailscaled` at 0.1%; the SSH session was 0.3%. + +## Raw evidence + +Files remain on the authorized target under +`rich@rpi5fan.lan:~/vllm-cpp-assembly/evidence/a76-bf16-20260810/`. + +| Raw file(s) | SHA-256 | +|---|---| +| `llama-before.txt` / `llama-after.txt` | `059f8ed212b8b420d3828c067fd99f0ee8b6d19008e8947c751ff18a942c66f5` / `be1230ef78366de856e2cac70210444cb5a10554c2fe4419c8fc454dc03095b8` | +| `llama-baseline.jsonl` / `.err` | `cf4638ea1857e4058b92a064b29e3ef3834c1d532b6c5db30cf3ffa92708932e` / `c8cd80de052c6bdf422751b2ad40ccce293c310c3b47824ef8763fabdc391c25` | +| `llama-profile.jsonl` / `.err` | `f2622e6f3cf722e8ccd4bbebc45786b87539193f979e17554fa732f1e90b16ba` / `d48e1c06fc3dbc5b7cad5b9e4ceee484acf24c85215cd020cecf3d6af24206c9` | +| `llama-cycles.data` / `llama-cycles-report.txt` | `3ad335eb930c87a8278b65604a028111683566fd1cdd810f727f48a99eef9a93` / `1403dd2e860ad6c634e3158caf09b387fa3f07ff0d35dc807e427cb1c3124950` | +| `vllm-default-before.txt` / `vllm-default-after.txt` | `cbc52c736b67896cd5f1b037404a5b6613c29a3e3e4020d9adca22e901e140bf` / `afefa40484b0a554a753a0086b6e288d99fbf9137a7ba3dcebfbbb983974e0ac` | +| `vllm-default-{1,2,3}.out` | `9b8d4fc6a6c4a82d77ec4dc152b6197e0a7139f72925d86331908e84453a7c17` / `b18fecbf4b9fa389223653e751cac370cfb9f203d5be764dfe53c61cdcd380dd` / `1b97d5bbf8f81cfc1efe7047dfce2563fbf595bc7e201f8d5d978fed792730c6` | +| `vllm-default-profile.out` / every vllm `.err` | `a3072d35030eff0259addb50d8c633c0233671ea171fbf781f7cbbe477af97c9` / `a63429dcbd680bc105f7fdcea35beeb01bb267c53be6814705b07e7ab5f302cf` | +| every vllm token JSON file | `0ec98eabb23e4148d540fcf79a2fe61678fb90fe462cdf28134af7a42fe6a826` | +| `vllm-default-cycles.data` / `vllm-default-cycles-report.txt` | `c5aa796bb6ac4418e53b55389bbddc6a85704ac487506eba721080c0941f6926` / `af21534402cc497bb3d676bfd767d4d63aea21a0b35f35ec195ae76e3340abfb` | +| `vllm-old-control.out` / `vllm-new-t3.out` | `338a4eccc61456028e255e2975a99bd774d306bd7429fefd42dc58b23812f77c` / `daf0cbdb5e5d3a860af4f4de5cc12e1799c6d5be7ff63f5534931c0481b4e7e8` | +| `frontend-control-summary.txt` | `cbc79b75e5987a7cd0ae7c6619adcbbde145b2b52e994ea3bc985f41ee4f3ed3` | diff --git a/docs/bench-evidence/rpi5-a76-llamacpp-20260806.md b/docs/bench-evidence/rpi5-a76-llamacpp-20260806.md index 5294c3782..5295c3d44 100644 --- a/docs/bench-evidence/rpi5-a76-llamacpp-20260806.md +++ b/docs/bench-evidence/rpi5-a76-llamacpp-20260806.md @@ -48,8 +48,11 @@ Thus the competitor gap is performance-only for the checked stream. - Pi: Raspberry Pi 5, four Cortex-A76 r4p1 cores, DotProd present, i8mm absent, 8 GiB RAM, Debian kernel `6.18.34+rpt-rpi-2712`. -- Model: `Qwen3.5-2B-UD-Q8_K_XL.gguf`, 2,823,978,240 bytes, SHA-256 +- Model: `Qwen3.5-2B-UD-Q8_K_XL.gguf`, 2,834,940,160 filesystem bytes, + SHA-256 `a53988df91157d78acaf3c95e22db179d13f6236061bdb86576494dc99b1bc3b`. + llama.cpp reports 2,823,978,240 logical tensor bytes; the original record + accidentally labelled that logical value as the file size. - vllm.cpp: branch head `9044c2a7d`; assembly-default `vllm-bench` SHA-256 `9eb57cf3760eaade9dcef03dda1648556577c44199369ad38bf42083efbc70a9`. - llama.cpp: official tag `b9892`, commit diff --git a/scripts/check-public-doc-tables.py b/scripts/check-public-doc-tables.py index 7d06d9070..d6a3f4c4c 100755 --- a/scripts/check-public-doc-tables.py +++ b/scripts/check-public-doc-tables.py @@ -576,7 +576,6 @@ def features_errors(text: str) -> list[str]: # 243245 since 2026-08-11 (measured 243245): ENG-RELEASE-CONTAINERS # owes the page a lifecycle line, paid for inside the release # paragraph rather than out of the page. Strictly DOWN. - # # 243368 since 2026-08-10 (measured 243368): reconciled once more when the # external-PR landing (#155 + #266) merged current main, which had itself @@ -610,7 +609,10 @@ def features_errors(text: str) -> list[str]: # carries #347's shorter Muse Glimmer wording on top of main's. Re-measured # rather than carried from either PR, and rather than arithmetic on the two: # zero slack is the only honest re-pin. Strictly DOWN from 243188. - "chars": 243186, + # 243124 since 2026-08-11 (#293 C0, measured): the Pi/A76 status was + # collapsed while narrowing the W0 causal claim. Details stay in the + # indexed W0 evidence and committed control spec. Strictly DOWN. + "chars": 243124, "h2_sections": 11, "long_paragraphs": 82, "oversized_cells": 44, From 22bd4fd1540a0a2e1c010ba223f1aa8ec96a217f Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 20:57:13 +0100 Subject: [PATCH 05/11] chore(bench): activate upstream-based #293 C1 control Record the isolated current-upstream implementation worktree and move the scoped polling discriminator from READY to ACTIVE without moving the broad SERVE-CLI-BENCH row. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/NOW.md | 2 +- .agents/coordination.md | 16 ++++++++++++++++ .agents/specs/cpu-bench-thread-budget.md | 6 +++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.agents/NOW.md b/.agents/NOW.md index 68dba5245..d680b2c93 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -22,7 +22,7 @@ Work: 7 external PRs landed (6 merged + #227 repaired). | Kimi-Linear-48B | 122/128 held; e2e NOT ESTABLISHED | tiktoken-only ckpt: no warm server | | 35B mid-band | **canonical 0.918-0.972x** c1-c32 (@`348c265d`, first c16/c32) | Decode-only window, ONE tool | | Qwen3.5-4B sm_120 | tput **1.0283x** `PENDING`; TTFT/TPOT/E2E 1.085/1.017/1.029x | Profile wave | -| RPi5 A76 | **#293 READY**: T4 contention | Blocking control; then F16 | +| RPi5 A76 | **#293 C1 ACTIVE** | `poll` vs blocking; then F16 | | SERVE-ASYNC-MIRROR | **#323 FIXED** (mitigation): graph declines while the mirror is live; 7/7 async gates | Graph read ids at REPLAY | | CPU levers (`QUANT-GGUF-CIQ-GEMM`) | Profile DONE: decode **47% threadpool sync**, prefill **~39% paged attn** | Parakeet encoder; attn dtype hoist | | `SERVE-METRICS` async (#277) | **`/metrics` was DEAD on the shipped server**: AsyncLLM folded nothing. Now live, ctest 366/366 | Config-gated families | diff --git a/.agents/coordination.md b/.agents/coordination.md index 3bdaecc9a..10e6a78c1 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -1633,6 +1633,22 @@ and the config/CLI plumbing to select a mode from outside the library are both s Records a spec-after-code ordering deviation in the spec preamble rather than hiding it behind commit order. +**Benchmark thread-budget control (#293 C1, 2026-08-10).** Codex +(gpt-5.6-sol), fresh helper implementer, isolated worktree +`/tmp/vllm-a76-c1-upstream`, branch +`helper/SERVE-CLI-BENCH-293-C1-UPSTREAM`, base `9ec8cae23`; corrected W0/C0 +evidence `0a07c53db`. NOT in the claims table below: that table keys whole +`SPIKE`/`ACTIVE` rows, while the broad `SERVE-CLI-BENCH` row must stay +`PARTIAL`; the committed [#293 spec](specs/cpu-bench-thread-budget.md) tracks +this leaf as `ACTIVE`, and the helper role marker provides the worktree lock. +Scope: same-binary `poll` vs concurrency-1-only `blocking-c1`, RED mutation, +native/QEMU gates, and interleaved Pi T4/T3 correctness/profile measurement. +Owns `examples/bench/bench_core.h`, `examples/bench/main.cpp`, +`tests/examples/test_bench.cpp`, and scoped #293/spec/evidence projections. C2 +AsyncLLM/output-event files are excluded even if C1 is decisively positive. +Also excludes kernels, model/loader, CUDA/Vulkan, C ABI, assembly, Pi builds, +service changes and remote Git operations. + | Claim | Row IDs | Agent | Worktree / remote dir | Branch | Owned scope | State | Last update | |---|---|---|---|---|---|---|---| | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | `KERNEL-GEMM-CPU-ELEM-A76` (`ACTIVE`) | Codex (gpt-5.6-sol), fresh helper implementer | isolated local worktree `/tmp/vllm-a76-bf16-impl`; build/test under buildx/QEMU only; execute/PMU on user-authorized disposable `rich@rpi5fan.lan` | `helper/KERNEL-GEMM-CPU-ELEM-A76-W0-W2`, base spec commit `999e1221e` plus this claim commit | Issue #284 W0-W2 only: refresh both-engine baseline/profile; add the BF16 elementwise benchmark/selection mutation proof; audit disassembly and retain only byte-exact C++/NEON changes with recursive measurements. Owns the new spec/row and scoped edits to `examples/cpu_kernel_bench/main.cpp`, its CLI test/CMake only if needed, `src/vt/cpu/cpu_matmul_elem*`, `src/vt/cpu/cpu_ops.cpp` only if profiling proves partitioning, focused tests, and indexed Pi evidence/record projections. Excludes assembly, model/loader/CUDA/Vulkan/C ABI changes, building on the Pi, service changes and PR/push. | `DONE` | 2026-08-10 — W0 complete and reranked before code: current AsyncLLM T4/T3 proves a runnable-thread/core-budget interaction but does not isolate polling; the old binary is historical only. W1-W4 await the #293 discriminator and fresh profile; no kernel/fixture change retained; row stays `ACTIVE`. | diff --git a/.agents/specs/cpu-bench-thread-budget.md b/.agents/specs/cpu-bench-thread-budget.md index 2d6d5d891..dba015d53 100644 --- a/.agents/specs/cpu-bench-thread-budget.md +++ b/.agents/specs/cpu-bench-thread-budget.md @@ -1,7 +1,7 @@ # CPU benchmark frontend wait + thread budget **Row:** `SERVE-CLI-BENCH` · **issue:** -[#293](https://github.com/mudler/vllm.cpp/issues/293) · **leaf state:** `READY` · +[#293](https://github.com/mudler/vllm.cpp/issues/293) · **leaf state:** `ACTIVE` · **trigger:** Raspberry Pi 5 W0 refresh for `KERNEL-GEMM-CPU-ELEM-A76` at `3520b7f7f` · **target:** current-source causal control first, C++ only. @@ -145,8 +145,8 @@ upstream event semantics and extend the local original-harness tests: | W | Deliverable | State | |---|---|---| -| C0 | Correct #284 evidence wording and register #293 | ready | -| C1 | Same-current-source `poll` vs `blocking-c1` control + RED mutation | pending | +| C0 | Correct #284 evidence wording and register #293 | complete (`0a07c53db`) | +| C1 | Same-current-source `poll` vs `blocking-c1` control + RED mutation | active | | C2 | If C1 positive, general event/epoch wait + deterministic multi-request tests | blocked by C1 | | C3 | QEMU gate and interleaved Pi T4/T3 causal profile | blocked by C1 | | C4 | Retain event wait or reject it; test reserved-core budget only if C1 is negative/insufficient | blocked by C3 | From db2988621c343195a83f9f1988675370860fea9a Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 21:10:40 +0100 Subject: [PATCH 06/11] feat(bench): add blocking C1 output wait control (#293) Reapply the same-binary concurrency-one polling discriminator unchanged onto current upstream. Keep poll as the default, reject blocking-c1 above c1, and record both the selected mode and blocking-call count in text and JSON output. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- examples/bench/bench_core.h | 153 +++++++++++++++++++++++++++------- examples/bench/main.cpp | 33 +++++++- tests/examples/test_bench.cpp | 43 ++++++++++ 3 files changed, 194 insertions(+), 35 deletions(-) diff --git a/examples/bench/bench_core.h b/examples/bench/bench_core.h index 091cb41b0..ec52916c8 100644 --- a/examples/bench/bench_core.h +++ b/examples/bench/bench_core.h @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +68,29 @@ namespace vllm::bench { +enum class OutputWaitMode { + kPoll, + kBlockingC1, +}; + +inline const char* OutputWaitModeName(OutputWaitMode mode) { + switch (mode) { + case OutputWaitMode::kPoll: + return "poll"; + case OutputWaitMode::kBlockingC1: + return "blocking-c1"; + } + throw std::invalid_argument("unknown benchmark output wait mode"); +} + +inline OutputWaitMode ParseOutputWaitMode(std::string_view name) { + if (name == "poll") return OutputWaitMode::kPoll; + if (name == "blocking-c1") return OutputWaitMode::kBlockingC1; + throw std::invalid_argument("unknown benchmark output wait '" + + std::string(name) + + "' (expected poll or blocking-c1)"); +} + // ── Config (mirrors the `vllm bench throughput` / `serve` knobs we support) ──── struct BenchConfig { // Empty => SYNTHETIC tiny CPU engine (smoke). Otherwise a model dir / .gguf. @@ -76,6 +100,10 @@ struct BenchConfig { std::string dataset_path; // Optional JSON output path containing generated IDs in submission order. std::string output_token_ids_path; + // Optional structured result artifact. This is separate from + // output_token_ids_path so the established array-only token fixture stays + // byte-compatible. + std::string output_json_path; // Optional speculative-decoding config JSON, e.g. // '{"method":"mtp","num_speculative_tokens":1}'. Empty => spec decode OFF // (production non-spec path, byte-identical to pre-existing runs). Parsed via @@ -86,6 +114,10 @@ struct BenchConfig { int input_len = 16; // L: target prompt tokens per request. int output_len = 16; // O: max_tokens per request (greedy => exactly O). int concurrency = 4; // C: max in-flight requests admitted to the engine. + // `poll` preserves the production benchmark's scan+yield loop. The + // concurrency-1-only `blocking-c1` control performs the same nowait probe, + // then blocks on the sole collector so Pi measurements can isolate polling. + OutputWaitMode output_wait = OutputWaitMode::kPoll; uint64_t seed = 0; // prompt-generation RNG seed. double temperature = 0; // <= 0 => greedy (deterministic). bool quiet = false; // suppress per-progress logging to stderr. @@ -129,6 +161,10 @@ struct BenchResult { // result so a benchmark artifact cannot silently mix frontend modes. bool pretokenized_admission = false; int max_concurrent_batches = 1; + OutputWaitMode output_wait = OutputWaitMode::kPoll; + // Number of calls made to AsyncLLM::get_output after a failed nowait probe. + // This proves a blocking-c1 artifact actually exercised its control path. + int64_t blocking_wait_calls = 0; int completed = 0; double duration_s = 0.0; int64_t total_input = 0; @@ -532,6 +568,11 @@ inline SamplingParams MakeSampling(const BenchConfig& cfg, int req_index) { inline BenchResult RunBench(const BenchConfig& cfg) { using Clock = std::chrono::steady_clock; + if (cfg.output_wait == OutputWaitMode::kBlockingC1 && cfg.concurrency != 1) { + throw std::invalid_argument( + "benchmark output wait 'blocking-c1' requires --concurrency 1"); + } + std::unique_ptr loaded; std::vector prompts; prompts.reserve(static_cast(cfg.num_prompts)); @@ -616,6 +657,7 @@ inline BenchResult RunBench(const BenchConfig& cfg) { int next = 0; // next prompt index to submit. int in_flight = 0; // requests currently admitted + unfinished. int done = 0; + int64_t blocking_wait_calls = 0; auto admit = [&]() { const int available = cfg.concurrency - in_flight; @@ -670,6 +712,36 @@ inline BenchResult RunBench(const BenchConfig& cfg) { in_flight += wave_size; }; + auto consume_output = [&](auto it, RequestOutput out) { + RequestRecord& rec = records[out.request_id]; + if (rec.prompt_tokens == 0 && !out.prompt_token_ids.empty()) { + rec.prompt_tokens = static_cast(out.prompt_token_ids.size()); + rec.prompt_token_ids = out.prompt_token_ids; + } + if (!out.outputs.empty() && !out.outputs[0].token_ids.empty()) { + const double t = now_s(); + const int n_new = static_cast(out.outputs[0].token_ids.size()); + if (rec.first_token_s < 0.0) { + rec.first_token_s = t; // TTFT reference. + } else { + rec.itls.push_back(t - rec.last_token_s); // one ITL per chunk. + } + rec.last_token_s = t; + rec.output_tokens += n_new; + rec.output_token_ids.insert(rec.output_token_ids.end(), + out.outputs[0].token_ids.begin(), + out.outputs[0].token_ids.end()); + } + if (out.finished && !rec.finished) { + rec.finished = true; + rec.completion_s = now_s(); + --in_flight; + ++done; + return active.erase(it); + } + return std::next(it); + }; + admit(); while (done < cfg.num_prompts) { bool observed_output = false; @@ -680,43 +752,28 @@ inline BenchResult RunBench(const BenchConfig& cfg) { continue; } observed_output = true; - RequestOutput& out = *ready; - RequestRecord& rec = records[out.request_id]; - if (rec.prompt_tokens == 0 && !out.prompt_token_ids.empty()) { - rec.prompt_tokens = static_cast(out.prompt_token_ids.size()); - rec.prompt_token_ids = out.prompt_token_ids; - } - if (!out.outputs.empty() && !out.outputs[0].token_ids.empty()) { - const double t = now_s(); - const int n_new = static_cast(out.outputs[0].token_ids.size()); - if (rec.first_token_s < 0.0) { - rec.first_token_s = t; // TTFT reference. - } else { - rec.itls.push_back(t - rec.last_token_s); // one ITL per chunk. + it = consume_output(it, std::move(*ready)); + } + if (!observed_output) { + if (cfg.output_wait == OutputWaitMode::kBlockingC1) { + // Validation above guarantees the sole active collector is also the + // sole admitted request. The scan already performed the required + // nowait probe, matching AsyncLLM::generate before this blocking wait. + if (active.size() != 1) { + throw std::logic_error( + "blocking-c1 benchmark must have exactly one active request"); } - rec.last_token_s = t; - rec.output_tokens += n_new; - rec.output_token_ids.insert(rec.output_token_ids.end(), - out.outputs[0].token_ids.begin(), - out.outputs[0].token_ids.end()); - } - if (out.finished && !rec.finished) { - rec.finished = true; - rec.completion_s = now_s(); - --in_flight; - ++done; - it = active.erase(it); + auto it = active.begin(); + ++blocking_wait_calls; + it = consume_output(it, engine.get_output(it->second)); + (void)it; } else { - ++it; + // Exact historical policy: yield rather than block on an arbitrary + // request, which could delay a ready peer at concurrency greater than 1. + std::this_thread::yield(); } } admit(); // keep C in flight as requests finish. - if (!observed_output) { - // The output-handler thread will publish the next per-request DELTA. - // Yield rather than block on an arbitrary request: blocking on one - // collector can delay ready outputs for other requests and distort ITL. - std::this_thread::yield(); - } } const double dur_s = now_s(); @@ -748,6 +805,8 @@ inline BenchResult RunBench(const BenchConfig& cfg) { res.async_scheduling_enabled = loaded->async_scheduling_enabled(); res.pretokenized_admission = pretokenized_admission; res.max_concurrent_batches = loaded->max_concurrent_batches(); + res.output_wait = cfg.output_wait; + res.blocking_wait_calls = blocking_wait_calls; res.prompt_token_ids.resize(static_cast(cfg.num_prompts)); res.output_token_ids.resize(static_cast(cfg.num_prompts)); res.completed = done; @@ -807,6 +866,32 @@ inline void WriteOutputTokenIds(const std::string& path, out << nlohmann::json(result.output_token_ids).dump() << '\n'; } +inline nlohmann::json ResultJson(const BenchConfig& cfg, + const BenchResult& result) { + return {{"output_wait", OutputWaitModeName(result.output_wait)}, + {"blocking_wait_calls", result.blocking_wait_calls}, + {"concurrency", cfg.concurrency}, + {"completed", result.completed}, + {"duration_s", result.duration_s}, + {"total_input", result.total_input}, + {"total_output", result.total_output}, + {"request_throughput", result.request_throughput}, + {"output_throughput", result.output_throughput}, + {"prefill_throughput", result.prefill_throughput}, + {"mean_per_stream_decode", result.mean_per_stream_decode}, + {"mean_ttft_ms", result.mean_ttft_ms}, + {"mean_tpot_ms", result.mean_tpot_ms}, + {"mean_e2el_ms", result.mean_e2el_ms}, + {"output_token_ids", result.output_token_ids}}; +} + +inline void WriteResultJson(const std::string& path, const BenchConfig& cfg, + const BenchResult& result) { + std::ofstream out(path); + if (!out) throw std::runtime_error("cannot write benchmark JSON: " + path); + out << ResultJson(cfg, result).dump() << '\n'; +} + // Print the summary table, mirroring serve.py's "Serving Benchmark Result" // block + our prefill/decode split. Fixed-width columns like serve.py // ("{:<40} {:<10.2f}"). @@ -827,6 +912,10 @@ inline void PrintReport(const BenchConfig& cfg, const BenchResult& r, std::fprintf(out, "\n============= vllm.cpp Benchmark Result =============\n"); std::fprintf(out, "%-42s %-12s\n", "Engine frontend:", r.async_frontend ? "AsyncLLM" : "LLMEngine"); + std::fprintf(out, "%-42s %-12s\n", "Output wait:", + OutputWaitModeName(r.output_wait)); + line_i("Blocking wait calls:", + static_cast(r.blocking_wait_calls)); line_i("Async scheduling enabled:", r.async_scheduling_enabled ? 1 : 0); line_i("Pretokenized prompt admission:", r.pretokenized_admission ? 1 : 0); line_i("Maximum concurrent batches:", r.max_concurrent_batches); diff --git a/examples/bench/main.cpp b/examples/bench/main.cpp index 08fb0bb87..fdf06c6b5 100644 --- a/examples/bench/main.cpp +++ b/examples/bench/main.cpp @@ -12,7 +12,8 @@ // vllm-bench [--model ] [--dataset-path ] // [--num-prompts N] [--input-len L] // [--output-len O] [--concurrency C] [--seed S] -// [--temperature T] [--output-token-ids ] +// [--temperature T] [--output-wait poll|blocking-c1] +// [--output-token-ids ] [--output-json ] // // With NO --model it builds a SYNTHETIC tiny CPU engine so the harness runs on // the dev box (toy weights => meaningless numbers; the point is the harness). @@ -46,6 +47,8 @@ void Usage(const char* argv0, std::FILE* out) { " [--output-len O] [--concurrency C] [--seed S]\n" " [--temperature T] [--max-num-batched-tokens B]\n" " [--output-token-ids ]\n" + " [--output-json ]\n" + " [--output-wait poll|blocking-c1]\n" " [--speculative-config ]\n" "\n" "Throughput/latency benchmark over the vllm.cpp V1 AsyncLLM, mirroring\n" @@ -86,6 +89,17 @@ bool ParseArgs(int argc, char** argv, BenchConfig& cfg, int& exit_code) { cfg.temperature = std::atof(NextArg(argc, argv, i)); } else if (flag == "--output-token-ids") { cfg.output_token_ids_path = NextArg(argc, argv, i); + } else if (flag == "--output-json") { + cfg.output_json_path = NextArg(argc, argv, i); + } else if (flag == "--output-wait") { + try { + cfg.output_wait = + vllm::bench::ParseOutputWaitMode(NextArg(argc, argv, i)); + } catch (const std::invalid_argument& e) { + std::fprintf(stderr, "vllm-bench: %s\n", e.what()); + exit_code = 2; + return false; + } } else if (flag == "--max-num-batched-tokens") { cfg.max_num_batched_tokens = std::atoi(NextArg(argc, argv, i)); } else if (flag == "--num-blocks") { @@ -111,6 +125,14 @@ bool ParseArgs(int argc, char** argv, BenchConfig& cfg, int& exit_code) { exit_code = 2; return false; } + if (cfg.output_wait == vllm::bench::OutputWaitMode::kBlockingC1 && + cfg.concurrency != 1) { + std::fprintf(stderr, + "vllm-bench: benchmark output wait 'blocking-c1' requires " + "--concurrency 1\n"); + exit_code = 2; + return false; + } return true; } @@ -125,12 +147,14 @@ int main(int argc, char** argv) { std::fprintf(stderr, "vllm-bench: %s engine | num_prompts=%d input_len=%d " - "output_len=%d concurrency=%d seed=%llu temp=%.2f dataset=%s\n", + "output_len=%d concurrency=%d seed=%llu temp=%.2f dataset=%s " + "output_wait=%s\n", cfg.model_path.empty() ? "SYNTHETIC (no --model)" : cfg.model_path.c_str(), cfg.num_prompts, cfg.input_len, cfg.output_len, cfg.concurrency, static_cast(cfg.seed), cfg.temperature, - cfg.dataset_path.empty() ? "generated" : cfg.dataset_path.c_str()); + cfg.dataset_path.empty() ? "generated" : cfg.dataset_path.c_str(), + vllm::bench::OutputWaitModeName(cfg.output_wait)); try { const vllm::bench::BenchResult res = vllm::bench::RunBench(cfg); @@ -138,6 +162,9 @@ int main(int argc, char** argv) { if (!cfg.output_token_ids_path.empty()) { vllm::bench::WriteOutputTokenIds(cfg.output_token_ids_path, res); } + if (!cfg.output_json_path.empty()) { + vllm::bench::WriteResultJson(cfg.output_json_path, cfg, res); + } if (res.completed != cfg.num_prompts) { std::fprintf(stderr, "vllm-bench: only %d/%d requests completed\n", res.completed, diff --git a/tests/examples/test_bench.cpp b/tests/examples/test_bench.cpp index 518505bd3..e3618b881 100644 --- a/tests/examples/test_bench.cpp +++ b/tests/examples/test_bench.cpp @@ -22,6 +22,7 @@ using vllm::bench::BenchConfig; using vllm::bench::BenchResult; using vllm::bench::DispatchBenchPromptAdmission; using vllm::bench::DispatchBenchPromptWaveAdmission; +using vllm::bench::OutputWaitMode; using vllm::bench::PretokenizeBenchPromptsThenStartClock; using vllm::bench::RunBench; @@ -296,6 +297,48 @@ TEST_CASE("bench: pretokenized vectors match timed-string InputProcessor") { CHECK(clock_value == 17); } +TEST_CASE("bench: blocking-c1 selector rejects concurrent requests by name") { + BenchConfig cfg; + cfg.num_prompts = 2; + cfg.input_len = 8; + cfg.output_len = 4; + cfg.concurrency = 2; + cfg.output_wait = OutputWaitMode::kBlockingC1; + + CHECK_THROWS_WITH_AS( + RunBench(cfg), + "benchmark output wait 'blocking-c1' requires --concurrency 1", + std::invalid_argument); +} + +TEST_CASE("bench: blocking-c1 exercises blocking control with exact tokens") { + BenchConfig cfg; + cfg.num_prompts = 3; + cfg.input_len = 8; + cfg.output_len = 4; + cfg.concurrency = 1; + cfg.seed = 91; + + const BenchResult poll = RunBench(cfg); + cfg.output_wait = OutputWaitMode::kBlockingC1; + const BenchResult blocking = RunBench(cfg); + + CHECK(poll.output_wait == OutputWaitMode::kPoll); + CHECK(poll.blocking_wait_calls == 0); + CHECK(blocking.output_wait == OutputWaitMode::kBlockingC1); + CHECK(blocking.blocking_wait_calls > 0); + CHECK(blocking.completed == poll.completed); + CHECK(blocking.total_input == poll.total_input); + CHECK(blocking.total_output == poll.total_output); + CHECK(blocking.output_token_ids == poll.output_token_ids); + + const nlohmann::json artifact = vllm::bench::ResultJson(cfg, blocking); + CHECK(artifact.at("output_wait") == "blocking-c1"); + CHECK(artifact.at("blocking_wait_calls").get() > 0); + CHECK(artifact.at("output_token_ids") == + nlohmann::json(blocking.output_token_ids)); +} + TEST_CASE("bench: synthetic engine completes all requests with sane metrics") { BenchConfig cfg; cfg.num_prompts = 8; From d304264477a8ba409d63015c86fff2fd3a5b3b2d Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 22:04:06 +0100 Subject: [PATCH 07/11] docs(cpu): record positive A76 C1 polling discriminator (#293) Record the hash-verified current-source Pi control: blocking the sole collector restores the four-thread arm while the three-thread control stays neutral and exact tokens hold. Keep the accepted llama.cpp floor unchanged because the pathological polling arm exceeds the binding spread limit, and leave C2 to fresh review and implementation. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/NOW.md | 2 +- .agents/coordination.md | 11 ++ .agents/engine-matrix.md | 2 +- .agents/kernel-matrix.md | 2 +- .agents/specs/cpu-a76-bf16-gemm.md | 14 +- .agents/specs/cpu-bench-thread-budget.md | 26 ++- docs/BENCHMARKS.md | 2 +- docs/FEATURES.md | 1 + docs/USAGE.md | 4 +- .../rpi5-a76-output-wait-c1-20260810.md | 156 ++++++++++++++++++ 10 files changed, 204 insertions(+), 16 deletions(-) create mode 100644 docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md diff --git a/.agents/NOW.md b/.agents/NOW.md index d680b2c93..9daf0a21c 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -22,7 +22,7 @@ Work: 7 external PRs landed (6 merged + #227 repaired). | Kimi-Linear-48B | 122/128 held; e2e NOT ESTABLISHED | tiktoken-only ckpt: no warm server | | 35B mid-band | **canonical 0.918-0.972x** c1-c32 (@`348c265d`, first c16/c32) | Decode-only window, ONE tool | | Qwen3.5-4B sm_120 | tput **1.0283x** `PENDING`; TTFT/TPOT/E2E 1.085/1.017/1.029x | Profile wave | -| RPi5 A76 | **#293 C1 ACTIVE** | `poll` vs blocking; then F16 | +| RPi5 A76 | **#293 C1 POSITIVE**: T4 decode 2.193x; T3 neutral | Review; C2 event; F16 | | SERVE-ASYNC-MIRROR | **#323 FIXED** (mitigation): graph declines while the mirror is live; 7/7 async gates | Graph read ids at REPLAY | | CPU levers (`QUANT-GGUF-CIQ-GEMM`) | Profile DONE: decode **47% threadpool sync**, prefill **~39% paged attn** | Parakeet encoder; attn dtype hoist | | `SERVE-METRICS` async (#277) | **`/metrics` was DEAD on the shipped server**: AsyncLLM folded nothing. Now live, ctest 366/366 | Config-gated families | diff --git a/.agents/coordination.md b/.agents/coordination.md index 10e6a78c1..02f3007b1 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -1649,6 +1649,17 @@ AsyncLLM/output-event files are excluded even if C1 is decisively positive. Also excludes kernels, model/loader, CUDA/Vulkan, C ABI, assembly, Pi builds, service changes and remote Git operations. +Implementer checkpoint `bdbfffcbf`: native 6/6 (47 assertions), scoped CTest +5/5, dispatch mutation killed by the three intended assertions, AArch64 QEMU +quant 23/23 (150,350 assertions), QEMU bench 6/6 (47 assertions), and A76 +assembly smoke green. The hash-verified same-binary Pi C1 is positive: T4 +blocking/poll is 2.193x decode and 0.458x E2E latency; T3 is neutral; all 12 +token files match. The pathological T4 poll spread is above 3%, so no binding +floor moves. Raw evidence and exact hashes are indexed in +[`rpi5-a76-output-wait-c1-20260810.md`](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md). +This implementation is complete and awaits fresh mutation review. C2 was not +started. + | Claim | Row IDs | Agent | Worktree / remote dir | Branch | Owned scope | State | Last update | |---|---|---|---|---|---|---|---| | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | `KERNEL-GEMM-CPU-ELEM-A76` (`ACTIVE`) | Codex (gpt-5.6-sol), fresh helper implementer | isolated local worktree `/tmp/vllm-a76-bf16-impl`; build/test under buildx/QEMU only; execute/PMU on user-authorized disposable `rich@rpi5fan.lan` | `helper/KERNEL-GEMM-CPU-ELEM-A76-W0-W2`, base spec commit `999e1221e` plus this claim commit | Issue #284 W0-W2 only: refresh both-engine baseline/profile; add the BF16 elementwise benchmark/selection mutation proof; audit disassembly and retain only byte-exact C++/NEON changes with recursive measurements. Owns the new spec/row and scoped edits to `examples/cpu_kernel_bench/main.cpp`, its CLI test/CMake only if needed, `src/vt/cpu/cpu_matmul_elem*`, `src/vt/cpu/cpu_ops.cpp` only if profiling proves partitioning, focused tests, and indexed Pi evidence/record projections. Excludes assembly, model/loader/CUDA/Vulkan/C ABI changes, building on the Pi, service changes and PR/push. | `DONE` | 2026-08-10 — W0 complete and reranked before code: current AsyncLLM T4/T3 proves a runnable-thread/core-budget interaction but does not isolate polling; the old binary is historical only. W1-W4 await the #293 discriminator and fresh profile; no kernel/fixture change retained; row stays `ACTIVE`. | diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index 15db89ec5..d89d93a2d 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -205,7 +205,7 @@ claims it. | `SERVE-HTTP-TRANSPORT` | Serving-socket transport parity: mirror vLLM's uvicorn/asyncio default `TCP_NODELAY` on every accepted SSE socket so per-token stream frames are not held by Nagle against the peer's delayed ACK. Implemented + CPU-tested; the non-binding localhost A/B sizing is COMPLETE and NEUTRAL within noise on c1/c2 ITL/TPOT/throughput (loopback ACKs are instant, so Nagle never coalesces ~100 ms-cadence token frames) — no gate-axis credit expected; the mirror stays for real-network parity. Future keep-alive / read-write-timeout / listening-socket option parity noted, not done | T0 | vLLM serves via uvicorn over asyncio `vllm/entrypoints/launcher.py:71,76`, `vllm/entrypoints/openai/api_server.py:591,630`; asyncio disables Nagle per accepted TCP stream socket `asyncio/base_events.py:192-197` (`_set_nodelay`) called from `asyncio/selector_events.py:950`; cpp-httplib default-off `third_party/httplib/httplib.h:142`, applied on accept only when set `third_party/httplib/httplib.h:12083` | `src/vllm/entrypoints/openai/api_server.cpp:69` (`set_tcp_nodelay(true)` in the ApiServer setup) | behavioral accepted-socket `getsockopt(TCP_NODELAY)` case `tests/vllm/entrypoints/openai/test_api_server.cpp:1076` (helper `:380`); RED accepted `TCP_NODELAY` 0 → GREEN 1, full `test_openai_api_server` **22/22 cases / 242 assertions**; non-binding sizing root `~/work/vllm.cpp-tcpnodelay-sizing/ff915e8…` (raw-set SHA `f5b52900…2128`) neutral within noise; closure [ledger](parity-ledger.md#L451) | [serve-tcp-nodelay.md](specs/serve-tcp-nodelay.md) | `DONE` | `ff915e8` | | `SERVE-C-ABI` | Stable LocalAI-style C FFI (**19** exported `VLLM_API` symbols at `VLLM_ABI_VERSION 10`; blocking and nonblocking request handles. Count corrected 2026-07-24 from a stale `17`, which predated ABI v4/v5 adding `tool_parser`/`reasoning_parser` and the chat entry points; `include/vllm.h` is the source of truth and README:231 already said 19). **ABI v9 2026-07-28 (`CLAIM-CAPI-ENGINE-CONFIG-V9`): the ABI carried strictly LESS engine config than `EngineParams` does** - `max_num_batched_tokens`, the scheduler `scheduling_policy` (`fcfs` / `priority` / `lpm`), and `kv_transfer_config` (the external KV connector / LMCache JSON) were reachable from the bundled server's flags and from NO embedder. All three added, inert at their defaults (zero-filled v8 growth == byte-identical pre-v9 engine); the connector NAME is validated against `KVConnectorFactory` at load, mirroring the server's startup check. `tokenizer_config_path` stopped being a declared-since-v1 no-op and now selects the chat template's source file. Malformed `speculative_config`/`kv_transfer_config` documents now report `VLLM_ERR_INVALID_ARGUMENT` (the contract vllm.h documented since v6) instead of `VLLM_ERR_MODEL_LOAD`, via a catch scoped to the parse block so a real `FromModelDir` failure still reports MODEL_LOAD. Driver: the LocalAI vllm-cpp backend could not expose LMCache or the prefill budget in a model config) | T0 | Original project ABI; pinned vLLM has no C ABI | `include/vllm.h:143,181,207`; `src/capi/vllm_c.cpp:229,264,327,391` | `tests/capi/test_capi.cpp:320,428,505,574,606,640`; `tests/capi/test_dlopen.cpp:77,86`; `tests/capi/c_header_compile.c:1` | [c-api-library.md](specs/c-api-library.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-C-ABI-SPIKE` | | `SERVE-CPP-API` | Rich `LLM` and `AsyncLLM` C++ API | T1 | `vllm/entrypoints/llm.py:66,422`; `vllm/v1/engine/async_llm.py:70` | - | - | `planned: specs/cpp-api.md` | `INVENTORIED` | - | -| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`) | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md) | `PARTIAL` | - | +| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); explicit `poll` / concurrency-one `blocking-c1` control `examples/bench/bench_core.h:426,495,595`; `examples/bench/main.cpp:51` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; C1 mode/rejection/report assertions `tests/examples/test_bench.cpp:18,29-32,61,81,97` and positive Pi control with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | | `ENG-RELEASE-BINARIES` | Downloadable host-ABI-specific `vllm-server` bundles: adaptive CPU and fat CUDA primary artifacts, optional per-SM diagnostics, and literal-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | Required W1-W11/W13 implementation is complete in draft PR #196: ten-SM gencode/AOT, adaptive CPU tiers, extracted-archive validation and supply chain, least-privilege immutable handoff, eight primary bundles, byte-derived indexes, attestation, and exact-file publication; W12 remains optional/non-primary | Local manifest/archive/workflow/mutation gates, full adaptive x86 tier execution, clean CPU archive, and Vulkan 35/35 backend + 11/11 cross-device archive run are green; hosted ten-SM completion, full eight-tuple dry run, matching-hardware gates, and tagged publication remain pending, so no published-binary claim exists | [release-binary-matrix.md](specs/release-binary-matrix.md) | `ACTIVE` | `CLAIM-ENG-RELEASE-BINARIES-W1-W13` | diff --git a/.agents/kernel-matrix.md b/.agents/kernel-matrix.md index 65c0a9fae..d368df34c 100644 --- a/.agents/kernel-matrix.md +++ b/.agents/kernel-matrix.md @@ -112,7 +112,7 @@ host/sched. Detail: state `KERNEL-FA2-GQA-SWAP-FLIP`. | ID | Item | Upstream | Our code | Tests/evidence | Spike/spec | State | Owner | |---|---|---|---|---|---|---|---| -| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); no issue-specific implementation retained | Accepted same-file Pi floor remains 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS. Fresh W0 diagnostic reranked before implementation: current AsyncLLM T4/T3 shows a runnable-thread/core-budget interaction without attributing polling; summed cycles rank barrier 42.21%, reached **F16** `Bt16Neon` 33.09%, Q8 12.05%. T3 restores decode but trades prefill; current TTFT spread is non-binding. W1-W4 await the [`SERVE-CLI-BENCH` #293 control](https://github.com/mudler/vllm.cpp/issues/293), its [spec](specs/cpu-bench-thread-budget.md), and a fresh profile ([W0 evidence](../docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md), [issue #284](https://github.com/mudler/vllm.cpp/issues/284)) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `ACTIVE` | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | +| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); no issue-specific implementation retained | Accepted same-file Pi floor remains 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS. W0 reranked before implementation: summed cycles ranked barrier 42.21%, reached **F16** `Bt16Neon` 33.09%, Q8 12.05%. Same-source #293 C1 now attributes the four-core collapse to benchmark polling: blocking/poll is 2.193x decode and 0.458x E2E latency at T4, neutral at T3, exact tokens; the binding floor stays unchanged because T4 poll spread exceeds 3%. W1-W4 await fresh C1 review, C2's general event wait and a fresh profile ([W0 evidence](../docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md), [C1 evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md), [issue #284](https://github.com/mudler/vllm.cpp/issues/284)) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `ACTIVE` | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | | `KERNEL-CPU-A76-Q8-DOT` | Cortex-A76 Q8_0 x Q8_0 SDOT, exact-order C++ and scheduled AAPCS64 variants for the Pi decode/prefill keep-quant GEMM | llama.cpp `ggml/src/ggml-cpu/quants.c:400` portable dot and `arch/arm/quants.c:1076-1160` Arm DotProd @ project pin `237ad9b96`; Qwen semantics remain vLLM `555967922` | [ACLE SDOT + HWCAP/MIDR selector](../src/vt/cpu/cpu_quant_dot_sdot.cpp#L85), [AAPCS64 two-block leaf](../src/vt/cpu/cpu_quant_dot_a76.S#L10), [Q8 dispatch](../src/vt/cpu/cpu_quant_dot.cpp#L675), [same-binary variants](../examples/cpu_kernel_bench/main.cpp#L858) | [direct exact variant/edge tests](../tests/vt/test_ops_quant_dot.cpp#L593); QEMU 20/20, 150258 assertions; physical Pi exact checksums/tokens; assembly vs compiler SDOT +3.66% M1/T1, +5.08% M128/T1, +3.69% M128/T4, but −2.43% M1/T4. Same-file llama.cpp floor measured/NOT MET: vllm.cpp 0.461x prefill / 0.653x decode+E2E, 0.758x peak RSS (better), exact-prompt output equal; [assembly evidence](../docs/bench-evidence/rpi5-a76-q8-dot-20260806.md#L40), [competitor evidence](../docs/bench-evidence/rpi5-a76-llamacpp-20260806.md) | [A76 Q8 dot spec](specs/cpu-a76-q8-dot.md) | `GATING` | - | | `KERNEL-ACCEL-PROVIDER-SELECT` | **WHICH implementation of an op runs, when more than one exists on a device** — the selection layer above every kernel family in this matrix. Distinct from `KERNEL-CUDA-DISPATCH-AOT`, which is about which ARCH a CUDA kernel is compiled/selected for; this is about which PROVIDER (ours, a vendor library, MLX, llama.cpp) serves the op at all | no single upstream file: this is the shape vLLM's runtime chain uses everywhere — flashinfer tactic registries, cuBLASLt/CUTLASS per-call heuristics, and torch's backend selection — rather than compile-time pinning | `vt::OpProvider` [op_provider.h](../include/vt/op_provider.h) + [op_provider.cpp](../src/vt/op_provider.cpp); the flat `[OpId][DeviceType]` `void*` table it replaces is gone from [ops.cpp](../src/vt/ops.cpp) with the ~70 op wrappers untouched. Providers registered today: `vt-native` (every backend kernel in the tree, priority 0, unconditional — behaviour preserved exactly), `mlx` (priority 100, Metal `kMatmul`/`kMatmulBT`, build-gated `VLLM_CPP_MLX`, [metal_mlx_provider.mm](../src/vt/metal/metal_mlx_provider.mm)), and — NEW 2026-07-23 (`CLAIM-BACKEND-SEAM-S5-1`, work row `S5`) — **`vt-cpu-ref` (priority −1000, the portable reference tier)**: the CPU kernel installed LAZILY as a negative-priority fallback on a UNIFIED-MEMORY device's first `GetOp` miss, mirroring `custom_op.py:138 forward_native`, so a partial backend runs an op it lacks natively instead of throwing. Native always wins (priority); gated on `Backend::UnifiedMemory()` (a discrete GPU never gets it — a CPU kernel on true device memory is corruption); observable via `GetReferenceTierHits()` + a one-time loud stderr line | [test_op_provider.cpp](../tests/vt/test_op_provider.cpp) 11 cases / 47 assertions — deterministic selection under REVERSED registration order, name tie-break, duplicate rejection, capability predicate, caps re-resolution, decline-and-fall-back, stats, runtime disable; [test_metal_backend.cpp](../tests/vt/test_metal_backend.cpp) 9 cases / 108 assertions on the M4 with MLX ON, including MLX-vs-MSL-vs-CPU NMSE per op at real shapes and an end-to-end DECLINE; **[test_reference_tier.cpp](../tests/vt/test_reference_tier.cpp) (S5): discrete-device refusal + unified-device zero-native-kernel fallback correctness + native-wins + observability, hardware-free via a fake backend on `kXPU`.** Linux CPU 156/156; dgx regression set ALL UNCHANGED — anchor `tests/vt/test_op_provider.cpp:64` | [Metal/MLX reuse study §6](specs/metal-mlx-reuse-study.md); [accelerator-seam-audit §10](specs/accelerator-seam-audit.md); [drop-in kernel ABI](specs/dropin-kernel-abi.md) (the complementary ARGUMENT half) | `ACTIVE` — mechanism landed and gated with THREE provider kinds (`vt-native`, `mlx`, `vt-cpu-ref`); the CUDA/CPU/Vulkan vendor provider rows it was designed for are not yet populated (so the row is deliberately left open) | `CLAIM-BACKEND-ACCEL-PROVIDER-1` | | `KERNEL-CUDA-DISPATCH-AOT` | CUDA runtime, streams, graphs, per-arch dispatch, and generated/AOT artifact selection | platform selection `vllm/platforms/cuda.py:205-493`; gencode handling `CMakeLists.txt:201-220`; JIT packages `cmake/external_projects/triton_kernels.cmake:1-28` | [cuda_backend.cu:20](../src/vt/cuda/cuda_backend.cu#L20), [CMakeLists.txt:37](../CMakeLists.txt#L37), [TritonAOT.cmake:57](../cmake/TritonAOT.cmake#L57) | [CUDA backend tests](../tests/vt/test_cuda_backend.cpp#L31); the Triton GDN AOT tree is now vendored per-arch — `sm_121a` (RUNTIME-VERIFIED) plus `sm_80/86/89/90a/100a` (DERIVED+BUILD-VERIFIED, `CLAIM-TRITON-AOT-PER-ARCH`, 2026-07-28), each a real per-target cubin (`cuobjdump sm=80/86/89/90/100`); the CUDA fast-path arch fan-out (non-GDN) remains as evidenced in `backend-matrix.md` | [inventory](specs/kernel-family-inventory.md); [per-arch Triton AOT](specs/triton-aot-per-arch.md) | `ANCHOR-BACKFILL` | - | diff --git a/.agents/specs/cpu-a76-bf16-gemm.md b/.agents/specs/cpu-a76-bf16-gemm.md index 484d40257..bb871247b 100644 --- a/.agents/specs/cpu-a76-bf16-gemm.md +++ b/.agents/specs/cpu-a76-bf16-gemm.md @@ -250,9 +250,11 @@ official b9892 denominator reproduced at 27.96 prefill and 3.907 decode tok/s. Full commands, samples, artifact hashes and raw evidence hashes are in the [W0 reranking evidence](../../docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md). -Disposition: the row remains `ACTIVE`, but W1-W4 are blocked until the -[`SERVE-CLI-BENCH` control in issue #293](https://github.com/mudler/vllm.cpp/issues/293) -and its [committed spec](cpu-bench-thread-budget.md) resolve or reject the -frontend-polling hypothesis, followed by a new four-core profile that ranks the -reached elementwise kernel. The BF16 fixture, C++ schedule candidates and -assembly gate were not attempted. Assembly remains forbidden. +Disposition: the row remains `ACTIVE`. The +[`SERVE-CLI-BENCH` C1 control in issue #293](https://github.com/mudler/vllm.cpp/issues/293) +is now positive at the implementer checkpoint: T4 blocking restores 2.193x +decode while T3 is neutral ([C1 evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)). +W1-W4 remain blocked on fresh C1 review, C2's general multi-request event wait, +and the subsequent four-core profile that ranks the reached elementwise +kernel. The BF16 fixture, C++ schedule candidates and assembly gate were not +attempted. Assembly remains forbidden. diff --git a/.agents/specs/cpu-bench-thread-budget.md b/.agents/specs/cpu-bench-thread-budget.md index dba015d53..0ae6880d9 100644 --- a/.agents/specs/cpu-bench-thread-budget.md +++ b/.agents/specs/cpu-bench-thread-budget.md @@ -146,10 +146,10 @@ upstream event semantics and extend the local original-harness tests: | W | Deliverable | State | |---|---|---| | C0 | Correct #284 evidence wording and register #293 | complete (`0a07c53db`) | -| C1 | Same-current-source `poll` vs `blocking-c1` control + RED mutation | active | -| C2 | If C1 positive, general event/epoch wait + deterministic multi-request tests | blocked by C1 | -| C3 | QEMU gate and interleaved Pi T4/T3 causal profile | blocked by C1 | -| C4 | Retain event wait or reject it; test reserved-core budget only if C1 is negative/insufficient | blocked by C3 | +| C1 | Same-current-source `poll` vs `blocking-c1` control + RED mutation | implemented at `bdbfffcbf`; fresh review pending | +| C2 | If C1 positive, general event/epoch wait + deterministic multi-request tests | unblocked by positive C1; excluded from this implementation | +| C3 | QEMU gate and interleaved Pi T4/T3 causal profile | complete ([evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | +| C4 | Retain event wait or reject it; test reserved-core budget only if C1 is negative/insufficient | blocked by C2 + fresh review | | C5 | Full model versus llama.cpp; return to the measured F16/kernel ranking | blocked by C4 | ## Risks/decisions @@ -170,4 +170,20 @@ upstream event semantics and extend the local original-harness tests: ## Outcome -Pending C1 current-source discriminator. +C1 implementer result: **positive, fresh review pending**. On the exact +current-source AArch64 binary, three interleaved idle/unthrottled processes per +arm show T4 `blocking-c1` / `poll` medians of 2.193x decode, 2.004x prefill and +0.458x E2E latency, while the T3 control is neutral at 1.003x decode, 1.001x +prefill and 0.997x E2E latency. All 12 token files exactly match the accepted +SHA-256; RSS is neutral. T4 blocking retires 76.5% fewer instructions and uses +54.2% less task-clock. This isolates scan-and-yield frontend contention as the +cause of the four-core collapse. + +No replacement performance floor is accepted here: the pathological T4 poll +arm still has 5.9-10.7% spread and fails the sub-3% binding gate, although the +effect is positive in every pair and the blocking/T3 arms are stable below +0.52%. `blocking-c1` remains an explicit concurrency-one control and `poll` +remains the default. C2's lost-wakeup-safe general event wait requires a fresh +implementer after review; it was deliberately not started. Full hashes, native +and QEMU counts, host state, raw evidence and medians are in the +[C1 evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md). diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index d6b4aabce..a296b0544 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -248,7 +248,7 @@ host mirror is freed once the device Marlin resident is built. | 64-token Qwen model gate | Byte-identical across x86, portable, SDOT and assembly arms; asm vs SDOT median TTFT -1.55%, TPOT neutral, E2E -0.13%; vs portable TTFT -33.40%, E2E -2.67%. Cortex-A76+DotProd selects assembly by default | | Same-file llama.cpp floor (pp17/tg64) | **NOT MET on speed**: prefill 12.81 vs 27.77 tok/s (0.461x), decode 2.55 vs 3.91 (0.653x), E2E 26,018.39 vs 16,998.49 ms ([competitor evidence](bench-evidence/rpi5-a76-llamacpp-20260806.md)) | | Peak RSS | **2.841 vs 3.747 GiB, 24.2% less**; 3 clean unthrottled reps; same-text 64-token greedy output byte-identical after trailing-newline normalization | -| #284 W0 | Barrier first; F16 next; #293 tests polling. | +| #293 C1 positive | T4 decode 2.193x; T3 neutral; exact tokens; poll spread keeps floor unchanged. | Same GGUF file both arms, `dgx.casa` GB10 aarch64 (20 cores), idle, 3 reps, llama.cpp `237ad9b96` built fresh on the same host. diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 816cdd347..fd0e547ca 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -43,6 +43,7 @@ are our reading of their documented behavior, not measurements. | CUDA graph decode capture | ◐ per-family | ✅ | ✅ | ✅ | | Partial-prefill concurrency | ☐ | ✅ | ✅ | ☐ | | Cascade attention | ☐ | ✅ | ◐ | ☐ | +| Benchmark output-wait selection | ◐ `poll` default; blocking diagnostic at c1 | n/a | n/a | n/a | ## KV cache and memory diff --git a/docs/USAGE.md b/docs/USAGE.md index bb2c45e30..987c472c4 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -133,7 +133,9 @@ Two more example binaries ship alongside it: `--max-num-batched-tokens`, and `--num-blocks`. It pretokenizes before timing and atomically publishes each concurrency wave. Set `VT_BENCH_PRETOKENIZE=0` for the timed-string rollback; the report names the - resolved mode. + resolved mode. Its `--output-wait` mode + defaults to `poll`; the explicit `blocking-c1` diagnostic is valid only with + `--concurrency 1` and is not a general multi-request policy. - `tokenize` ([`examples/tokenize/main.cpp`](../examples/tokenize/main.cpp)), a tokenizer smoke tool taking ` `. GGUF `tokenizer.ggml.pre` names accepted: `qwen35`, `qwen2`, `llama-bpe`, diff --git a/docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md b/docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md new file mode 100644 index 000000000..6da9b7544 --- /dev/null +++ b/docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md @@ -0,0 +1,156 @@ +# Raspberry Pi 5 output-wait C1 discriminator, 2026-08-10 + +This is the current-source causal control for +[`SERVE-CLI-BENCH` issue #293](https://github.com/mudler/vllm.cpp/issues/293) +and the prerequisite discriminator requested by +[`KERNEL-GEMM-CPU-ELEM-A76` issue #284](https://github.com/mudler/vllm.cpp/issues/284). +It compares the existing yielding poll loop with a concurrency-one-only +blocking wait in the same binary. It does not implement the general +multi-request event mechanism and does not change the default. + +## Decision + +The C1 polling hypothesis is **positive**, pending the required fresh mutation +review and operator gate: + +- at four compute threads, `blocking-c1` improves median per-stream decode + 2.193x, output throughput 2.185x and prefill 2.004x; median E2E falls from + 56.993 to 26.081 seconds; +- at three compute threads, the same control is neutral: decode is 1.003x, + prefill 1.001x and E2E is 0.997x relative to polling; +- all 12 runs emit the accepted token-file SHA-256 + `0ec98eabb23e4148d540fcf79a2fe61678fb90fe462cdf28134af7a42fe6a826`; + the blocking arm records exactly 64 blocking calls per run and polling + records zero; and +- T4 PMU medians fall by 46.1% cycles, 76.5% instructions and 54.2% task-clock + under the blocking control. At T3 the corresponding changes are 5.1%, 5.5% + and 25.0%, with no material wall-time change. + +This establishes that the benchmark consumer's scan-and-yield policy causes +the four-core collapse. It does not make `blocking-c1` a general solution: +blocking an arbitrary collector at concurrency greater than one can delay a +ready peer, so that mode deliberately rejects `--concurrency` above one. + +The existing llama.cpp floor remains unchanged. The T4 polling arm is itself +pathological and has 5.9-10.7% spread across latency/throughput axes, above the +3% acceptance limit. The causal effect occurs in every interleaved pair and +the blocking/T3 arms are stable below 0.52%, but these medians are not promoted +to a replacement binding floor. C2 must implement and independently review the +lost-wakeup-safe general event/epoch wait before the A76 full-model gate and +fresh kernel profile resume. + +## Same-binary result + +All values are medians of three separate, interleaved processes. Higher is +better for tok/s; lower is better for latency and RSS. `blocking / poll` is the +ratio in every final column. + +| Threads | Mode | Prefill tok/s | Decode tok/s | Output tok/s | TTFT ms | TPOT ms | E2E ms | Peak RSS KiB | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 4 | poll | 4.9219 | 1.1797 | 1.1229 | 3,453.92 | 847.69 | 56,992.80 | 2,978,448 | +| 4 | blocking-c1 | 9.8644 | 2.5865 | 2.4539 | 1,723.37 | 386.62 | 26,080.51 | 2,978,704 | +| 4 | blocking / poll | **2.004x** | **2.193x** | **2.185x** | **0.499x** | **0.456x** | **0.458x** | 1.0001x | +| 3 | poll | 8.1268 | 2.5353 | 2.3755 | 2,091.85 | 394.43 | 26,941.99 | 2,978,192 | +| 3 | blocking-c1 | 8.1389 | 2.5424 | 2.3816 | 2,088.74 | 393.33 | 26,872.68 | 2,978,704 | +| 3 | blocking / poll | 1.001x | 1.003x | 1.003x | 0.999x | 0.997x | 0.997x | 1.0002x | + +The individual T4 results show that the conclusion does not depend on the +median-selected process: + +| Rep | Mode | Prefill tok/s | Decode tok/s | TTFT ms | TPOT ms | E2E ms | +|---:|---|---:|---:|---:|---:|---:| +| 1 | poll | 5.2644 | 1.1683 | 3,229.23 | 855.94 | 57,153.65 | +| 1 | blocking-c1 | 9.8609 | 2.5744 | 1,723.97 | 388.44 | 26,195.70 | +| 2 | blocking-c1 | 9.8644 | 2.5865 | 1,723.37 | 386.62 | 26,080.51 | +| 2 | poll | 4.9219 | 1.2518 | 3,453.92 | 798.86 | 53,782.22 | +| 3 | poll | 4.7375 | 1.1797 | 3,588.39 | 847.69 | 56,992.80 | +| 3 | blocking-c1 | 9.9050 | 2.5876 | 1,716.30 | 386.45 | 26,062.78 | + +Maximum-minus-minimum spread about the median is 0.45-0.51% for the T4 +blocking performance axes, 0.14-0.43% for both T3 arms, and 5.92-10.71% for +the T4 polling axes. Peak RSS varies by at most 0.02% in every arm. + +## PMU result + +`perf stat` wrapped the entire process at every leg. These are median totals; +the stat run is counting-only, so a lost-sample count is not applicable. + +| Threads | Mode | Cycles | Instructions | Task-clock ns | Instructions/cycle from median totals | +|---:|---|---:|---:|---:|---:| +| 4 | poll | 462,788,342,577 | 1,162,875,406,457 | 228,289,315,838 | 2.513 | +| 4 | blocking-c1 | 249,435,618,828 | 273,661,479,884 | 104,649,636,414 | 1.097 | +| 4 | blocking / poll | 0.539x | 0.235x | 0.458x | - | +| 3 | poll | 203,695,408,808 | 265,302,921,340 | 108,157,222,000 | 1.302 | +| 3 | blocking-c1 | 193,367,448,020 | 250,728,593,259 | 81,105,074,218 | 1.297 | +| 3 | blocking / poll | 0.949x | 0.945x | 0.750x | - | + +The T3 poll arm consumes about 3.93 CPUs for the same wall time as the +2.96-CPU blocking arm. At T4, polling consumes about 3.97 CPUs for more than +twice as long and retires over four times the instructions. That matched +control is the direct evidence for frontend contention; the earlier W0 barrier +profile alone did not identify the responsible thread. + +## Correctness, build and host + +- Source head: `bdbfffcbf8b02f4c54fb1a0000c7dab7c10f5c3d`, based on operator + integration head `9ec8cae23` and fetched `upstream/main` `7d45913a7`. +- AArch64 Release `vllm-bench`: SHA-256 + `c6bdb1a46f433e40bc18dddf43cdc85855c1e6ad4e74ec81b28d6fa5982329f6`, + GNU build ID `d01ab3ed05056ce805ded8bd1a132b07bb452b65`. +- Model: `Qwen3.5-2B-UD-Q8_K_XL.gguf`, 2,834,940,160 bytes, SHA-256 + `a53988df91157d78acaf3c95e22db179d13f6236061bdb86576494dc99b1bc3b`. +- Workload: one request, 17 actual input tokens, 64 output tokens, + concurrency one, seed zero, temperature zero, cores 0-3, + `VT_CPU_Q8_DOT=auto`, and `VLLM_CPP_CPU_THREADS` 4 or 3. +- Host: `rpi5fan`, four Cortex-A76 cores, Linux + `6.18.34+rpt-rpi-2712`, `ondemand` governor; captured frequencies + 1.9-2.4 GHz, temperatures 49.9-63.1 C, and `throttled=0x0` before and after + every leg. No process used at least 1% CPU before any leg. +- Native focused gate: 6/6 cases, 47 assertions. Full scoped native CTest: + 5/5 (`test_bench`, `test_async_llm`, `test_output_processor`, + `test_engine_core_proc`, `test_cpu_threadpool`). +- QEMU/buildx AArch64 gate: quant 23/23 cases and 150,350 assertions; + benchmark 6/6 cases and 47 assertions; A76 assembly smoke selected + `a76-asm` and completed with checksum `0xbb60dafd79d08ed6`. +- Dispatch mutation: changing the `blocking-c1` branch to `poll` fails the + three intended blocking-call assertions; the source was restored to SHA-256 + `5d5bcb3330c1164a10608d5c60b2fbbf4972681e3a6cb1437053eb0404e4efbd`. +- CLI rejection gate exits 2 with `benchmark output wait 'blocking-c1' + requires --concurrency 1`. + +## Raw evidence and recipe + +The immutable raw set is on the Pi at +`~/vllm-cpp-assembly/evidence/a76-c1-bdbfffcbf-20260810-rerun1/`. +Its `SHA256SUMS` contains 98 entries and has SHA-256 +`a1ae17387fff08ea454fd19ce33c7fd21f8b8ca14aa2ffb4c59b7cacef77be78`. +Every entry was verified on the Pi and again after copying the set locally. +Each leg contains JSON, exact token IDs, stdout/stderr, `perf stat`, fresh +process `getrusage`, and before/after host snapshots. The harness and +standard-library RSS wrapper have SHA-256 +`35166849604a534ebbc7d0969bbc7624bcb8931bbbc30db38a5ea92f3ab599f6` +and `ad310102e1d88fa6395c2c41a9594f62d195891bdb1b79e322cd31611db805f5`. + +The first harness attempt stopped before inference because the Pi image has no +GNU `/usr/bin/time`; that partial set remains separately at +`~/vllm-cpp-assembly/evidence/a76-c1-bdbfffcbf-20260810/`. The accepted rerun +uses a fresh standard-library Python process per leg and +`resource.getrusage(RUSAGE_CHILDREN).ru_maxrss`; the binary and workload are +unchanged. + +The core command in each interleaved leg was: + +```sh +perf stat -x, -e cycles:u,instructions:u,task-clock,context-switches,cpu-migrations -- \ + taskset -c 0-3 env VLLM_CPP_CPU_THREADS=<3|4> VT_CPU_Q8_DOT=auto \ + vllm-bench --model Qwen3.5-2B-UD-Q8_K_XL.gguf \ + --num-prompts 1 --input-len 16 --output-len 64 --concurrency 1 \ + --seed 0 --temperature 0 --output-wait \ + --output-token-ids .tokens.json --output-json .json +``` + +For diagnostic continuity only, the C1 T4 blocking medians are 0.353x +llama.cpp b9892 prefill, 0.662x decode, 0.651x output-equivalent throughput and +1.535x E2E latency using the previous shape-matched denominator. Prompt content +still differs between the two harnesses, so this is not a new cross-engine +binding comparison and no llama.cpp floor is moved. From d810936955d3c3b5b6746e5879a2b1086cac79eb Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 10 Aug 2026 23:35:29 +0100 Subject: [PATCH 08/11] test(bench): bind C1 nowait and text reporting (#293) Add a test-only wait trace that proves each blocking-c1 wait follows an empty nowait probe, cover the human-readable report, and repair the row anchors. The trace compiles out of production; native and AArch64 vllm-bench artifacts retain the reviewed executable bytes. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/engine-matrix.md | 2 +- examples/bench/bench_core.h | 39 +++++++++++++++++++++- tests/examples/test_bench.cpp | 61 +++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index d89d93a2d..fbb334901 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -205,7 +205,7 @@ claims it. | `SERVE-HTTP-TRANSPORT` | Serving-socket transport parity: mirror vLLM's uvicorn/asyncio default `TCP_NODELAY` on every accepted SSE socket so per-token stream frames are not held by Nagle against the peer's delayed ACK. Implemented + CPU-tested; the non-binding localhost A/B sizing is COMPLETE and NEUTRAL within noise on c1/c2 ITL/TPOT/throughput (loopback ACKs are instant, so Nagle never coalesces ~100 ms-cadence token frames) — no gate-axis credit expected; the mirror stays for real-network parity. Future keep-alive / read-write-timeout / listening-socket option parity noted, not done | T0 | vLLM serves via uvicorn over asyncio `vllm/entrypoints/launcher.py:71,76`, `vllm/entrypoints/openai/api_server.py:591,630`; asyncio disables Nagle per accepted TCP stream socket `asyncio/base_events.py:192-197` (`_set_nodelay`) called from `asyncio/selector_events.py:950`; cpp-httplib default-off `third_party/httplib/httplib.h:142`, applied on accept only when set `third_party/httplib/httplib.h:12083` | `src/vllm/entrypoints/openai/api_server.cpp:69` (`set_tcp_nodelay(true)` in the ApiServer setup) | behavioral accepted-socket `getsockopt(TCP_NODELAY)` case `tests/vllm/entrypoints/openai/test_api_server.cpp:1076` (helper `:380`); RED accepted `TCP_NODELAY` 0 → GREEN 1, full `test_openai_api_server` **22/22 cases / 242 assertions**; non-binding sizing root `~/work/vllm.cpp-tcpnodelay-sizing/ff915e8…` (raw-set SHA `f5b52900…2128`) neutral within noise; closure [ledger](parity-ledger.md#L451) | [serve-tcp-nodelay.md](specs/serve-tcp-nodelay.md) | `DONE` | `ff915e8` | | `SERVE-C-ABI` | Stable LocalAI-style C FFI (**19** exported `VLLM_API` symbols at `VLLM_ABI_VERSION 10`; blocking and nonblocking request handles. Count corrected 2026-07-24 from a stale `17`, which predated ABI v4/v5 adding `tool_parser`/`reasoning_parser` and the chat entry points; `include/vllm.h` is the source of truth and README:231 already said 19). **ABI v9 2026-07-28 (`CLAIM-CAPI-ENGINE-CONFIG-V9`): the ABI carried strictly LESS engine config than `EngineParams` does** - `max_num_batched_tokens`, the scheduler `scheduling_policy` (`fcfs` / `priority` / `lpm`), and `kv_transfer_config` (the external KV connector / LMCache JSON) were reachable from the bundled server's flags and from NO embedder. All three added, inert at their defaults (zero-filled v8 growth == byte-identical pre-v9 engine); the connector NAME is validated against `KVConnectorFactory` at load, mirroring the server's startup check. `tokenizer_config_path` stopped being a declared-since-v1 no-op and now selects the chat template's source file. Malformed `speculative_config`/`kv_transfer_config` documents now report `VLLM_ERR_INVALID_ARGUMENT` (the contract vllm.h documented since v6) instead of `VLLM_ERR_MODEL_LOAD`, via a catch scoped to the parse block so a real `FromModelDir` failure still reports MODEL_LOAD. Driver: the LocalAI vllm-cpp backend could not expose LMCache or the prefill budget in a model config) | T0 | Original project ABI; pinned vLLM has no C ABI | `include/vllm.h:143,181,207`; `src/capi/vllm_c.cpp:229,264,327,391` | `tests/capi/test_capi.cpp:320,428,505,574,606,640`; `tests/capi/test_dlopen.cpp:77,86`; `tests/capi/c_header_compile.c:1` | [c-api-library.md](specs/c-api-library.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-C-ABI-SPIKE` | | `SERVE-CPP-API` | Rich `LLM` and `AsyncLLM` C++ API | T1 | `vllm/entrypoints/llm.py:66,422`; `vllm/v1/engine/async_llm.py:70` | - | - | `planned: specs/cpp-api.md` | `INVENTORIED` | - | -| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); explicit `poll` / concurrency-one `blocking-c1` control `examples/bench/bench_core.h:426,495,595`; `examples/bench/main.cpp:51` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; C1 mode/rejection/report assertions `tests/examples/test_bench.cpp:18,29-32,61,81,97` and positive Pi control with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | +| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); output-wait selector, parser and default `examples/bench/bench_core.h:69-101,125-132`; direct validation and wait implementation `examples/bench/bench_core.h:495-501,636-676`; CLI parser and direct validation `examples/bench/main.cpp:94-102,128-135` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; text-report, direct-rejection, nowait-before-blocking, JSON and exact-token assertions `tests/examples/test_bench.cpp:21-119`; positive Pi C1 with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | | `ENG-RELEASE-BINARIES` | Downloadable host-ABI-specific `vllm-server` bundles: adaptive CPU and fat CUDA primary artifacts, optional per-SM diagnostics, and literal-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | Required W1-W11/W13 implementation is complete in draft PR #196: ten-SM gencode/AOT, adaptive CPU tiers, extracted-archive validation and supply chain, least-privilege immutable handoff, eight primary bundles, byte-derived indexes, attestation, and exact-file publication; W12 remains optional/non-primary | Local manifest/archive/workflow/mutation gates, full adaptive x86 tier execution, clean CPU archive, and Vulkan 35/35 backend + 11/11 cross-device archive run are green; hosted ten-SM completion, full eight-tuple dry run, matching-hardware gates, and tagged publication remain pending, so no published-binary claim exists | [release-binary-matrix.md](specs/release-binary-matrix.md) | `ACTIVE` | `CLAIM-ENG-RELEASE-BINARIES-W1-W13` | diff --git a/examples/bench/bench_core.h b/examples/bench/bench_core.h index ec52916c8..7c5ff264f 100644 --- a/examples/bench/bench_core.h +++ b/examples/bench/bench_core.h @@ -73,6 +73,17 @@ enum class OutputWaitMode { kBlockingC1, }; +#ifdef VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE +// Test-only observation seam. Production benchmark translation units do not +// define VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE, so tracing adds no state or work to +// the measured loop. +enum class OutputWaitTestEvent { + kNowaitReady, + kNowaitEmpty, + kBlockingWait, +}; +#endif + inline const char* OutputWaitModeName(OutputWaitMode mode) { switch (mode) { case OutputWaitMode::kPoll: @@ -118,6 +129,9 @@ struct BenchConfig { // concurrency-1-only `blocking-c1` control performs the same nowait probe, // then blocks on the sole collector so Pi measurements can isolate polling. OutputWaitMode output_wait = OutputWaitMode::kPoll; +#ifdef VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE + std::vector* output_wait_test_trace = nullptr; +#endif uint64_t seed = 0; // prompt-generation RNG seed. double temperature = 0; // <= 0 => greedy (deterministic). bool quiet = false; // suppress per-progress logging to stderr. @@ -742,11 +756,27 @@ inline BenchResult RunBench(const BenchConfig& cfg) { return std::next(it); }; +#ifdef VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE +#define VLLM_BENCH_TRACE_NOWAIT(expression) \ + ([&]() { \ + std::optional traced_ready = (expression); \ + if (cfg.output_wait_test_trace != nullptr) { \ + cfg.output_wait_test_trace->push_back( \ + traced_ready.has_value() ? OutputWaitTestEvent::kNowaitReady \ + : OutputWaitTestEvent::kNowaitEmpty); \ + } \ + return traced_ready; \ + }()) +#else +#define VLLM_BENCH_TRACE_NOWAIT(expression) (expression) +#endif + admit(); while (done < cfg.num_prompts) { bool observed_output = false; for (auto it = active.begin(); it != active.end();) { - std::optional ready = engine.get_output_nowait(it->second); + std::optional ready = VLLM_BENCH_TRACE_NOWAIT( + engine.get_output_nowait(it->second)); if (!ready.has_value()) { ++it; continue; @@ -764,6 +794,12 @@ inline BenchResult RunBench(const BenchConfig& cfg) { "blocking-c1 benchmark must have exactly one active request"); } auto it = active.begin(); +#ifdef VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE + if (cfg.output_wait_test_trace != nullptr) { + cfg.output_wait_test_trace->push_back( + OutputWaitTestEvent::kBlockingWait); + } +#endif ++blocking_wait_calls; it = consume_output(it, engine.get_output(it->second)); (void)it; @@ -775,6 +811,7 @@ inline BenchResult RunBench(const BenchConfig& cfg) { } admit(); // keep C in flight as requests finish. } +#undef VLLM_BENCH_TRACE_NOWAIT const double dur_s = now_s(); diff --git a/tests/examples/test_bench.cpp b/tests/examples/test_bench.cpp index e3618b881..bf67302a0 100644 --- a/tests/examples/test_bench.cpp +++ b/tests/examples/test_bench.cpp @@ -4,6 +4,7 @@ // — this asserts the HARNESS: all N requests finish, throughput > 0, TTFT > 0, // and the token accounting is coherent. The real parity numbers come from a GB10 // run with --model (dgx-pending), which this same code path drives. +#define VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE #include "bench_core.h" #include @@ -23,6 +24,7 @@ using vllm::bench::BenchResult; using vllm::bench::DispatchBenchPromptAdmission; using vllm::bench::DispatchBenchPromptWaveAdmission; using vllm::bench::OutputWaitMode; +using vllm::bench::OutputWaitTestEvent; using vllm::bench::PretokenizeBenchPromptsThenStartClock; using vllm::bench::RunBench; @@ -297,6 +299,37 @@ TEST_CASE("bench: pretokenized vectors match timed-string InputProcessor") { CHECK(clock_value == 17); } +TEST_CASE("bench: text report identifies the exercised output wait") { + BenchConfig cfg; + cfg.concurrency = 1; + BenchResult result; + result.async_frontend = true; + result.output_wait = OutputWaitMode::kBlockingC1; + result.blocking_wait_calls = 37; + + std::FILE* output = std::tmpfile(); + REQUIRE(output != nullptr); + vllm::bench::PrintReport(cfg, result, output); + REQUIRE(std::fflush(output) == 0); + REQUIRE(std::fseek(output, 0, SEEK_END) == 0); + const long size = std::ftell(output); + REQUIRE(size >= 0); + REQUIRE(std::fseek(output, 0, SEEK_SET) == 0); + std::string text(static_cast(size), '\0'); + CHECK(std::fread(text.data(), 1, text.size(), output) == text.size()); + std::fclose(output); + + auto report_line = [&](std::string_view label) { + const size_t begin = text.find(label); + REQUIRE(begin != std::string::npos); + const size_t end = text.find('\n', begin); + REQUIRE(end != std::string::npos); + return text.substr(begin, end - begin); + }; + CHECK(report_line("Output wait:").find("blocking-c1") != std::string::npos); + CHECK(report_line("Blocking wait calls:").find("37") != std::string::npos); +} + TEST_CASE("bench: blocking-c1 selector rejects concurrent requests by name") { BenchConfig cfg; cfg.num_prompts = 2; @@ -312,6 +345,19 @@ TEST_CASE("bench: blocking-c1 selector rejects concurrent requests by name") { } TEST_CASE("bench: blocking-c1 exercises blocking control with exact tokens") { + const std::filesystem::path bench_core = + std::filesystem::path(__FILE__).parent_path().parent_path().parent_path() / + "examples/bench/bench_core.h"; + std::ifstream bench_core_stream(bench_core); + REQUIRE(bench_core_stream.good()); + const std::string bench_core_source{ + std::istreambuf_iterator(bench_core_stream), + std::istreambuf_iterator()}; + REQUIRE(bench_core_source.find( + "VLLM_BENCH_TRACE_NOWAIT(\n" + " engine.get_output_nowait(it->second))") != + std::string::npos); + BenchConfig cfg; cfg.num_prompts = 3; cfg.input_len = 8; @@ -321,6 +367,8 @@ TEST_CASE("bench: blocking-c1 exercises blocking control with exact tokens") { const BenchResult poll = RunBench(cfg); cfg.output_wait = OutputWaitMode::kBlockingC1; + std::vector output_wait_trace; + cfg.output_wait_test_trace = &output_wait_trace; const BenchResult blocking = RunBench(cfg); CHECK(poll.output_wait == OutputWaitMode::kPoll); @@ -337,6 +385,19 @@ TEST_CASE("bench: blocking-c1 exercises blocking control with exact tokens") { CHECK(artifact.at("blocking_wait_calls").get() > 0); CHECK(artifact.at("output_token_ids") == nlohmann::json(blocking.output_token_ids)); + + REQUIRE(!output_wait_trace.empty()); + bool observed_blocking_wait = false; + bool every_blocking_wait_has_preceding_empty_nowait = true; + for (size_t i = 0; i < output_wait_trace.size(); ++i) { + if (output_wait_trace[i] != OutputWaitTestEvent::kBlockingWait) continue; + observed_blocking_wait = true; + if (i == 0 || output_wait_trace[i - 1] != OutputWaitTestEvent::kNowaitEmpty) { + every_blocking_wait_has_preceding_empty_nowait = false; + } + } + CHECK(observed_blocking_wait); + CHECK(every_blocking_wait_has_preceding_empty_nowait); } TEST_CASE("bench: synthetic engine completes all requests with sane metrics") { From 966a13976513f203eec83be842dff3088f3a5ae2 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 11 Aug 2026 00:51:24 +0100 Subject: [PATCH 09/11] test(bench): share production nowait evaluation (#293) Make the output-wait test observer inspect the result only after the same unconditional AsyncLLM call used by production. Replace the source-text guard with a deterministic runtime probe whose bypass mutation fails normally, while preserving the measured AArch64 Release binary byte-for-byte. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/engine-matrix.md | 2 +- examples/bench/bench_core.h | 52 +++++++++++++++++++++++------------ tests/examples/test_bench.cpp | 28 ++++++++++--------- 3 files changed, 51 insertions(+), 31 deletions(-) diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index fbb334901..0d3d93f67 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -205,7 +205,7 @@ claims it. | `SERVE-HTTP-TRANSPORT` | Serving-socket transport parity: mirror vLLM's uvicorn/asyncio default `TCP_NODELAY` on every accepted SSE socket so per-token stream frames are not held by Nagle against the peer's delayed ACK. Implemented + CPU-tested; the non-binding localhost A/B sizing is COMPLETE and NEUTRAL within noise on c1/c2 ITL/TPOT/throughput (loopback ACKs are instant, so Nagle never coalesces ~100 ms-cadence token frames) — no gate-axis credit expected; the mirror stays for real-network parity. Future keep-alive / read-write-timeout / listening-socket option parity noted, not done | T0 | vLLM serves via uvicorn over asyncio `vllm/entrypoints/launcher.py:71,76`, `vllm/entrypoints/openai/api_server.py:591,630`; asyncio disables Nagle per accepted TCP stream socket `asyncio/base_events.py:192-197` (`_set_nodelay`) called from `asyncio/selector_events.py:950`; cpp-httplib default-off `third_party/httplib/httplib.h:142`, applied on accept only when set `third_party/httplib/httplib.h:12083` | `src/vllm/entrypoints/openai/api_server.cpp:69` (`set_tcp_nodelay(true)` in the ApiServer setup) | behavioral accepted-socket `getsockopt(TCP_NODELAY)` case `tests/vllm/entrypoints/openai/test_api_server.cpp:1076` (helper `:380`); RED accepted `TCP_NODELAY` 0 → GREEN 1, full `test_openai_api_server` **22/22 cases / 242 assertions**; non-binding sizing root `~/work/vllm.cpp-tcpnodelay-sizing/ff915e8…` (raw-set SHA `f5b52900…2128`) neutral within noise; closure [ledger](parity-ledger.md#L451) | [serve-tcp-nodelay.md](specs/serve-tcp-nodelay.md) | `DONE` | `ff915e8` | | `SERVE-C-ABI` | Stable LocalAI-style C FFI (**19** exported `VLLM_API` symbols at `VLLM_ABI_VERSION 10`; blocking and nonblocking request handles. Count corrected 2026-07-24 from a stale `17`, which predated ABI v4/v5 adding `tool_parser`/`reasoning_parser` and the chat entry points; `include/vllm.h` is the source of truth and README:231 already said 19). **ABI v9 2026-07-28 (`CLAIM-CAPI-ENGINE-CONFIG-V9`): the ABI carried strictly LESS engine config than `EngineParams` does** - `max_num_batched_tokens`, the scheduler `scheduling_policy` (`fcfs` / `priority` / `lpm`), and `kv_transfer_config` (the external KV connector / LMCache JSON) were reachable from the bundled server's flags and from NO embedder. All three added, inert at their defaults (zero-filled v8 growth == byte-identical pre-v9 engine); the connector NAME is validated against `KVConnectorFactory` at load, mirroring the server's startup check. `tokenizer_config_path` stopped being a declared-since-v1 no-op and now selects the chat template's source file. Malformed `speculative_config`/`kv_transfer_config` documents now report `VLLM_ERR_INVALID_ARGUMENT` (the contract vllm.h documented since v6) instead of `VLLM_ERR_MODEL_LOAD`, via a catch scoped to the parse block so a real `FromModelDir` failure still reports MODEL_LOAD. Driver: the LocalAI vllm-cpp backend could not expose LMCache or the prefill budget in a model config) | T0 | Original project ABI; pinned vLLM has no C ABI | `include/vllm.h:143,181,207`; `src/capi/vllm_c.cpp:229,264,327,391` | `tests/capi/test_capi.cpp:320,428,505,574,606,640`; `tests/capi/test_dlopen.cpp:77,86`; `tests/capi/c_header_compile.c:1` | [c-api-library.md](specs/c-api-library.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-C-ABI-SPIKE` | | `SERVE-CPP-API` | Rich `LLM` and `AsyncLLM` C++ API | T1 | `vllm/entrypoints/llm.py:66,422`; `vllm/v1/engine/async_llm.py:70` | - | - | `planned: specs/cpp-api.md` | `INVENTORIED` | - | -| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); output-wait selector, parser and default `examples/bench/bench_core.h:69-101,125-132`; direct validation and wait implementation `examples/bench/bench_core.h:495-501,636-676`; CLI parser and direct validation `examples/bench/main.cpp:94-102,128-135` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; text-report, direct-rejection, nowait-before-blocking, JSON and exact-token assertions `tests/examples/test_bench.cpp:21-119`; positive Pi C1 with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | +| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); output-wait selector, parser and default `examples/bench/bench_core.h:69-109,134-141`; direct validation and wait implementation `examples/bench/bench_core.h:507-510,630-694`; CLI parser and direct validation `examples/bench/main.cpp:94-102,128-135` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; text-report, direct-rejection, runtime nowait-before-blocking, JSON and exact-token assertions `tests/examples/test_bench.cpp:21-122`; positive Pi C1 with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | | `ENG-RELEASE-BINARIES` | Downloadable host-ABI-specific `vllm-server` bundles: adaptive CPU and fat CUDA primary artifacts, optional per-SM diagnostics, and literal-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | Required W1-W11/W13 implementation is complete in draft PR #196: ten-SM gencode/AOT, adaptive CPU tiers, extracted-archive validation and supply chain, least-privilege immutable handoff, eight primary bundles, byte-derived indexes, attestation, and exact-file publication; W12 remains optional/non-primary | Local manifest/archive/workflow/mutation gates, full adaptive x86 tier execution, clean CPU archive, and Vulkan 35/35 backend + 11/11 cross-device archive run are green; hosted ten-SM completion, full eight-tuple dry run, matching-hardware gates, and tagged publication remain pending, so no published-binary claim exists | [release-binary-matrix.md](specs/release-binary-matrix.md) | `ACTIVE` | `CLAIM-ENG-RELEASE-BINARIES-W1-W13` | diff --git a/examples/bench/bench_core.h b/examples/bench/bench_core.h index 7c5ff264f..a7917d0dd 100644 --- a/examples/bench/bench_core.h +++ b/examples/bench/bench_core.h @@ -82,6 +82,14 @@ enum class OutputWaitTestEvent { kNowaitEmpty, kBlockingWait, }; + +// Runtime probe controls used only by test_bench. The seeded collector value is +// input to the real AsyncLLM method; the observer below runs only after that +// method has returned, so neither control can replace the production call. +struct OutputWaitTestProbe { + bool seed_first_collector = false; + bool stop_after_first_nowait = false; +}; #endif inline const char* OutputWaitModeName(OutputWaitMode mode) { @@ -131,6 +139,7 @@ struct BenchConfig { OutputWaitMode output_wait = OutputWaitMode::kPoll; #ifdef VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE std::vector* output_wait_test_trace = nullptr; + OutputWaitTestProbe output_wait_test_probe; #endif uint64_t seed = 0; // prompt-generation RNG seed. double temperature = 0; // <= 0 => greedy (deterministic). @@ -756,27 +765,37 @@ inline BenchResult RunBench(const BenchConfig& cfg) { return std::next(it); }; + admit(); #ifdef VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE -#define VLLM_BENCH_TRACE_NOWAIT(expression) \ - ([&]() { \ - std::optional traced_ready = (expression); \ - if (cfg.output_wait_test_trace != nullptr) { \ - cfg.output_wait_test_trace->push_back( \ - traced_ready.has_value() ? OutputWaitTestEvent::kNowaitReady \ - : OutputWaitTestEvent::kNowaitEmpty); \ - } \ - return traced_ready; \ - }()) -#else -#define VLLM_BENCH_TRACE_NOWAIT(expression) (expression) + if (cfg.output_wait_test_probe.seed_first_collector) { + if (active.size() != 1 || active.begin()->second.collector == nullptr) { + throw std::logic_error( + "nowait runtime probe requires exactly one active collector"); + } + RequestOutput seeded; + seeded.request_id = active.begin()->first; + active.begin()->second.collector->put(std::move(seeded)); + } #endif - - admit(); while (done < cfg.num_prompts) { bool observed_output = false; for (auto it = active.begin(); it != active.end();) { - std::optional ready = VLLM_BENCH_TRACE_NOWAIT( - engine.get_output_nowait(it->second)); + // Keep this evaluation unconditional: test instrumentation observes the + // already-evaluated value below and cannot replace the production call. + std::optional ready = + engine.get_output_nowait(it->second); +#ifdef VLLM_BENCH_TEST_OUTPUT_WAIT_TRACE + if (cfg.output_wait_test_trace != nullptr) { + cfg.output_wait_test_trace->push_back( + ready.has_value() ? OutputWaitTestEvent::kNowaitReady + : OutputWaitTestEvent::kNowaitEmpty); + } + if (cfg.output_wait_test_probe.stop_after_first_nowait) { + throw std::logic_error( + ready.has_value() ? "nowait runtime probe observed seeded output" + : "nowait runtime probe missed seeded output"); + } +#endif if (!ready.has_value()) { ++it; continue; @@ -811,7 +830,6 @@ inline BenchResult RunBench(const BenchConfig& cfg) { } admit(); // keep C in flight as requests finish. } -#undef VLLM_BENCH_TRACE_NOWAIT const double dur_s = now_s(); diff --git a/tests/examples/test_bench.cpp b/tests/examples/test_bench.cpp index bf67302a0..e4230e149 100644 --- a/tests/examples/test_bench.cpp +++ b/tests/examples/test_bench.cpp @@ -345,19 +345,6 @@ TEST_CASE("bench: blocking-c1 selector rejects concurrent requests by name") { } TEST_CASE("bench: blocking-c1 exercises blocking control with exact tokens") { - const std::filesystem::path bench_core = - std::filesystem::path(__FILE__).parent_path().parent_path().parent_path() / - "examples/bench/bench_core.h"; - std::ifstream bench_core_stream(bench_core); - REQUIRE(bench_core_stream.good()); - const std::string bench_core_source{ - std::istreambuf_iterator(bench_core_stream), - std::istreambuf_iterator()}; - REQUIRE(bench_core_source.find( - "VLLM_BENCH_TRACE_NOWAIT(\n" - " engine.get_output_nowait(it->second))") != - std::string::npos); - BenchConfig cfg; cfg.num_prompts = 3; cfg.input_len = 8; @@ -400,6 +387,21 @@ TEST_CASE("bench: blocking-c1 exercises blocking control with exact tokens") { CHECK(every_blocking_wait_has_preceding_empty_nowait); } +TEST_CASE("bench: runtime executes the production nowait call") { + BenchConfig cfg; + cfg.num_prompts = 1; + cfg.input_len = 8; + cfg.output_len = 4; + cfg.concurrency = 1; + cfg.output_wait = OutputWaitMode::kBlockingC1; + cfg.output_wait_test_probe.seed_first_collector = true; + cfg.output_wait_test_probe.stop_after_first_nowait = true; + + CHECK_THROWS_WITH_AS( + RunBench(cfg), "nowait runtime probe observed seeded output", + std::logic_error); +} + TEST_CASE("bench: synthetic engine completes all requests with sane metrics") { BenchConfig cfg; cfg.num_prompts = 8; From 51fa50046702178076420b1fc0c5c93f2da52a88 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 11 Aug 2026 03:22:21 +0100 Subject: [PATCH 10/11] docs(cpu): checkpoint A76 C1 on current main (#293) Record the reviewed post-rebase operator gate without replacing the older binary's provenance. The current-main same-binary Pi control confirms four-core polling contention while T3 remains neutral, but the unstable poll arm still forbids a binding floor change. Correct the low-severity evidence precision finding by reporting the raw perf-stat T3 CPU-utilization medians to two decimal places. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/NOW.md | 2 +- .agents/coordination.md | 10 +- .agents/engine-matrix.md | 2 +- .agents/kernel-matrix.md | 2 +- .agents/specs/cpu-a76-bf16-gemm.md | 15 +- .agents/specs/cpu-bench-thread-budget.md | 38 ++--- docs/BENCHMARKS.md | 2 +- .../rpi5-a76-output-wait-c1-main-20260811.md | 138 ++++++++++++++++++ 8 files changed, 180 insertions(+), 29 deletions(-) create mode 100644 docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md diff --git a/.agents/NOW.md b/.agents/NOW.md index 9daf0a21c..7561b4b56 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -22,7 +22,7 @@ Work: 7 external PRs landed (6 merged + #227 repaired). | Kimi-Linear-48B | 122/128 held; e2e NOT ESTABLISHED | tiktoken-only ckpt: no warm server | | 35B mid-band | **canonical 0.918-0.972x** c1-c32 (@`348c265d`, first c16/c32) | Decode-only window, ONE tool | | Qwen3.5-4B sm_120 | tput **1.0283x** `PENDING`; TTFT/TPOT/E2E 1.085/1.017/1.029x | Profile wave | -| RPi5 A76 | **#293 C1 POSITIVE**: T4 decode 2.193x; T3 neutral | Review; C2 event; F16 | +| RPi5 A76 | **#293 C1 GATED**: T4 decode 2.149x; T3 neutral | C2 event; F16 profile | | SERVE-ASYNC-MIRROR | **#323 FIXED** (mitigation): graph declines while the mirror is live; 7/7 async gates | Graph read ids at REPLAY | | CPU levers (`QUANT-GGUF-CIQ-GEMM`) | Profile DONE: decode **47% threadpool sync**, prefill **~39% paged attn** | Parakeet encoder; attn dtype hoist | | `SERVE-METRICS` async (#277) | **`/metrics` was DEAD on the shipped server**: AsyncLLM folded nothing. Now live, ctest 366/366 | Config-gated families | diff --git a/.agents/coordination.md b/.agents/coordination.md index 02f3007b1..c9ca3e470 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -1657,8 +1657,14 @@ blocking/poll is 2.193x decode and 0.458x E2E latency; T3 is neutral; all 12 token files match. The pathological T4 poll spread is above 3%, so no binding floor moves. Raw evidence and exact hashes are indexed in [`rpi5-a76-output-wait-c1-20260810.md`](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md). -This implementation is complete and awaits fresh mutation review. C2 was not -started. +After fresh review and rebase onto `upstream/main` `4ba051406`, immutable head +`960647bf5` passed the operator's native and QEMU gates and a second +hash-verified Pi series. T4 blocking/poll is 2.149x decode and 0.465x E2E +latency; T3 remains neutral; exact tokens hold. T4 poll spread is 13.33-15.96%, +so no binding floor moves. The +[current-main evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md) +preserves that binary separately from the original run. C1 is complete; C2 was +not started. | Claim | Row IDs | Agent | Worktree / remote dir | Branch | Owned scope | State | Last update | |---|---|---|---|---|---|---|---| diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index 0d3d93f67..835aec690 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -205,7 +205,7 @@ claims it. | `SERVE-HTTP-TRANSPORT` | Serving-socket transport parity: mirror vLLM's uvicorn/asyncio default `TCP_NODELAY` on every accepted SSE socket so per-token stream frames are not held by Nagle against the peer's delayed ACK. Implemented + CPU-tested; the non-binding localhost A/B sizing is COMPLETE and NEUTRAL within noise on c1/c2 ITL/TPOT/throughput (loopback ACKs are instant, so Nagle never coalesces ~100 ms-cadence token frames) — no gate-axis credit expected; the mirror stays for real-network parity. Future keep-alive / read-write-timeout / listening-socket option parity noted, not done | T0 | vLLM serves via uvicorn over asyncio `vllm/entrypoints/launcher.py:71,76`, `vllm/entrypoints/openai/api_server.py:591,630`; asyncio disables Nagle per accepted TCP stream socket `asyncio/base_events.py:192-197` (`_set_nodelay`) called from `asyncio/selector_events.py:950`; cpp-httplib default-off `third_party/httplib/httplib.h:142`, applied on accept only when set `third_party/httplib/httplib.h:12083` | `src/vllm/entrypoints/openai/api_server.cpp:69` (`set_tcp_nodelay(true)` in the ApiServer setup) | behavioral accepted-socket `getsockopt(TCP_NODELAY)` case `tests/vllm/entrypoints/openai/test_api_server.cpp:1076` (helper `:380`); RED accepted `TCP_NODELAY` 0 → GREEN 1, full `test_openai_api_server` **22/22 cases / 242 assertions**; non-binding sizing root `~/work/vllm.cpp-tcpnodelay-sizing/ff915e8…` (raw-set SHA `f5b52900…2128`) neutral within noise; closure [ledger](parity-ledger.md#L451) | [serve-tcp-nodelay.md](specs/serve-tcp-nodelay.md) | `DONE` | `ff915e8` | | `SERVE-C-ABI` | Stable LocalAI-style C FFI (**19** exported `VLLM_API` symbols at `VLLM_ABI_VERSION 10`; blocking and nonblocking request handles. Count corrected 2026-07-24 from a stale `17`, which predated ABI v4/v5 adding `tool_parser`/`reasoning_parser` and the chat entry points; `include/vllm.h` is the source of truth and README:231 already said 19). **ABI v9 2026-07-28 (`CLAIM-CAPI-ENGINE-CONFIG-V9`): the ABI carried strictly LESS engine config than `EngineParams` does** - `max_num_batched_tokens`, the scheduler `scheduling_policy` (`fcfs` / `priority` / `lpm`), and `kv_transfer_config` (the external KV connector / LMCache JSON) were reachable from the bundled server's flags and from NO embedder. All three added, inert at their defaults (zero-filled v8 growth == byte-identical pre-v9 engine); the connector NAME is validated against `KVConnectorFactory` at load, mirroring the server's startup check. `tokenizer_config_path` stopped being a declared-since-v1 no-op and now selects the chat template's source file. Malformed `speculative_config`/`kv_transfer_config` documents now report `VLLM_ERR_INVALID_ARGUMENT` (the contract vllm.h documented since v6) instead of `VLLM_ERR_MODEL_LOAD`, via a catch scoped to the parse block so a real `FromModelDir` failure still reports MODEL_LOAD. Driver: the LocalAI vllm-cpp backend could not expose LMCache or the prefill budget in a model config) | T0 | Original project ABI; pinned vLLM has no C ABI | `include/vllm.h:143,181,207`; `src/capi/vllm_c.cpp:229,264,327,391` | `tests/capi/test_capi.cpp:320,428,505,574,606,640`; `tests/capi/test_dlopen.cpp:77,86`; `tests/capi/c_header_compile.c:1` | [c-api-library.md](specs/c-api-library.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-C-ABI-SPIKE` | | `SERVE-CPP-API` | Rich `LLM` and `AsyncLLM` C++ API | T1 | `vllm/entrypoints/llm.py:66,422`; `vllm/v1/engine/async_llm.py:70` | - | - | `planned: specs/cpp-api.md` | `INVENTORIED` | - | -| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); output-wait selector, parser and default `examples/bench/bench_core.h:69-109,134-141`; direct validation and wait implementation `examples/bench/bench_core.h:507-510,630-694`; CLI parser and direct validation `examples/bench/main.cpp:94-102,128-135` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; text-report, direct-rejection, runtime nowait-before-blocking, JSON and exact-token assertions `tests/examples/test_bench.cpp:21-122`; positive Pi C1 with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | +| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); output-wait selector, parser and default `examples/bench/bench_core.h:69-109,134-141`; direct validation and wait implementation `examples/bench/bench_core.h:507-510,630-694`; CLI parser and direct validation `examples/bench/main.cpp:94-102,128-135` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; text-report, direct-rejection, runtime nowait-before-blocking, JSON and exact-token assertions `tests/examples/test_bench.cpp:21-122`; reviewed current-main Pi C1 with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | | `ENG-RELEASE-BINARIES` | Downloadable host-ABI-specific `vllm-server` bundles: adaptive CPU and fat CUDA primary artifacts, optional per-SM diagnostics, and literal-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | Required W1-W11/W13 implementation is complete in draft PR #196: ten-SM gencode/AOT, adaptive CPU tiers, extracted-archive validation and supply chain, least-privilege immutable handoff, eight primary bundles, byte-derived indexes, attestation, and exact-file publication; W12 remains optional/non-primary | Local manifest/archive/workflow/mutation gates, full adaptive x86 tier execution, clean CPU archive, and Vulkan 35/35 backend + 11/11 cross-device archive run are green; hosted ten-SM completion, full eight-tuple dry run, matching-hardware gates, and tagged publication remain pending, so no published-binary claim exists | [release-binary-matrix.md](specs/release-binary-matrix.md) | `ACTIVE` | `CLAIM-ENG-RELEASE-BINARIES-W1-W13` | diff --git a/.agents/kernel-matrix.md b/.agents/kernel-matrix.md index d368df34c..7baafccac 100644 --- a/.agents/kernel-matrix.md +++ b/.agents/kernel-matrix.md @@ -112,7 +112,7 @@ host/sched. Detail: state `KERNEL-FA2-GQA-SWAP-FLIP`. | ID | Item | Upstream | Our code | Tests/evidence | Spike/spec | State | Owner | |---|---|---|---|---|---|---|---| -| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); no issue-specific implementation retained | Accepted same-file Pi floor remains 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS. W0 reranked before implementation: summed cycles ranked barrier 42.21%, reached **F16** `Bt16Neon` 33.09%, Q8 12.05%. Same-source #293 C1 now attributes the four-core collapse to benchmark polling: blocking/poll is 2.193x decode and 0.458x E2E latency at T4, neutral at T3, exact tokens; the binding floor stays unchanged because T4 poll spread exceeds 3%. W1-W4 await fresh C1 review, C2's general event wait and a fresh profile ([W0 evidence](../docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md), [C1 evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md), [issue #284](https://github.com/mudler/vllm.cpp/issues/284)) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `ACTIVE` | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | +| `KERNEL-GEMM-CPU-ELEM-A76` | Raspberry Pi 5 Cortex-A76 specialization of the byte-exact BF16 `[N,K]` elementwise GEMM; C++/NEON first, with handwritten assembly blocked until the C++ path beats llama.cpp end to end | llama.cpp b9892 `ee445f93d`: `ggml/src/ggml-cpu/vec.cpp:139-260` BF16 dot, `ggml/src/ggml-cpu/ggml-cpu.c:1158-1245` 16x16 chunk worker, `ggml/src/ggml-cpu/simd-mappings.h` AArch64 SIMD mappings | Baseline [Bt16Neon/BtM4Neon](../src/vt/cpu/cpu_matmul_elem.cpp#L152); no issue-specific implementation retained | Accepted same-file Pi floor remains 0.461x prefill, 0.653x decode/E2E, 0.758x peak RSS. W0 reranked before implementation: summed cycles ranked barrier 42.21%, reached **F16** `Bt16Neon` 33.09%, Q8 12.05%. Reviewed current-main #293 C1 attributes the four-core collapse to benchmark polling: blocking/poll is 2.149x decode and 0.465x E2E latency at T4, neutral at T3, exact tokens; the binding floor stays unchanged because T4 poll spread exceeds 3%. W1-W4 await C2's general event wait and a fresh profile ([W0 evidence](../docs/bench-evidence/rpi5-a76-bf16-w0-20260810.md), [current-main C1 evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md), [issue #284](https://github.com/mudler/vllm.cpp/issues/284)) | [A76 BF16 GEMM spec](specs/cpu-a76-bf16-gemm.md) | `ACTIVE` | `CLAIM-KERNEL-GEMM-CPU-ELEM-A76` | | `KERNEL-CPU-A76-Q8-DOT` | Cortex-A76 Q8_0 x Q8_0 SDOT, exact-order C++ and scheduled AAPCS64 variants for the Pi decode/prefill keep-quant GEMM | llama.cpp `ggml/src/ggml-cpu/quants.c:400` portable dot and `arch/arm/quants.c:1076-1160` Arm DotProd @ project pin `237ad9b96`; Qwen semantics remain vLLM `555967922` | [ACLE SDOT + HWCAP/MIDR selector](../src/vt/cpu/cpu_quant_dot_sdot.cpp#L85), [AAPCS64 two-block leaf](../src/vt/cpu/cpu_quant_dot_a76.S#L10), [Q8 dispatch](../src/vt/cpu/cpu_quant_dot.cpp#L675), [same-binary variants](../examples/cpu_kernel_bench/main.cpp#L858) | [direct exact variant/edge tests](../tests/vt/test_ops_quant_dot.cpp#L593); QEMU 20/20, 150258 assertions; physical Pi exact checksums/tokens; assembly vs compiler SDOT +3.66% M1/T1, +5.08% M128/T1, +3.69% M128/T4, but −2.43% M1/T4. Same-file llama.cpp floor measured/NOT MET: vllm.cpp 0.461x prefill / 0.653x decode+E2E, 0.758x peak RSS (better), exact-prompt output equal; [assembly evidence](../docs/bench-evidence/rpi5-a76-q8-dot-20260806.md#L40), [competitor evidence](../docs/bench-evidence/rpi5-a76-llamacpp-20260806.md) | [A76 Q8 dot spec](specs/cpu-a76-q8-dot.md) | `GATING` | - | | `KERNEL-ACCEL-PROVIDER-SELECT` | **WHICH implementation of an op runs, when more than one exists on a device** — the selection layer above every kernel family in this matrix. Distinct from `KERNEL-CUDA-DISPATCH-AOT`, which is about which ARCH a CUDA kernel is compiled/selected for; this is about which PROVIDER (ours, a vendor library, MLX, llama.cpp) serves the op at all | no single upstream file: this is the shape vLLM's runtime chain uses everywhere — flashinfer tactic registries, cuBLASLt/CUTLASS per-call heuristics, and torch's backend selection — rather than compile-time pinning | `vt::OpProvider` [op_provider.h](../include/vt/op_provider.h) + [op_provider.cpp](../src/vt/op_provider.cpp); the flat `[OpId][DeviceType]` `void*` table it replaces is gone from [ops.cpp](../src/vt/ops.cpp) with the ~70 op wrappers untouched. Providers registered today: `vt-native` (every backend kernel in the tree, priority 0, unconditional — behaviour preserved exactly), `mlx` (priority 100, Metal `kMatmul`/`kMatmulBT`, build-gated `VLLM_CPP_MLX`, [metal_mlx_provider.mm](../src/vt/metal/metal_mlx_provider.mm)), and — NEW 2026-07-23 (`CLAIM-BACKEND-SEAM-S5-1`, work row `S5`) — **`vt-cpu-ref` (priority −1000, the portable reference tier)**: the CPU kernel installed LAZILY as a negative-priority fallback on a UNIFIED-MEMORY device's first `GetOp` miss, mirroring `custom_op.py:138 forward_native`, so a partial backend runs an op it lacks natively instead of throwing. Native always wins (priority); gated on `Backend::UnifiedMemory()` (a discrete GPU never gets it — a CPU kernel on true device memory is corruption); observable via `GetReferenceTierHits()` + a one-time loud stderr line | [test_op_provider.cpp](../tests/vt/test_op_provider.cpp) 11 cases / 47 assertions — deterministic selection under REVERSED registration order, name tie-break, duplicate rejection, capability predicate, caps re-resolution, decline-and-fall-back, stats, runtime disable; [test_metal_backend.cpp](../tests/vt/test_metal_backend.cpp) 9 cases / 108 assertions on the M4 with MLX ON, including MLX-vs-MSL-vs-CPU NMSE per op at real shapes and an end-to-end DECLINE; **[test_reference_tier.cpp](../tests/vt/test_reference_tier.cpp) (S5): discrete-device refusal + unified-device zero-native-kernel fallback correctness + native-wins + observability, hardware-free via a fake backend on `kXPU`.** Linux CPU 156/156; dgx regression set ALL UNCHANGED — anchor `tests/vt/test_op_provider.cpp:64` | [Metal/MLX reuse study §6](specs/metal-mlx-reuse-study.md); [accelerator-seam-audit §10](specs/accelerator-seam-audit.md); [drop-in kernel ABI](specs/dropin-kernel-abi.md) (the complementary ARGUMENT half) | `ACTIVE` — mechanism landed and gated with THREE provider kinds (`vt-native`, `mlx`, `vt-cpu-ref`); the CUDA/CPU/Vulkan vendor provider rows it was designed for are not yet populated (so the row is deliberately left open) | `CLAIM-BACKEND-ACCEL-PROVIDER-1` | | `KERNEL-CUDA-DISPATCH-AOT` | CUDA runtime, streams, graphs, per-arch dispatch, and generated/AOT artifact selection | platform selection `vllm/platforms/cuda.py:205-493`; gencode handling `CMakeLists.txt:201-220`; JIT packages `cmake/external_projects/triton_kernels.cmake:1-28` | [cuda_backend.cu:20](../src/vt/cuda/cuda_backend.cu#L20), [CMakeLists.txt:37](../CMakeLists.txt#L37), [TritonAOT.cmake:57](../cmake/TritonAOT.cmake#L57) | [CUDA backend tests](../tests/vt/test_cuda_backend.cpp#L31); the Triton GDN AOT tree is now vendored per-arch — `sm_121a` (RUNTIME-VERIFIED) plus `sm_80/86/89/90a/100a` (DERIVED+BUILD-VERIFIED, `CLAIM-TRITON-AOT-PER-ARCH`, 2026-07-28), each a real per-target cubin (`cuobjdump sm=80/86/89/90/100`); the CUDA fast-path arch fan-out (non-GDN) remains as evidenced in `backend-matrix.md` | [inventory](specs/kernel-family-inventory.md); [per-arch Triton AOT](specs/triton-aot-per-arch.md) | `ANCHOR-BACKFILL` | - | diff --git a/.agents/specs/cpu-a76-bf16-gemm.md b/.agents/specs/cpu-a76-bf16-gemm.md index bb871247b..45f1ee434 100644 --- a/.agents/specs/cpu-a76-bf16-gemm.md +++ b/.agents/specs/cpu-a76-bf16-gemm.md @@ -252,9 +252,12 @@ Full commands, samples, artifact hashes and raw evidence hashes are in the Disposition: the row remains `ACTIVE`. The [`SERVE-CLI-BENCH` C1 control in issue #293](https://github.com/mudler/vllm.cpp/issues/293) -is now positive at the implementer checkpoint: T4 blocking restores 2.193x -decode while T3 is neutral ([C1 evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)). -W1-W4 remain blocked on fresh C1 review, C2's general multi-request event wait, -and the subsequent four-core profile that ranks the reached elementwise -kernel. The BF16 fixture, C++ schedule candidates and assembly gate were not -attempted. Assembly remains forbidden. +is positive, reviewed and operator-gated on current main: T4 blocking is +2.149x decode and 0.465x E2E latency while T3 is neutral +([current-main evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md); +[original evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)). +The accepted llama.cpp floor stays unchanged because T4 polling remains above +the binding spread limit. W1-W4 remain blocked on C2's general multi-request +event wait and the subsequent four-core profile that ranks the reached +elementwise kernel. The BF16 fixture, C++ schedule candidates and assembly +gate were not attempted. Assembly remains forbidden. diff --git a/.agents/specs/cpu-bench-thread-budget.md b/.agents/specs/cpu-bench-thread-budget.md index 0ae6880d9..8a3120bea 100644 --- a/.agents/specs/cpu-bench-thread-budget.md +++ b/.agents/specs/cpu-bench-thread-budget.md @@ -146,10 +146,10 @@ upstream event semantics and extend the local original-harness tests: | W | Deliverable | State | |---|---|---| | C0 | Correct #284 evidence wording and register #293 | complete (`0a07c53db`) | -| C1 | Same-current-source `poll` vs `blocking-c1` control + RED mutation | implemented at `bdbfffcbf`; fresh review pending | +| C1 | Same-current-source `poll` vs `blocking-c1` control + RED mutation | reviewed and current-main operator-gated at `960647bf5` | | C2 | If C1 positive, general event/epoch wait + deterministic multi-request tests | unblocked by positive C1; excluded from this implementation | -| C3 | QEMU gate and interleaved Pi T4/T3 causal profile | complete ([evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | -| C4 | Retain event wait or reject it; test reserved-core budget only if C1 is negative/insufficient | blocked by C2 + fresh review | +| C3 | QEMU gate and interleaved Pi T4/T3 causal profile | current-main confirmed ([operator evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md); [original evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md)) | +| C4 | Retain event wait or reject it; test reserved-core budget only if C1 is negative/insufficient | blocked by C2 | | C5 | Full model versus llama.cpp; return to the measured F16/kernel ranking | blocked by C4 | ## Risks/decisions @@ -170,20 +170,24 @@ upstream event semantics and extend the local original-harness tests: ## Outcome -C1 implementer result: **positive, fresh review pending**. On the exact -current-source AArch64 binary, three interleaved idle/unthrottled processes per -arm show T4 `blocking-c1` / `poll` medians of 2.193x decode, 2.004x prefill and -0.458x E2E latency, while the T3 control is neutral at 1.003x decode, 1.001x -prefill and 0.997x E2E latency. All 12 token files exactly match the accepted -SHA-256; RSS is neutral. T4 blocking retires 76.5% fewer instructions and uses -54.2% less task-clock. This isolates scan-and-yield frontend contention as the -cause of the four-core collapse. +C1 result: **positive, reviewed and operator-gated on current main**. The +rebased `960647bf5` AArch64 binary reproduces the discriminator over three +interleaved idle/unthrottled processes per arm: T4 `blocking-c1` / `poll` +medians are 2.149x decode, 1.973x prefill and 0.465x E2E latency, while the T3 +control is neutral at 1.003x decode, 0.999x prefill and 0.998x E2E latency. +All 12 token files exactly match the accepted SHA-256; RSS is neutral. T4 +blocking retires 63.0% fewer instructions and uses 53.4% less task-clock. This +confirms scan-and-yield frontend contention as the cause of the four-core +collapse after the upstream rebase. No replacement performance floor is accepted here: the pathological T4 poll -arm still has 5.9-10.7% spread and fails the sub-3% binding gate, although the -effect is positive in every pair and the blocking/T3 arms are stable below -0.52%. `blocking-c1` remains an explicit concurrency-one control and `poll` -remains the default. C2's lost-wakeup-safe general event wait requires a fresh -implementer after review; it was deliberately not started. Full hashes, native +arm has 13.33-15.96% spread and fails the sub-3% binding gate, although the +effect is positive in every pair and the blocking/T3 arms are stable. The +older source/binary evidence remains preserved rather than being overwritten. +`blocking-c1` remains an explicit concurrency-one control and `poll` remains +the default. C2's lost-wakeup-safe general event wait requires a fresh +implementer; it was deliberately not started. Full latest-main hashes, native and QEMU counts, host state, raw evidence and medians are in the -[C1 evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md). +[operator checkpoint](../../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md); +the [original C1 evidence](../../docs/bench-evidence/rpi5-a76-output-wait-c1-20260810.md) +retains its historical provenance. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index a296b0544..64f163159 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -248,7 +248,7 @@ host mirror is freed once the device Marlin resident is built. | 64-token Qwen model gate | Byte-identical across x86, portable, SDOT and assembly arms; asm vs SDOT median TTFT -1.55%, TPOT neutral, E2E -0.13%; vs portable TTFT -33.40%, E2E -2.67%. Cortex-A76+DotProd selects assembly by default | | Same-file llama.cpp floor (pp17/tg64) | **NOT MET on speed**: prefill 12.81 vs 27.77 tok/s (0.461x), decode 2.55 vs 3.91 (0.653x), E2E 26,018.39 vs 16,998.49 ms ([competitor evidence](bench-evidence/rpi5-a76-llamacpp-20260806.md)) | | Peak RSS | **2.841 vs 3.747 GiB, 24.2% less**; 3 clean unthrottled reps; same-text 64-token greedy output byte-identical after trailing-newline normalization | -| #293 C1 positive | T4 decode 2.193x; T3 neutral; exact tokens; poll spread keeps floor unchanged. | +| #293 C1 | [T4 decode 2.149x; floor unchanged](bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md) | Same GGUF file both arms, `dgx.casa` GB10 aarch64 (20 cores), idle, 3 reps, llama.cpp `237ad9b96` built fresh on the same host. diff --git a/docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md b/docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md new file mode 100644 index 000000000..ded48e910 --- /dev/null +++ b/docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md @@ -0,0 +1,138 @@ +# Raspberry Pi 5 output-wait C1 current-main confirmation, 2026-08-11 + +This is the post-rebase operator checkpoint for +[`SERVE-CLI-BENCH` issue #293](https://github.com/mudler/vllm.cpp/issues/293) +and the prerequisite discriminator for +[`KERNEL-GEMM-CPU-ELEM-A76` issue #284](https://github.com/mudler/vllm.cpp/issues/284). +It repeats the concurrency-one `poll` / `blocking-c1` control after rebasing +the reviewed implementation onto current `upstream/main`. The +[original C1 evidence](rpi5-a76-output-wait-c1-20260810.md) remains the record +for its older source and binary; this checkpoint does not overwrite it. + +## Decision + +The current-main result confirms that the C1 polling hypothesis is positive: + +- at four compute threads, `blocking-c1` / `poll` is 2.149x median per-stream + decode, 2.151x output throughput, 1.973x prefill and 0.465x E2E latency; +- at three compute threads, the control remains neutral: 1.003x decode, + 0.999x prefill and 0.998x E2E latency; +- all 12 processes exit zero and emit the accepted token-file SHA-256 + `0ec98eabb23e4148d540fcf79a2fe61678fb90fe462cdf28134af7a42fe6a826`; + every blocking arm reports exactly 64 blocking calls and every poll arm + reports zero; and +- T4 blocking uses 45.1% fewer user cycles, 63.0% fewer user instructions and + 53.4% less task-clock than polling. The T3 wall-time-neutral control uses + 5.1%, 5.7% and 25.0% less respectively. + +The accepted llama.cpp floor does not move. T4 polling has 13.33-15.96% +spread across the six performance axes, well above the sub-3% binding gate. +Every interleaved T4 pair improves in the expected direction and the T4 +blocking/T3 arms are stable, so the causal conclusion survives the rebase; the +unstable denominator still prevents promoting a replacement performance +floor. `blocking-c1` remains a diagnostic restricted to concurrency one, +`poll` remains the default, and C2's lost-wakeup-safe general event wait is +still the next implementation. + +## Same-binary result + +Values are medians of three separate, order-alternated processes. Higher is +better for throughput; lower is better for latency and RSS. Final rows are +`blocking-c1 / poll`. + +| Threads | Mode | Prefill tok/s | Decode tok/s | Output tok/s | TTFT ms | TPOT ms | E2E ms | Peak RSS KiB | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 4 | poll | 5.0046 | 1.2028 | 1.1404 | 3,396.86 | 831.37 | 56,121.80 | 2,978,704 | +| 4 | blocking-c1 | 9.8763 | 2.5848 | 2.4528 | 1,721.29 | 386.88 | 26,092.51 | 2,978,464 | +| 4 | blocking / poll | **1.973x** | **2.149x** | **2.151x** | **0.507x** | **0.465x** | **0.465x** | 0.9999x | +| 3 | poll | 8.1281 | 2.5294 | 2.3705 | 2,091.51 | 395.34 | 26,998.68 | 2,977,952 | +| 3 | blocking-c1 | 8.1176 | 2.5364 | 2.3762 | 2,094.21 | 394.26 | 26,933.80 | 2,978,192 | +| 3 | blocking / poll | 0.999x | 1.003x | 1.002x | 1.001x | 0.997x | 0.998x | 1.0001x | + +The T4 polling maximum-minus-minimum spread about the median is 15.35% +prefill, 14.01% decode, 13.64% output, 15.96% TTFT, 13.91% TPOT and 13.33% +E2E. The T4 blocking arm is 0.023-2.04% across those axes; every T3 arm is at +or below 0.343%. Peak-RSS spread is below 0.026% in every arm. + +## PMU result + +`perf stat` wrapped each complete process. These are median totals from the +same legs; user cycles and instructions were requested explicitly. + +| Threads | Mode | Cycles | Instructions | Task-clock ns | IPC from median totals | +|---:|---|---:|---:|---:|---:| +| 4 | poll | 455,038,839,265 | 708,685,367,178 | 224,842,433,752 | 1.557 | +| 4 | blocking-c1 | 249,764,922,813 | 262,507,520,194 | 104,800,580,477 | 1.051 | +| 4 | blocking / poll | 0.549x | 0.370x | 0.466x | - | +| 3 | poll | 204,253,742,508 | 258,064,105,230 | 108,440,568,102 | 1.264 | +| 3 | blocking-c1 | 193,827,363,627 | 243,472,258,058 | 81,301,312,384 | 1.256 | +| 3 | blocking / poll | 0.949x | 0.943x | 0.750x | - | + +The T4 poll arm consumes about 3.96 CPUs for more than twice the blocking +wall time. At T3, polling consumes about 3.94 CPUs while blocking consumes +about 2.96 CPUs for the same wall time. This independently reproduces the +frontend-contention discriminator; it does not establish a general wait +implementation or a cross-engine floor. + +## Source, build and gates + +- Source: `960647bf5e0467c0121aa9d69b85c0e0b192b10c`, based on fetched + `upstream/main` `4ba051406`; the C1 implementation had a fresh post-rebase + review before this operator checkpoint. +- AArch64 Release `vllm-bench`: SHA-256 + `d03b612eaa59590f4c5df25b3bc128e298aa89bdee6eb5bf5633d3ff18e8f058`. + Exported `vllm-cpu-kernel-bench`: SHA-256 + `80ab743acdee0d2bc749a8c871672f8852e45ce56c271f6ae9a5673171b774a9`. +- Native scoped CTest: 5/5 (`test_engine_core_proc`, + `test_output_processor`, `test_async_llm`, `test_bench` and + `test_cpu_threadpool`). +- QEMU/buildx AArch64 gate: quant 23/23 cases and 150,350 assertions; + `test_bench` 8/8 cases and 63 assertions; A76 smoke selected `a76-asm` and + returned checksum `0xbb60dafd79d08ed6`. QEMU timings are void. +- Model: `Qwen3.5-2B-UD-Q8_K_XL.gguf`, SHA-256 + `a53988df91157d78acaf3c95e22db179d13f6236061bdb86576494dc99b1bc3b`. +- Workload: one request, 17 actual input tokens, 64 output tokens, + concurrency one, seed zero, temperature zero, cores 0-3, + `VT_CPU_Q8_DOT=auto`, with `VLLM_CPP_CPU_THREADS` set to four or three. + +## Host and raw evidence + +The Pi snapshots identify four Cortex-A76 r4p1 cores, Linux +`6.18.34+rpt-rpi-2712`, the `ondemand` governor, 2.1-2.4 GHz captured +frequencies, 49.9-62.6 C temperatures and `throttled=0x0` throughout. The +highest pre-series process was the SSH session at 0.4% CPU; individual pre-leg +snapshots peaked at 0.2%, and no contender reached 1% CPU. + +The immutable raw set remains at +`~/vllm-cpp-assembly/evidence/a76-c1-960647bf5-20260811-operator/` on +`rich@rpi5fan.lan`. Its `SHA256SUMS` has 98 entries and SHA-256 +`2d4f26091f02589f63e818639f503539b7281e06eda4eb68c5883a4c6dc6a149`. +All 98 entries were verified again after copying the set locally; the manifest +contains absolute Pi paths, so the local verification remapped only that fixed +directory prefix. Each leg contains JSON, exact token IDs, text output, +stderr, `perf stat`, process `getrusage`, and before/after host snapshots. + +The remote harness and standard-library RSS wrapper were independently +rehashed as +`02b75b2a69bc6dc5150071a252ee53862aa8cf37d80b79294ab38a4f85b51445` +and +`ad310102e1d88fa6395c2c41a9594f62d195891bdb1b79e322cd31611db805f5`. +Every raw host snapshot repeats the expected benchmark/model hashes. Every +JSON field, text report, exit status, embedded token stream and token file was +checked across all 12 legs. + +The core command remains: + +```sh +perf stat -x, -e cycles:u,instructions:u,task-clock,context-switches,cpu-migrations -- \ + taskset -c 0-3 env VLLM_CPP_CPU_THREADS=<3|4> VT_CPU_Q8_DOT=auto \ + vllm-bench --model Qwen3.5-2B-UD-Q8_K_XL.gguf \ + --num-prompts 1 --input-len 16 --output-len 64 --concurrency 1 \ + --seed 0 --temperature 0 --output-wait \ + --output-token-ids .tokens.json --output-json .json +``` + +For continuity only, the latest T4 blocking medians remain about 0.353x the +prior llama.cpp b9892 prefill denominator, 0.662x decode and 1.536x E2E +latency. The two harnesses still use different prompt content, so those values +are diagnostic and do not replace the accepted same-file floor. From cc4b195e6b92e631a2f76b37706b4d5b638911a9 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 11 Aug 2026 12:57:13 +0100 Subject: [PATCH 11/11] test(bench): bind rebased C1 evidence (#293) Require both benchmark admission arms to expose populated, exact per-request prompt IDs, so dropping the production assignment fails normally. Add semantic mutation evidence for the A76 inventory and STATUS ratchet changes, re-anchor the SERVE-CLI-BENCH row, and project the confirmed C1 polling cause without changing the llama.cpp floor. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: Codex:gpt-5.6-sol [Codex] --- .agents/engine-matrix.md | 2 +- docs/STATUS.md | 2 +- scripts/check-public-doc-tables.py | 8 ++++---- tests/examples/test_bench.cpp | 19 ++++++++++++++++++ tests/scripts/test_agent_record.py | 20 +++++++++++++++++++ tests/scripts/test_check_public_doc_tables.py | 11 +++++++++- 6 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index 835aec690..40c657c5d 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -205,7 +205,7 @@ claims it. | `SERVE-HTTP-TRANSPORT` | Serving-socket transport parity: mirror vLLM's uvicorn/asyncio default `TCP_NODELAY` on every accepted SSE socket so per-token stream frames are not held by Nagle against the peer's delayed ACK. Implemented + CPU-tested; the non-binding localhost A/B sizing is COMPLETE and NEUTRAL within noise on c1/c2 ITL/TPOT/throughput (loopback ACKs are instant, so Nagle never coalesces ~100 ms-cadence token frames) — no gate-axis credit expected; the mirror stays for real-network parity. Future keep-alive / read-write-timeout / listening-socket option parity noted, not done | T0 | vLLM serves via uvicorn over asyncio `vllm/entrypoints/launcher.py:71,76`, `vllm/entrypoints/openai/api_server.py:591,630`; asyncio disables Nagle per accepted TCP stream socket `asyncio/base_events.py:192-197` (`_set_nodelay`) called from `asyncio/selector_events.py:950`; cpp-httplib default-off `third_party/httplib/httplib.h:142`, applied on accept only when set `third_party/httplib/httplib.h:12083` | `src/vllm/entrypoints/openai/api_server.cpp:69` (`set_tcp_nodelay(true)` in the ApiServer setup) | behavioral accepted-socket `getsockopt(TCP_NODELAY)` case `tests/vllm/entrypoints/openai/test_api_server.cpp:1076` (helper `:380`); RED accepted `TCP_NODELAY` 0 → GREEN 1, full `test_openai_api_server` **22/22 cases / 242 assertions**; non-binding sizing root `~/work/vllm.cpp-tcpnodelay-sizing/ff915e8…` (raw-set SHA `f5b52900…2128`) neutral within noise; closure [ledger](parity-ledger.md#L451) | [serve-tcp-nodelay.md](specs/serve-tcp-nodelay.md) | `DONE` | `ff915e8` | | `SERVE-C-ABI` | Stable LocalAI-style C FFI (**19** exported `VLLM_API` symbols at `VLLM_ABI_VERSION 10`; blocking and nonblocking request handles. Count corrected 2026-07-24 from a stale `17`, which predated ABI v4/v5 adding `tool_parser`/`reasoning_parser` and the chat entry points; `include/vllm.h` is the source of truth and README:231 already said 19). **ABI v9 2026-07-28 (`CLAIM-CAPI-ENGINE-CONFIG-V9`): the ABI carried strictly LESS engine config than `EngineParams` does** - `max_num_batched_tokens`, the scheduler `scheduling_policy` (`fcfs` / `priority` / `lpm`), and `kv_transfer_config` (the external KV connector / LMCache JSON) were reachable from the bundled server's flags and from NO embedder. All three added, inert at their defaults (zero-filled v8 growth == byte-identical pre-v9 engine); the connector NAME is validated against `KVConnectorFactory` at load, mirroring the server's startup check. `tokenizer_config_path` stopped being a declared-since-v1 no-op and now selects the chat template's source file. Malformed `speculative_config`/`kv_transfer_config` documents now report `VLLM_ERR_INVALID_ARGUMENT` (the contract vllm.h documented since v6) instead of `VLLM_ERR_MODEL_LOAD`, via a catch scoped to the parse block so a real `FromModelDir` failure still reports MODEL_LOAD. Driver: the LocalAI vllm-cpp backend could not expose LMCache or the prefill budget in a model config) | T0 | Original project ABI; pinned vLLM has no C ABI | `include/vllm.h:143,181,207`; `src/capi/vllm_c.cpp:229,264,327,391` | `tests/capi/test_capi.cpp:320,428,505,574,606,640`; `tests/capi/test_dlopen.cpp:77,86`; `tests/capi/c_header_compile.c:1` | [c-api-library.md](specs/c-api-library.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-C-ABI-SPIKE` | | `SERVE-CPP-API` | Rich `LLM` and `AsyncLLM` C++ API | T1 | `vllm/entrypoints/llm.py:66,422`; `vllm/v1/engine/async_llm.py:70` | - | - | `planned: specs/cpp-api.md` | `INVENTORIED` | - | -| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); output-wait selector, parser and default `examples/bench/bench_core.h:69-109,134-141`; direct validation and wait implementation `examples/bench/bench_core.h:507-510,630-694`; CLI parser and direct validation `examples/bench/main.cpp:94-102,128-135` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; text-report, direct-rejection, runtime nowait-before-blocking, JSON and exact-token assertions `tests/examples/test_bench.cpp:21-122`; reviewed current-main Pi C1 with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | +| `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29`; production queue `vllm/v1/engine/core.py:200-231,622-669`; pinned comparison pretokenizes before timing and synchronously admits a complete concurrency wave before each explicit step `tools/bench/vllm_closed_loop_metrics.py:57-102,137-167`; per-request event wait `vllm/v1/engine/output_processor.py:45-96` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; production `AsyncLLM` benchmark frontend + auditable scheduler depth; #206 default pre-encodes every prompt before `t0` and admits token IDs, exact `VT_BENCH_PRETOKENIZE=0` retains timed-string admission, and `BenchResult` reports the resolved path (`examples/bench/bench_core.h:190-233,616-665`). Atomic queue/wave admission prepares every request and collector before one all-or-zero ordered core publish (`include/vllm/v1/engine/core_proc.h:90-126`; `src/vllm/v1/engine/async_llm.cpp:125-237`); output-wait selector, parser and default `examples/bench/bench_core.h:69-109,134-141`; direct validation and wait implementation `examples/bench/bench_core.h:594-597,768-831`; CLI parser and direct validation `examples/bench/main.cpp:94-102,128-135` | server-help/production-frontend/metric assertions plus #206 parser/callback identity, preparation-before-clock, special-token/InputProcessor parity, report-mode capture and synthetic exact-ID A/B (`tests/examples/test_bench.cpp:123-186,347-376`). Atomic queue 3/3·13 (`tests/vllm/v1/test_engine_core_proc.cpp:233-299`); ordered/rollback/shutdown wave gates and complete async suite 12/12·433 (`tests/vllm/v1/test_async_llm.cpp:288-484`), all CPU-GREEN. Real `a33993a7` A/B **FAILED** token identity (98/128 requests, 15,507/16,384 positions); no timing credit. Fresh mutation review, operator gate and real counterbalanced retry remain pending; text-report, direct-rejection, runtime nowait-before-blocking, JSON and exact-token assertions `tests/examples/test_bench.cpp:302-402`; reviewed current-main Pi C1 with unchanged binding floor ([evidence](../docs/bench-evidence/rpi5-a76-output-wait-c1-main-20260811.md)) | [CLI/serve/benchmark spike](specs/cli-serve-bench.md); [#206 campaign contract](specs/sm120-qwen35-pareto-2026-08-09.md); [CPU wait/thread-budget control](specs/cpu-bench-thread-budget.md) | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | | `ENG-RELEASE-BINARIES` | Downloadable host-ABI-specific `vllm-server` bundles: adaptive CPU and fat CUDA primary artifacts, optional per-SM diagnostics, and literal-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | Required W1-W11/W13 implementation is complete in draft PR #196: ten-SM gencode/AOT, adaptive CPU tiers, extracted-archive validation and supply chain, least-privilege immutable handoff, eight primary bundles, byte-derived indexes, attestation, and exact-file publication; W12 remains optional/non-primary | Local manifest/archive/workflow/mutation gates, full adaptive x86 tier execution, clean CPU archive, and Vulkan 35/35 backend + 11/11 cross-device archive run are green; hosted ten-SM completion, full eight-tuple dry run, matching-hardware gates, and tagged publication remain pending, so no published-binary claim exists | [release-binary-matrix.md](specs/release-binary-matrix.md) | `ACTIVE` | `CLAIM-ENG-RELEASE-BINARIES-W1-W13` | diff --git a/docs/STATUS.md b/docs/STATUS.md index 38bf71a72..2b2e8f323 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -110,7 +110,7 @@ token-for-token correctness against the pinned oracle. | GGUF loading (F32/F16/BF16/Q4_0/Q8_0/Q2_K/Q3_K/Q4_K/Q5_K/Q6_K/IQ2_XXS/IQ3_XXS/IQ2_S/MXFP4/NVFP4) | Supported; compute-in-quant (keep-quant) on CPU AND now CUDA for the six K-block encodings PLUS Q2_K/IQ2_XXS/IQ3_XXS (DeepSeek-V4 W8, 2026-07-29 - the FIRST CUDA keep-quant GGUF k-quant GEMM `KERNEL-QUANT-CIQ-GEMM-CUDA`, MMVQ-style dequant-in-kernel, GB10-gated 92401/92401 vs the CPU oracle, so a CUDA runner keeps blocks compressed and dots them on the GPU instead of the ARM cores); **NVFP4 now COMPUTES IN FP4 on CUDA for the dense-MLP and full-attention projections (2026-07-29, `CLAIM-GGUF-NVFP4-COMPUTE`), no longer materialize-only** | Weights in six block encodings stay compressed from file to matmul on CPU (no BF16 expansion). NVFP4 (ggml type 40) DEQUANTIZES, including the per-tensor (per-expert) `.scale` sidecar the container keeps outside the blocks; gated BIT-EXACT against the compressed-tensors NVFP4 path on real Qwen3.6-27B bytes from both containers. **It no longer expands to bf16 on CUDA:** an NVFP4 matmul/expert weight is REPACKED at load into the same (`weight_packed [N,K/2]`, `weight_scale [N,K/16]`) operand pair the compressed-tensors path produces - a pure byte permutation, gated BYTE-IDENTICAL against that container - and the existing `vt::MatmulNvfp4*` kernels run on it, so no new kernel exists and no numerics are re-derived. Covers the dense MLP + full-attention q/k/v/o and the MoE shared/routed experts; the GDN `in_proj_*` family and `ssm_out` still expand (the V-head reorder rewrites their layout) and a CPU build still expands everything - the documented `part` subset. **MEASURED GB10 (2026-07-29), same-binary A/B, 2 reps/arm:** peak RSS **50.8 -> 25.7 GiB**, load-and-generate **1:58 -> 0:41**; the 256 moved projections cost 35 840 MiB expanded vs 10 080 fp4-resident (3.56x). **The safetensors-sibling divergence CLOSES:** the fp4 arm is token-IDENTICAL over 24 greedy tokens where the same binary's bf16 arm diverges at index 4. REPORTED, not gated: the containers are not the same model - the GGUF NVFP4-quantizes 192 GDN `in_proj` tensors the safetensors keeps BF16 (mean rel. weight error ~0.18) and their activation global scales differ, so identity is not guaranteed and a cross-container throughput arm is invalid. SACRED gates unmoved: `test_qwen27_paged_engine` 235/235, `test_qwen36_paged_engine` 315/315. **The MoE (35B) stacked-expert arm is HARDWARE-GATED too (2026-07-29)**: the real 35B A3B NVFP4 GGUF loads and generates through the fp4 path, its 120 routed-expert stacks x 256 experts repack to the modelopt safetensors' operands with ZERO differing bytes over 840 sampled (tensor, expert) slabs, and all 840 per-expert `.scale[e]` are bit-identical to that expert's `weight_scale_2` - the scale INDEXING, mutation-proved against a `scales[0]`-for-all and an expert-0-slab-for-all mutant. Same-binary A/B: peak RSS 68.5 -> 22.7 GiB (3.01x), load-and-generate 1:51.9 -> 0:28.8, tokens IDENTICAL (the 35B routed experts run the W4A16 grouped GEMM in both arms). Recorded OPEN: this case's 24-token greedy stream is NOT run-to-run stable (1 of 3 `use_a16` and 1 of 4 reference runs differed), so the binding results are the weight-level byte identity and the residency audit, not token-exactness; `test_qwen36_paged_engine` is token-exact at ITS engine params, so the instability belongs to this case's configuration and attributing it is owed work. It also FIXED a latent defect the MoE arm made reachable: the two fp4 fused MoE blocks issued the router GEMM assuming the safetensors `[K,N]` gate layout and threw `matmul: inner dims mismatch` on the GGUF's `[N,K]`; `MoeRouterLogits` now branches on `nk` (inert for safetensors, SACRED unmoved). **Q2_K (id 10) + IQ2_XXS (id 16) DEQUANTIZE (2026-07-29, `CLAIM-DSV4-GGUF-LOADER`):** the ~2-bit types the single-Spark `DeepSeek-V4-Flash-GGUF UD-IQ2_XXS`/`UD-Q2_K_XL` vehicles use, ported 1:1 from llama.cpp `ggml-quants.c` (`iq2xxs_grid` codebook + signs; Q2_K nibble sub-scale/min), unit-gated on hand-derived bytes (`test_gguf_dequant` 15/15). Dequant-only (no vec_dot -> expand-bf16). A V4-GGUF model cannot RUN yet: the name map (tensor-manifest-blocked) + the V4 forward (W3-W8) remain. **Multi-shard split GGUF READING (2026-08-03, `CLAIM-GGUF-SPLIT-SHARDS`):** `GgufFile::Open` now transparently stitches llama.cpp `gguf-split` shards (`...-00001-of-00003.gguf`) — every shard mmap'd, tensor tables merged, KV metadata from shard `00001`, sibling mappings kept alive by the primary so keep-quant mmap-borrows stay valid across shards (`OwnsSpan` is shard-aware); `VT_GGUF_NO_SPLIT=1` opts out; unit-gated (`test_gguf` split-merge / no-split / count-mismatch cases, 33/33 local). This unblocks the real 3-shard `unsloth/DeepSeek-V4-Flash-0731 UD-IQ2_M` (~91 GiB), whose layout is the NATIVE `deepseek4` arch — per-block `ffn_gate_tid2eid` hash tables (hash layers 0/1/2) + `hc_*` MHC + DSA compressor/indexer are all PRESENT (name-map 1328/1328), `vocab_size` derives from `token_embd` — NOT a standard llama.cpp conversion, so no loader-layout change is owed. It now loads THROUGH 1324/1328 tensors; the sole remaining gap is 4 routed-expert slabs quantized with IQ2_S (id 22, ×2) + MXFP4 (id 39, ×2) — encodings we have GGUF block traits for but no keep-quant vec_dot, so they hit the expand→dequant path which lacks them. Expanding those 4 expert tensors to bf16 would add ~17 GiB (~106 GiB total → OOM-reboot risk), so the memory-safe fix is an IQ2_S+MXFP4 keep-quant kernel (CPU dequant dispatch + `iq2s_grid` + CUDA `DotSuperblock`), spec'd as the next brick **IQ2_S (id 22) + MXFP4 (id 39) DEQUANTIZE + KEEP-QUANT on CPU (2026-08-03, `CLAIM-DSV4-UDIQ2M-QUANT`, off-GPU):** the extra per-tensor "dynamic" encodings the `unsloth/DeepSeek-V4-Flash-GGUF UD-IQ2_M` checkpoint mixes into its last routed-expert slabs (IQ2_S `ffn_gate/up` dotting Q8_K, MXFP4 `ffn_down` dotting Q8_0) — ported 1:1 from llama.cpp `ggml-quants.c` @ 237ad9b96 (`iq2s_grid` 1024-entry codebook + DIRECT sign bytes; MXFP4 `kvalues_mxfp4` + `e8m0_to_fp32_half` micro-scaling, distinct from the compressed-tensors `E8M0ToF32` NVFP4 path). CPU dequant + keep-quant `vec_dot`, unit-gated on hand-derived golden bytes (`test_gguf_dequant` 17/17), an INDEPENDENT f64 dequant-then-dot + GEMM NMSE (`test_ops_quant_dot` 19/19), and keep-quant routing (`test_gguf_keep_quant` 37/37) — all CPU-green, so UD-IQ2_M's four previously-`unsupported ggml type 22/39` slabs now load COMPRESSED (no ~17 GiB bf16 expansion that OOM-reboots the box). CUDA: the IQ2_S device `DotSuperblock` is wired into the Q8_K grouped-MoE GEMM and now **CUDA-BUILT + LINKED on GB10 (sm_121a, CUDA 13.0, `-Werror`, 2026-08-03 integration)** — it compiles clean and the merged binary links; MXFP4's device dot (`DotMXFP4`) is written but NOT wired (Q8_0-activation needs a separate 32-block GEMM) so it is marked `[[maybe_unused]]` to keep the ready math without tripping nvcc #177-D, and on GPU MXFP4 CPU-fallbacks like Q4_0/Q8_0. The V4-GGUF forward + a real UD-IQ2_M GPU load/coherence run are owed | | AWQ / GPTQ quantization | W0 spike + W1 CPU INT4 dequant primitive; not yet loadable end to end | INT4 unpack+dequant-to-bf16 for BOTH community formats, mirroring vLLM 1:1 (AWQ reverse-order `awq_triton.py`; GPTQ `qdq_4.cuh` with zero_offset v1/v2 + act-order g_idx). Unit-gated RED-first (hand-computed known bytes + double-precision roundtrip). NOT wired to a loader, no GPU Marlin compute, no model run yet: config recognizer (W2), Marlin GPU GEMM riding the vendored NVFP4 Marlin (W4), CPU e2e (W3), GPTQ 8/2/3-bit (W5) and MoE (W6) are named next bricks. See [.agents/specs/awq-gptq-quant.md](../.agents/specs/awq-gptq-quant.md) | | MXFP4 (compressed-tensors `mxfp4-pack-quantized`) | Compute PROVEN (#38); GQA-swap ON (#49); decode-graph+gate_up FUSION default-ON. `VT_MARLIN_DENSE` DEFAULT-ON (`KERNEL-MARLIN-DENSE-EXEC`): dense marlin 48-CTA byte-faithful (32B 0.000, 263/263), binding beats #51 every axis (c1 1.020, c8 0.969, mem 2.63x). **`QUANT-CT-MXFP4-FINAL-STACK` TERMINAL — both last levers exhausted: num_splits cap `VT_FA2_NSPLITS_CAP` gated-OFF (c1-only, self-corrects@c8; 32B strict char-identical); glue folds via `vt::FusedChain`; `FLASH-AUDIT` #68: c8 flash gap +12.5us/call is occupancy/L2-bound; `-use_fast_math` TRIED, REGRESSES flash (168.8→189.8), rejected. c1 1.020x PASS, c2-c8 0.962-0.969.** state.md | Shared with DeepSeek-V4-Flash + Kimi-K3 MXFP4 paths. CPU E8M0 dequant 5/5·1142. GPU W4A4 + MoE-expert e2e later | -| CPU backend vs llama.cpp | Pi/A76 `GATING`; #284 W0 | Floor unchanged: 0.461x/0.653x. T4/T3 proves only thread-budget sensitivity; #293 tests polling. | +| CPU backend vs llama.cpp | Pi/A76 `GATING`; #284 W0 | Floor 0.461x/0.653x unchanged. #293 C1: T4 polling caused collapse; T3 neutral. C2 event next. | | Paged KV cache + prefix caching | Supported | Block-paged full attention, hybrid full-attention + GDN state groups, automatic prefix caching (APC) on by default for dense models (cache-ON gated end to end: token-identical output, cache hits, faster TTFT) | | fp8 KV cache (`cache_dtype=fp8`) | In progress (W1 CPU brick), not yet usable end-to-end | HIGH-priority memory/throughput lever (halves the KV footprint). W0 spike + W1 CPU brick landed (`KV-FP8` ACTIVE): fp8-e4m3 K/V STORE (`Quantize(hp/scale)`) + the paged-attention READ dequant (`Dequant(fp8)*scale`) + the `cache_dtype` config parse, all CPU-gated RED-first (`test_ops_fp8_kv_cache` 8/8·511; a wrong store direction fails 3/480). Storage is 1-byte fp8 (`DType::kI8`) + a `Fp8KVCacheDataType` interpretation enum, per-tensor k/v scales (mirroring vLLM `BaseKVCacheMethod`). The CUDA store + fp8 paged-attention read (the GPU memory-halving path, DGX-blocked), the runner/spec integration (half-sized KV blocks + checkpoint-scale threading + `--kv-cache-dtype`/`--calculate-kv-scales`), fp8_e5m2 and per-head scales are named W2-W5 in [.agents/specs/fp8-kv-cache.md](../.agents/specs/fp8-kv-cache.md). No model can run with an fp8 KV cache yet | | Prefix-cache matching unit (`--prefix-match-unit`) | Partial (resolver landed, config/scheduler wiring pending) | 0.26-new knob setting the finest token boundary a prefix-cache hit can land on (the `hash_block_size`). W1: `resolve_kv_cache_block_sizes` ported 1:1 (hybrid `hash_block_size = prefix_match_unit if set else gcd(group block sizes)`; single-group inert; back-off on no-consumer / mamba-non-align; throws on non-divisible), CPU unit-gated RED-first (default gcd vs `=16`). Pending: the config/CLI/ABI field (W2), scheduler threading of a finer-than-block matching unit (W3, needs the KV-block-pool align path), and the benchmark (W4). Default path byte-identical (dense single-group models ignore it). | diff --git a/scripts/check-public-doc-tables.py b/scripts/check-public-doc-tables.py index d6a3f4c4c..e9ac8b3b9 100755 --- a/scripts/check-public-doc-tables.py +++ b/scripts/check-public-doc-tables.py @@ -609,10 +609,10 @@ def features_errors(text: str) -> list[str]: # carries #347's shorter Muse Glimmer wording on top of main's. Re-measured # rather than carried from either PR, and rather than arithmetic on the two: # zero slack is the only honest re-pin. Strictly DOWN from 243188. - # 243124 since 2026-08-11 (#293 C0, measured): the Pi/A76 status was - # collapsed while narrowing the W0 causal claim. Details stay in the - # indexed W0 evidence and committed control spec. Strictly DOWN. - "chars": 243124, + # 243122 since 2026-08-11 (#293 C1, measured): the Pi/A76 row now records + # polling as the confirmed T4 cause and names C2 without repeating the + # indexed evidence. Strictly DOWN from 243124. + "chars": 243122, "h2_sections": 11, "long_paragraphs": 82, "oversized_cells": 44, diff --git a/tests/examples/test_bench.cpp b/tests/examples/test_bench.cpp index e4230e149..869fc833e 100644 --- a/tests/examples/test_bench.cpp +++ b/tests/examples/test_bench.cpp @@ -512,6 +512,17 @@ TEST_CASE("bench: pretokenized default preserves prompt and output token IDs") { cfg.seed = 31; cfg.temperature = 0.0; + const vllm::tok::Tokenizer tokenizer = + vllm::bench::detail::BuildSyntheticTokenizer(); + std::vector> expected_prompt_token_ids; + expected_prompt_token_ids.reserve(static_cast(cfg.num_prompts)); + for (int i = 0; i < cfg.num_prompts; ++i) { + const std::string prompt = vllm::bench::detail::BuildPrompt( + tokenizer, cfg.input_len, cfg.seed + static_cast(i)); + expected_prompt_token_ids.push_back( + tokenizer.EncodeWithSpecialTokens(prompt)); + } + BenchResult pretokenized; { ScopedEnv env("VT_BENCH_PRETOKENIZE", std::nullopt); @@ -530,6 +541,14 @@ TEST_CASE("bench: pretokenized default preserves prompt and output token IDs") { static_cast(cfg.num_prompts)); REQUIRE(timed_string.prompt_token_ids.size() == static_cast(cfg.num_prompts)); + REQUIRE(expected_prompt_token_ids.size() == + static_cast(cfg.num_prompts)); + for (size_t i = 0; i < expected_prompt_token_ids.size(); ++i) { + CAPTURE(i); + REQUIRE_FALSE(expected_prompt_token_ids[i].empty()); + CHECK(pretokenized.prompt_token_ids[i] == expected_prompt_token_ids[i]); + CHECK(timed_string.prompt_token_ids[i] == expected_prompt_token_ids[i]); + } CHECK(pretokenized.prompt_token_ids == timed_string.prompt_token_ids); CHECK(pretokenized.output_token_ids == timed_string.output_token_ids); CHECK(pretokenized.total_input == timed_string.total_input); diff --git a/tests/scripts/test_agent_record.py b/tests/scripts/test_agent_record.py index 0fbd01a5e..ff25bc99a 100644 --- a/tests/scripts/test_agent_record.py +++ b/tests/scripts/test_agent_record.py @@ -271,6 +271,26 @@ def test_model_row_ratchet_is_load_bearing(self) -> None: agent_record.check_matrices(errors) require(errors, r"\d+ MODEL rows; expected \d+") + def test_kernel_row_ratchet_matches_the_current_inventory(self) -> None: + """The #284 A76 family is a real 52nd kernel row, not a relaxed pin.""" + path, expected = agent_record.MATRICES["KERNEL"] + self.assertEqual(expected, 52) + + errors: list[str] = [] + rows = agent_record.parse_claim_rows(path, errors) + self.assertEqual(errors, []) + self.assertEqual(len(rows), expected) + + for mutated_expected in (expected - 1, expected + 1): + with self.subTest(expected=mutated_expected): + mutated_errors: list[str] = [] + with mock.patch.dict( + agent_record.MATRICES, + {"KERNEL": (path, mutated_expected)}, + ): + agent_record.check_matrices(mutated_errors) + require(mutated_errors, r"\d+ KERNEL rows; expected \d+") + def test_engine_summary_rejects_stale_area_rollup(self) -> None: source = agent_record.ENGINE_MATRIX.read_text(encoding="utf-8") current = next( diff --git a/tests/scripts/test_check_public_doc_tables.py b/tests/scripts/test_check_public_doc_tables.py index ea4e5f3a8..f862c0364 100644 --- a/tests/scripts/test_check_public_doc_tables.py +++ b/tests/scripts/test_check_public_doc_tables.py @@ -532,6 +532,15 @@ def test_the_rebased_character_ratchet_is_byte_tight(self) -> None: text = doc_tables.STATUS.read_text(encoding="utf-8") self.assertEqual(doc_tables.STATUS_RATCHET["chars"], len(text)) + def test_the_a76_c1_repin_records_the_cause_and_next_step(self) -> None: + text = doc_tables.STATUS.read_text(encoding="utf-8") + self.assertIn( + "#293 C1: T4 polling caused collapse; T3 neutral. C2 event next.", + text, + ) + self.assertNotIn("#293 tests polling", text) + self.assertEqual(doc_tables.STATUS_RATCHET["chars"], len(text)) + def test_a_retired_claim_cannot_come_back_for_free(self) -> None: """A claim the page RETIRED must cost something to reinstate. @@ -606,7 +615,7 @@ def test_the_status_ratchet_only_ever_moves_down(self) -> None: 243584 over 48 commits), so the ceiling never needs to rise. """ ceiling = { - "chars": 243188, + "chars": 243122, "h2_sections": 11, "long_paragraphs": 82, "oversized_cells": 44,