perf(dspark): unblock upstream ncu; the DRAM attribution is CORROBORATED (#442)

benchmarks/marlin_moe_standalone.cpp

@@ -0,0 +1,213 @@
+// OUR arm of the #442 standalone Marlin harness.
+//
+// Mirrors scripts/marlin-moe-standalone.py exactly: same 35B-A3B decode shapes
+// (hidden 2048, moe_intermediate 512, E=256, top_k=8, moe_block_size 8), same
+// gate_up GEMM, same expert-pool control over the occupied block count. Prints
+// us/call and us/block so our plateau can be laid against upstream's 5.2-5.5.
+//
+// Not a test: no assertions, no goldens. It measures the kernel only.
+//
+// RUN IT UNDER THE BOX LOCK: `flock $HOME/gpu.lock ...`, NOT /tmp/gpu.lock,
+// which coordinates with nothing. `nvidia-smi` showing no compute apps does
+// not mean the GPU is unreserved, so check `fuser -v $HOME/gpu.lock` first.
+// Absolute timings taken unlocked are upper bounds; only interleaved RATIOS
+// survive contention.
+//
+// NOT WIRED INTO ANY BUILD TARGET (#442). Nothing compiles this file, so it
+// carries no -Werror and no CI, and it will rot against
+// vt::MoeGroupedGemmNvfp4Marlin's signature. The recorded measurements were
+// taken from an out-of-tree build. Wiring it into examples/CMakeLists.txt the
+// way benchmarks/vulkan_gemm_ab.cpp is wired is owed.
+//
+// Its routing RNG is a DIFFERENT stream from the python arm's, so the two
+// arms occupy different block counts at the same --experts pool. Comparisons
+// between them are NORMALISED by blocks, not matched on them; neither arm can
+// yet take an externally supplied routing tensor.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <chrono>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "vt/backend.h"
+#include "vt/cuda/marlin_repack.h"
+#include "vt/dtype.h"
+#include "vt/ops.h"
+
+namespace {
+
+using vt::Backend;
+using vt::Device;
+using vt::DeviceType;
+using vt::DType;
+using vt::Queue;
+using vt::Tensor;
+
+Device Gpu() { return Device{DeviceType::kCUDA, 0}; }
+
+Tensor MakeT(void* data, DType dt, Device dev, const std::vector<int64_t>& shape) {
+  Tensor t;
+  t.data = data;
+  t.dtype = dt;
+  t.device = dev;
+  t.rank = static_cast<int>(shape.size());
+  int64_t stride = 1;
+  for (int i = t.rank - 1; i >= 0; --i) {
+    t.shape[i] = shape[static_cast<size_t>(i)];
+    t.stride[i] = stride;
+    stride *= shape[static_cast<size_t>(i)];
+  }
+  return t;
+}
+
+class Dev {
+ public:
+  Dev(Backend& b, Queue& q, DType dt, const std::vector<int64_t>& shape,
+      const void* host = nullptr)
+      : b_(b) {
+    int64_t numel = 1;
+    for (auto s : shape) numel *= s;
+    bytes_ = static_cast<size_t>(numel) * vt::SizeOf(dt);
+    p_ = b_.Alloc(bytes_ == 0 ? 1 : bytes_);
+    if (host != nullptr) b_.Copy(q, p_, host, bytes_);
+    t_ = MakeT(p_, dt, Gpu(), shape);
+  }
+  ~Dev() { b_.Free(p_); }
+  Dev(const Dev&) = delete;
+  Dev& operator=(const Dev&) = delete;
+  Tensor& tensor() { return t_; }
+  void* ptr() { return p_; }
+
+ private:
+  Backend& b_;
+  void* p_ = nullptr;
+  size_t bytes_ = 0;
+  Tensor t_;
+};
+
+int IntArg(int argc, char** argv, const char* name, int fallback) {
+  for (int i = 1; i + 1 < argc; ++i)
+    if (std::strcmp(argv[i], name) == 0) return std::atoi(argv[i + 1]);
+  return fallback;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  const int pool_arg = IntArg(argc, argv, "--experts", 0);
+  const int iters = IntArg(argc, argv, "--iters", 80);
+  const int warmup = IntArg(argc, argv, "--warmup", 20);
+  const int M = IntArg(argc, argv, "--m", 9);
+  const int zero_ws = IntArg(argc, argv, "--zero-ws", 1);
+
+  const int E = 256, K = 2048, N = 512, top_k = 8;
+  const int pool = pool_arg > 0 ? pool_arg : E;
+  const int size_n = 2 * N;  // gate_up
+  const int size_k = K;
+
+  Backend& b = vt::GetBackend(DeviceType::kCUDA);
+  Queue q{Gpu(), nullptr};
+  void* stream = nullptr;
+  const int dev_id = 0;
+
+  // Weights: random packed nibbles, repacked per expert into Marlin layout.
+  // Marlin's runtime is data independent, so random bits time like real ones.
+  std::mt19937 rng(1234);
+  const size_t raw_bytes = static_cast<size_t>(size_n) * size_k / 2;
+  std::vector<uint8_t> raw(raw_bytes);
+  for (auto& x : raw) x = static_cast<uint8_t>(rng() & 0xFF);
+  const size_t scale_bytes = static_cast<size_t>(size_n) * size_k / 16;
+  std::vector<uint8_t> raw_s(scale_bytes);
+  for (auto& x : raw_s) x = 0x38;  // fp8-e4m3 ~ 0.5, safely positive
+
+  Dev staging(b, q, DType::kI8, {size_n, size_k / 2}, raw.data());
+  Dev staging_s(b, q, DType::kI8, {size_n, size_k / 16}, raw_s.data());
+
+  Dev wq(b, q, DType::kI32, {E, size_k / 16, size_n * 2});
+  Dev sc(b, q, DType::kI8, {E, size_k / 16, size_n});
+  const float sf = 1.0f;
+  std::vector<float> gs(static_cast<size_t>(E),
+                        vt::cuda::MarlinNvfp4ProcessGlobalScale(1.0f, sf));
+
+  const size_t wq_expert_words = static_cast<size_t>(size_n) * size_k / 2 / 4;
+  for (int e = 0; e < E; ++e) {
+    vt::cuda::MarlinRepackExpertWeight(
+        stream, dev_id,
+        static_cast<uint32_t*>(wq.ptr()) + static_cast<size_t>(e) * wq_expert_words,
+        static_cast<const uint8_t*>(staging.ptr()), size_k, size_n);
+    vt::cuda::MarlinProcessExpertScales(
+        stream, static_cast<const uint8_t*>(staging_s.ptr()),
+        static_cast<uint8_t*>(sc.ptr()) + static_cast<size_t>(e) * scale_bytes,
+        size_k, size_n, sf);
+  }
+  b.Synchronize(q);
+  Dev dgs(b, q, DType::kF32, {E}, gs.data());
+
+  // Routing, drawn from `pool` distinct experts -- the block-count control.
+  const int P = M * top_k;
+  std::vector<int32_t> topk_ids(static_cast<size_t>(P));
+  std::vector<float> topk_w(static_cast<size_t>(P), 1.0f);
+  for (int i = 0; i < P; ++i)
+    topk_ids[static_cast<size_t>(i)] = static_cast<int32_t>(rng() % static_cast<unsigned>(pool));
+
+  const int block = vt::cuda::MarlinMoeAlignBlockSizeSelect(M, top_k, E);
+  int max_tok = 0, max_blk = 0;
+  vt::cuda::MarlinMoeAlignSizes(M, top_k, E, block, &max_tok, &max_blk);
+  Dev dtid(b, q, DType::kI32, {M, top_k}, topk_ids.data());
+  Dev dtw(b, q, DType::kF32, {M, top_k}, topk_w.data());
+  Dev sorted_ids(b, q, DType::kI32, {max_tok});
+  Dev expert_ids(b, q, DType::kI32, {max_blk});
+  Dev num_pad(b, q, DType::kI32, {1});
+  vt::cuda::MarlinMoeAlignBlockSize(stream, static_cast<const int32_t*>(dtid.ptr()), M,
+                                    top_k, E, block,
+                                    static_cast<int32_t*>(sorted_ids.ptr()),
+                                    static_cast<int32_t*>(expert_ids.ptr()),
+                                    static_cast<int32_t*>(num_pad.ptr()));
+  b.Synchronize(q);
+  int32_t past = 0;
+  b.Copy(q, &past, num_pad.ptr(), sizeof(int32_t));
+  b.Synchronize(q);
+
+  const int sms = vt::cuda::MarlinDeviceSms(dev_id);
+  Dev ws(b, q, DType::kI32, {sms * 4});
+  Dev dact(b, q, DType::kBF16, {M, K});
+  Dev dout(b, q, DType::kBF16, {P, size_n});
+
+  vt::MoeMarlinArgs args{};
+  args.moe_block_size = block;
+  args.top_k = top_k;
+  args.size_m = M;
+  args.size_n = size_n;
+  args.size_k = size_k;
+  args.mul_topk_weights = false;
+
+  // vLLM's arm does NOT re-zero the workspace per call (the kernel leaves it
+  // reset), so timing ours WITH a per-call memset adds a launch upstream never
+  // pays. --zero-ws 0 removes that asymmetry.
+  b.Memset(q, ws.ptr(), 0, static_cast<size_t>(sms) * 4 * sizeof(int32_t));
+  auto once = [&]() {
+    if (zero_ws) b.Memset(q, ws.ptr(), 0, static_cast<size_t>(sms) * 4 * sizeof(int32_t));
+    vt::MoeGroupedGemmNvfp4Marlin(q, dout.tensor(), dact.tensor(), wq.tensor(),
+                                  sc.tensor(), dgs.tensor(), ws.tensor(),
+                                  sorted_ids.tensor(), expert_ids.tensor(),
+                                  num_pad.tensor(), dtw.tensor(), args);
+  };
+
+  for (int i = 0; i < warmup; ++i) once();
+  b.Synchronize(q);
+  const auto t0 = std::chrono::steady_clock::now();
+  for (int i = 0; i < iters; ++i) once();
+  b.Synchronize(q);
+  const auto t1 = std::chrono::steady_clock::now();
+
+  const double us =
+      std::chrono::duration<double, std::micro>(t1 - t0).count() / iters;
+  const int blocks = past / block;
+  std::printf("OURS gate_up M=%d pool=%d zero_ws=%d blocks=%d us_per_call=%.3f us_per_block=%.4f\n",
+              M, pool, zero_ws, blocks, us, us / (blocks > 0 ? blocks : 1));
+  return 0;
+}

docs/BENCHMARKS.md

@@ -351,7 +351,7 @@ in the tree, default-OFF, for reproducibility; detail in the benchmark record.
 | MTP | Qwen3.6-27B NVFP4 | token-identical to vLLM MTP, **~4% faster at c1**; on-par at c2-c8 | `DONE` |
 | DFlash | Qwen3.6-27B NVFP4 | **2.9x over spec-off** (10.16 → 29.32 tok/s), at/above vLLM DFlash-on (**1.003x**, non-overlapping bands) | `DONE` |
 | n-gram | Qwen3.6-27B NVFP4 | draft-free (`SPEC-NGRAM`); 27B 5/5 STRICT our-ngram-ON == vLLM-ngram-ON, 180/180 drafts accepted (correctness only, no speed row yet) | `DONE` |
-| DSpark | 27B NVFP4 dense k=15; 35B-A3B MoE k=8 | MoE **0.975x** code / **1.012x** prose vs the pinned graphed oracle (PINNED CLOCKS, non-overlapping). NOT parity: **~0.966x +/- 0.01** over three within-session pairs; C_tmp cap perf-NEUTRAL; storage refuted (#442) | `ACTIVE` |
+| DSpark | 27B NVFP4 dense k=15; 35B-A3B MoE k=8 | MoE 35B-A3B: valid within-session ratios **0.957-0.989** across boots (one run rejected on a drift gate). NOT parity; Marlin localisation REFUTED (#442) | `ACTIVE` |
 | Breadth (EAGLE1/3, suffix, ngram-gpu, dynamic-k, ...) | n/a | enumerated from vLLM source + `INVENTORIED` 2026-08-06 (`.agents/specs/spec-decode-inventory.md`), unmeasured | `INVENTORIED` |
 
 ## How we measure

docs/STATUS.md

@@ -526,48 +526,45 @@ token, 17 steps instead of 18), and our acceptance equals its modal value
 (48.6%, 4.94 tokens/step). Per-step the engines are aligned (30.4 vs ~30.1 ms,
 34.7 vs ~34.5). Under a LOW-NOISE harness (clocks pinned at 1800
 MHz, 30 reps, drift bracketed at -0.088%, the oracle's non-modal draws excluded)
-the code cell is **0.975x with NON-OVERLAPPING distributions** — a real gap, not
-noise, and the earlier "within resolution" reading was too generous. Ours slowed
+that session's code cell measured **0.975x with NON-OVERLAPPING
+distributions** — a real gap, not noise, and the earlier "within resolution"
+reading was too generous. It is one of four within-session ratios, listed
+below. Ours slowed
 more than the oracle when the clock was pinned, so the residual is
-SM-clock-sensitive work. PAIRED profiling localises it exactly: the SAME
-`marlin_moe_wna16::Marlin` kernel, the SAME 1520 launches, ours 249.22 ms vs
-upstream 230.39 ms -- **8.2% slower inside one kernel**, which at ~34% of wall is
-2.8% end-to-end and accounts for the whole measured 2.5%. Not an algorithm difference, and not the launch
-geometry either: the full template arguments match, `determine_exec_config` is
-byte-identical to the pinned upstream copy, and every OTHER kernel matches to
-0.2%. The inputs match too (scale bytes per expert,
-256-byte alignment, cudaMalloc residency), and the work counts were MEASURED:
-upstream loops 4.4% MORE blocks per launch (40.6 vs 38.9) and is still faster, so
-routing is refuted and normalising by work makes our deficit bigger -- **4.21 vs
-3.73 us per block, ~12.8% slower per unit of work**. Every source-level explanation is now
-eliminated -- kernel source, template instantiation, grid config, block size,
-shared-memory budget, reduction flags, scale layout, alignment, residency, CUDA
-toolkit (13.0 both) and arch all match -- and `ncu` plus cuobjdump then showed the
-COMPILED KERNELS ARE EQUIVALENT (94 registers and 3664 SASS instructions on both,
-upstream running its family-compatible sm_120 cubin against our sm_121a). The
-residual is therefore runtime and is now ATTRIBUTED: the kernel is DRAM-bound
-(L2 hit 9.5%) and we sustain **186.6 GB/s against upstream's 210.7**, a 12.9%
-effective-bandwidth gap that IS the whole per-unit-work difference. Weight
-residency is already staged correctly (cudaMalloc + one upload), and the slab itself is byte-for-byte the
-same size and stride as upstream's tensor (268 MB, no padding), so the cause is
-memory-system behaviour that no allocation change we can name would alter; upstream's ncu counters would settle it but its engine will not initialise under
-ncu in either replay mode. A C_tmp over-allocation (15-30 MB vs upstream's
-3.15 MB) was found and fixed, but an in-session A/B shows it is perf-NEUTRAL
-(+0.03%) -- an apparent +2.9% was machine drift, since GB10 cannot lock memory
-clocks. The ratio has now been measured WITHIN a single session three times --
-0.9757, 0.9646 and (ours->oracle->ours at free clocks, drift -0.89%) 0.9569 --
-so it is **~0.966 +/- 0.01, consistently below 1.0**, while the absolute numbers
-move up to 5% BETWEEN sessions for the same binary. Storage was raised as a
-possible distortion and is refuted: the weights are on local NVMe (no NAS mount
-exists on the box), a run reads 22.06 GB once at load, decode-time RSS is 4.8 GB
-because the mapping is released after upload, and 8 warm reps hold a 0.5%
-spread -- decode touches no storage. (That NVMe is 98% full, 76 GB free, which
-is its own operational risk given ENOSPC has previously produced a green report
-over a gate that never ran.) Editing
-the kernel, its launch config, layout or flags is NOT indicated: all are proven
-identical. (The repack kernels that appear to take 40% of a long run are
-LOAD-TIME.) NOT parity. The Gemma4 `1 + N` layout is coded and unit-tested but has
-never run on real weights.
+SM-clock-sensitive work. PAIRED profiling appeared to localise the residual to
+`marlin_moe_wna16::Marlin` (ours 249.22 ms vs upstream 230.39 ms over the same
+1520 launches), and every source-level explanation was eliminated -- kernel
+source, template instantiation, grid, block size, shared memory, flags, scale
+layout, alignment, residency, toolkit, arch -- with ncu and cuobjdump showing
+the compiled kernels EQUIVALENT (94 registers, 3664 SASS instructions on
+both). THAT LOCALISATION IS REFUTED. `scripts/marlin-moe-standalone.py` and
+`benchmarks/marlin_moe_standalone.cpp` drive each engine's own kernel outside
+its engine -- which is also what finally lets ncu attach to upstream,
+previously recorded as impossible in both replay modes -- and at matched work
+the two are indistinguishable: over 12 interleaved paired points ours averages
+5.3187 us/block against upstream's 5.3330, ratio 0.9973, sign flipping between
+runs, inside one standard deviation. The in-situ 8.2% therefore describes the
+RUNS, not the kernel, and so do the 12.8%-per-unit-work and 186.6-vs-210.7
+GB/s figures derived from it -- the latter also divided GRAPHED times by
+EAGER-mode block counts, so numerator and denominator came from different
+execution modes. END-TO-END, valid within-session paired ratios are 0.9757,
+0.9646, 0.9569 and 0.9889 (a fifth run was REJECTED on a -2.13% drift gate): a
+spread of 0.957-0.989 ACROSS BOOTS, with no single value being the ratio and
+the gap not resolved better than 1-4% on this hardware. An earlier claim here
+that the gap was 1.1% rather than 3.4% is WITHDRAWN: decomposed, our arm moved
++1.10% between those sessions while the ORACLE denominator moved -2.17%, so
+most of it was the oracle's boot state, and differencing ratios across boots
+is what this page forbids elsewhere. The warm-up arm does demonstrably remove
+a 6.6% WITHIN-RUN drift, which makes a run internally valid without moving the
+ratio. Standing traps: dram__bytes.sum reads n/a on GB10, so ncu's Memory
+Throughput % excludes DRAM traffic and is not a bandwidth utilisation; the GPU
+lock is $HOME/gpu.lock, and absolute timings from runs that took /tmp/gpu.lock
+ran unserialised and are LOWER bounds on achievable bandwidth, so a clean re-
+take can only raise the plateau; any MoE comparison that lets routing vary
+between arms measures the draw, not the change; and both blocks AND distinct
+experts must be controlled, since cost per distinct expert spans 4.47-7.50 us
+and is flat only above ~40 experts. NOT parity, and the row stays open.
+
 Multimodal
 (image/video/audio) is correctness-complete and its OpenAI-server wiring has
 landed all three CPU bricks (content-part parse + processor routing, the

scripts/dspark-paired-e2e.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# DSpark 35B-A3B paired end-to-end measurement: ours -> oracle -> ours.
+#
+# COMMITTED because the headline ratio rests on it and a median cannot be
+# recomputed from a record that only stores medians. Per-rep values for the
+# runs it produced are in .agents/benchmark-record.md.
+#
+# FOUR CONTROLS, none of which is optional -- each was added because its
+# absence produced a wrong number:
+#   * $HOME/gpu.lock, NOT /tmp/gpu.lock. The latter coordinates with nothing.
+#   * a DISCARDED warm-up arm: the GB10 SM clock ramps over MINUTES
+#     (1449 -> 2190 MHz observed), so dropping rep 1 is not enough.
+#   * settle barriers between arms: vLLM asserts free GPU memory does not GROW
+#     during its startup profile, and GB10 releases our pages lazily, so an
+#     oracle started straight after our arm aborts.
+#   * a host-RAM headroom guard: gpu_memory_utilization reserves HOST RAM here,
+#     so an oracle without headroom takes the MACHINE down, not the process.
+#
+# READ THE RESULT WITH ITS GATE: if the two `ours` arms differ by >= 1%, the run
+# is REJECTED, not averaged. And ratios are comparable only WITHIN one boot --
+# both arms move several percent across reboots, so a ratio from boot A and one
+# from boot B cannot be differenced.
+# Paired DSpark measurement, ours -> oracle -> ours, under the REAL lock, with a
+# SETTLE BARRIER between arms.
+#
+# Why the barrier: vLLM profiles free GPU memory at startup and asserts it does
+# not GROW during profiling. On GB10 the unified-memory allocator returns our
+# engine's pages lazily, so an oracle started immediately after our arm sees
+# free memory rise mid-profile and dies with
+#   "Error in memory profiling. Initial free memory 68.53 GiB, current 89.42 GiB"
+# That is a harness sequencing defect, not a property of either engine.
+set -uo pipefail
+DIR="$HOME/work/dspark-w6"
+OUT="${VT_PAIRED_LOG:-$DIR/paired_e2e.log}"; : > "$OUT"
+export PATH="/usr/local/cuda/bin:$HOME/venvs/vllm-oracle-next/bin:$HOME/.local/bin:$PATH"
+export VLLM_ENABLE_V1_MULTIPROCESSING=0
+CLI="$DIR/src/build/examples/vllm-cli"
+T35=$(ls -d "$HOME"/.cache/huggingface/hub/models--nvidia--Qwen3.6-35B-A3B-NVFP4/snapshots/*/ | head -1)
+D35=$(ls -d "$HOME"/.cache/huggingface/hub/models--RedHatAI--Qwen3.6-35B-A3B-speculator.dspark/snapshots/*/ | head -1)
+P=$(printf 'def fibonacci(n):\n    ')
+
+clocks() { nvidia-smi --query-gpu=clocks.sm,clocks.max.sm,temperature.gpu --format=csv,noheader; }
+
+settle() {
+  # Wait for every compute app to disappear, then give the allocator a fixed
+  # grace period. Polling free memory is not an option: GB10 reports it [N/A].
+  local n
+  for _ in $(seq 1 60); do
+    n=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null | grep -c . || echo 0)
+    [ "$n" -eq 0 ] && break
+    sleep 5
+  done
+  sleep 60
+}
+
+ours() {
+  echo "=== OURS $1 clocks=$(clocks)" >> "$OUT"
+  "$CLI" --model "$T35" --max-num-seqs 2 --prompt "$P" --max-tokens 128 --temperature 0 \
+    --repeat 10 \
+    --speculative-config "{\"method\":\"dspark\",\"model\":\"$D35\",\"num_speculative_tokens\":8}" >> "$OUT" 2>&1
+  echo "OURS_${1}_RC=$?" >> "$OUT"
+}
+
+go() {
+  echo "=== boot: $(who -b | tr -s ' ')" >> "$OUT"
+  # WARM-UP ARM, DISCARDED. On a freshly booted GB10 the SM clock ramps over
+  # MINUTES (measured 1449 -> 2190 MHz across one paired run), so discarding
+  # rep 1 is not enough -- the whole first arm reads low. A previous run
+  # bracketed ours at 133.7 before and 142.5 after, a 6.6% drift that swamps
+  # the ~3% being measured. This arm exists to be thrown away.
+  ours warmup
+  settle
+  ours before
+  settle
+  echo "=== ORACLE clocks=$(clocks)" >> "$OUT"
+  "$HOME/venvs/vllm-oracle-next/bin/python" "$DIR/${VT_ORACLE_SCRIPT:-fibacc_lowmem.py}" >> "$OUT" 2>&1
+  echo "ORACLE_RC=$?" >> "$OUT"
+  settle
+  ours after
+  echo "=== clocks END: $(clocks)" >> "$OUT"
+}
+export -f go ours settle clocks; export OUT DIR CLI T35 D35 P HOME
+
+docker stop local-ai-worker >> "$OUT" 2>&1 || true
+flock -w 28800 "$HOME/gpu.lock" bash -c go
+echo "=== paired_e2e done $(date -Is)" >> "$OUT"