andreaborio · marmyx77 · Jul 15, 2026 · Jul 15, 2026 · Jul 15, 2026 · Jul 15, 2026
diff --git a/.gitignore b/.gitignore
@@ -21,3 +21,9 @@ __pycache__/
 /misc/
 .*.swp
 .DS_Store
+tests/test_qwen35_rocm
+tests/test_qwen_*
+tests/test_q4k_top8
+tests/test_ssd_residency
+*.orig
+labestia_server.log
diff --git a/Makefile b/Makefile
@@ -18,7 +18,10 @@ DEPFLAGS ?= -MMD -MP
 
 BUILD_GIT_SHA ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
 BUILD_GIT_SUFFIX ?= $(shell test -z "$$(git status --porcelain --untracked-files=normal 2>/dev/null)" || printf '%s' -dirty)
-CFLAGS += -DDS4_BUILD_GIT_SHA=\"$(BUILD_GIT_SHA)$(BUILD_GIT_SUFFIX)\"
+# Pass the sha as a bare token and stringify it in ds4_build.c: a quoted
+# -D value does not survive the extra shell evaluation in recursive targets
+# (strix-halo/cuda pass CFLAGS="$(CFLAGS) ..." through $(MAKE)).
+CFLAGS += -DDS4_BUILD_GIT_SHA=$(BUILD_GIT_SHA)$(BUILD_GIT_SUFFIX)
 
 LDLIBS ?= -lm -pthread
 METAL_SRCS := $(wildcard metal/*.metal)

diff --git a/RESULT_QWEN_ROCM.md b/RESULT_QWEN_ROCM.md
@@ -0,0 +1,82 @@
+# Qwen3.6-35B-A3B on ROCm (Radeon 780M) — result
+
+Date: 2026-07-16. Hardware: Minisforum UM790 Pro, Ryzen 9 7940HS, **iGPU Radeon
+780M (gfx1103, run as gfx1100 via HSA_OVERRIDE_GFX_VERSION=11.0.0)**, 64 GB DDR5,
+Pop!_OS, ROCm 6.3.1. Branch `rocm-qwen` (fork of andreaborio/ds4).
+
+## Status: **Qwen3.6-35B-A3B runs end-to-end on the 780M GPU.**
+
+Resident mode (19.37 GiB payload wired in GTT, no SSD streaming). Coherent
+generation; numerically matches the scalar CPU reference.
+
+### Measured (greedy, temp 0)
+
+| path | prefill t/s | generation t/s | notes |
+|---|---|---|---|
+| ds4 **ROCm GPU** (this port) | 11.8–14.3 | **13.4–14.9** | resident, 780M |
+| ds4 CPU reference | ~10 | ~10–11.5 | scalar oracle |
+| llama.cpp CPU (same artifact) | ~100 (pp64) | 16.1 (tg64) | 8 threads |
+
+The GPU path already beats the ds4 CPU reference. It does not yet beat
+llama.cpp — expected for a v1 correctness-first port (no HIP-graph batching, F32
+throughout, top-8 MoE run as two top-4 halves). Andrea's Metal numbers (58–66
+t/s on M5 Pro) are on ~2× the memory bandwidth and a mature kernel set.
+
+### Correctness (M2 parity gate)
+
+CPU-vs-GPU top-logprobs, prompt "The sea is", 5 steps:
+- **top-1 agreement: 5/5** (GPU selects the same token as the CPU reference at
+  every step)
+- **cosine 0.9994–0.9999** on the shared top-k logit vectors (Andrea's own
+  Metal-vs-llama.cpp gate is 0.99994)
+- max logit diff 0.5–1.7 out of ~26 (sub-1%) — pure f32 accumulation-order
+  difference between the scalar CPU path and the GPU kernels.
+
+Greedy 96-token text is token-identical to the CPU golden for the first ~40
+tokens, then diverges at a near-tie — the documented upstream behavior, not a
+bug.
+
+Per-op kernel parity (`tests/test_qwen35_rocm.c`, all 22 checks pass on the
+780M vs `ds4_qwen_ref.c`): split_q_gate, sigmoid_mul (elements/rows), rope,
+router (exact top-8 + weights), gqa decode/prefill, gated_delta_step (kd=128
+fast path + kd=64 generic), gated_delta_sequence_128, causal_conv step/sequence,
+rmsnorm_gated, gated_delta_controls, dequant_embedding_q8_0. The GDN recurrence
+(the novel, highest-risk kernel) matches to ~1e-8 relative.
+
+## What was built
+
+- **`rocm/ds4_rocm_qwen.cuh`**: 19 HIP kernels (1:1 port of `metal/qwen35.metal`;
+  Metal simdgroup → RDNA3 wave32, simd_sum/max/min → `__shfl_xor` butterfly) +
+  the 20 `ds4_gpu_qwen35_*` entry points. Byte-stride args mirror `ds4_metal.m`.
+- **Top-8 MoE**: the shared ROCm Q4_K MoE core is specialized for DeepSeek's
+  top-6 (fused `sum6` kernels). Qwen's top-8 runs as **two top-4 halves + add**
+  (`moe_split_selected_kernel` + `routed_moe_batch_two_half`), leaving the
+  DeepSeek path untouched.
+- **Host wiring (`ds4.c`)**: Qwen GPU forward/session/dispatch guards widened
+  from `__APPLE__` to `DS4_QWEN_GPU_BUILD`. No `DS4_BACKEND_ROCM` enum exists —
+  ROCm runs under `DS4_BACKEND_CUDA` — so `ds4_backend_is_qwen_gpu()` replaces
+  the `backend==METAL` checks. Env gate `DS4_QWEN_EXPERIMENTAL_ROCM=1`; resident
+  forced (SSD streaming for Qwen not ported).
+- **Artifact**: `tests/qwen/normalize_qwen36_gguf.py` reproduces the blessed
+  `Qwen3.6-35B-A3B-ds4-Q4_K_S.gguf` from the Unsloth UD-Q4_K_S source (Q6_K
+  ffn_down_exps → Q4_K, output → Q8_0, pad-id + chat template). ds4 accepts it;
+  inventory **f32=361 / q8_0=252 / q4_k=120 == the prereg histogram exactly**.
+- **Forward-ported La Bestia ROCm hardening** (multi-model range cache, readback
+  event ring + host-side waits, adaptive free-floor) onto the fork; build-sha
+  bug fixed.
+
+## Run
+
+```sh
+DS4_QWEN_EXPERIMENTAL_ROCM=1 HSA_OVERRIDE_GFX_VERSION=11.0.0 \
+  ./ds4 -m gguf/Qwen3.6-35B-A3B-ds4-Q4_K_S.gguf -c 4096 -p "…"
+```
+
+## Not done / next
+
+- Soak at 8k and 32k context; formal `ds4-bench` CSV.
+- Throughput: HIP-graph batching, fused gate/up+SwiGLU for the two-half MoE,
+  parallel GQA decode variant (only the serial GQA kernel is wired).
+- Server path on ROCm (chat.html) — engine wiring is done; needs a server smoke.
+- **Caveat**: the Qwen server is text-only (no tool-call), so it cannot replace
+  Ollama for JARVIS function-calling until upstream adds tool parsing.
diff --git a/ds4.c b/ds4.c
@@ -78,6 +78,25 @@ static bool ds4_backend_uses_graph(ds4_backend backend) {
     return backend == DS4_BACKEND_METAL || backend == DS4_BACKEND_CUDA;
 }
 
+/* The Qwen3.6 GPU runtime is Apple Metal today; the ROCm build reuses the
+ * legacy metal_graph_* graph backend, which presents at runtime as the CUDA
+ * backend enum (ds4_build_backend() still reports "rocm").  This helper marks
+ * the graph backend that carries the Qwen GPU path for the current build so
+ * the many backend equality checks stay in one place. */
+#if defined(__APPLE__) || (defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU))
+#define DS4_QWEN_GPU_BUILD 1
+#endif
+static bool ds4_backend_is_qwen_gpu(ds4_backend backend) {
+#if defined(__APPLE__)
+    return backend == DS4_BACKEND_METAL;
+#elif defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+    return backend == DS4_BACKEND_CUDA;
+#else
+    (void)backend;
+    return false;
+#endif
+}
+
 static bool ds4_backend_supports_ssd_streaming(ds4_backend backend) {
     if (backend == DS4_BACKEND_METAL) return true;
     if (backend == DS4_BACKEND_CUDA) {
@@ -12665,7 +12684,7 @@ void ds4_internal_qwen35_gpu_graph_free(
     qwen35_gpu_graph_free(storage);
 }
 
-#if defined(__APPLE__)
+#if defined(__APPLE__) || defined(DS4_ROCM_BUILD)
 /* =========================================================================
  * Qwen3.6 Metal one-token forward.
  * =========================================================================
@@ -14018,7 +14037,7 @@ static DS4_MAYBE_UNUSED bool qwen35_gpu_forward_token(
     return qwen35_gpu_forward_token_commands(
         logits, model, weights, graph, token, position, false, true);
 }
-#endif /* __APPLE__ */
+#endif /* __APPLE__ || DS4_ROCM_BUILD */
 
 /* =========================================================================
  * Metal Release Graph State.
@@ -28503,7 +28522,7 @@ struct ds4_session {
     ds4_dist_session *distributed;
 #ifndef DS4_NO_GPU
     ds4_gpu_graph graph;
-#if defined(__APPLE__)
+#ifdef DS4_QWEN_GPU_BUILD
     ds4_qwen35_gpu_graph qwen35_gpu_graph;
 #endif
 #endif
@@ -28839,7 +28858,7 @@ static bool ds4_session_is_qwen35_cpu(const ds4_session *s) {
 static DS4_MAYBE_UNUSED bool ds4_session_is_qwen35_metal(
         const ds4_session *s) {
     return ds4_session_is_qwen35(s) && s->engine->qwen_metal_runtime &&
-           s->engine->backend == DS4_BACKEND_METAL;
+           ds4_backend_is_qwen_gpu(s->engine->backend);
 }
 
 static uint32_t ds4_session_vocab_size(const ds4_session *s) {
@@ -28888,7 +28907,7 @@ static bool ds4_session_qwen35_timeline_valid(const ds4_session *s) {
     if (ds4_session_is_qwen35_cpu(s)) {
         return s->qwen35_cpu_cache.n_tokens == checkpoint_len;
     }
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
     if (ds4_session_is_qwen35_metal(s)) {
         return s->qwen35_gpu_graph.state_valid &&
                s->qwen35_gpu_graph.n_tokens == checkpoint_len;
@@ -28902,7 +28921,7 @@ static bool ds4_session_qwen35_reset_timeline(ds4_session *s) {
     bool ok = true;
     if (ds4_session_is_qwen35_cpu(s)) {
         ds4_qwen35_cpu_cache_reset(&s->qwen35_cpu_cache);
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
     } else if (ds4_session_is_qwen35_metal(s)) {
         /* qwen35_gpu_forward_token already performs a full reset after any
          * late, state-mutating failure.  Avoid zeroing the entire context a
@@ -31037,11 +31056,18 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) {
         return true;
     }
 
-    if (e->backend != DS4_BACKEND_METAL) {
+    /* The adaptive planner is capability-gated, not backend-gated: any
+     * backend that can produce a live host-memory snapshot gets the adaptive
+     * budget (ROCm UMA APUs need it as much as Metal does — without a
+     * host-aware floor the kernel TTM can evict the GPU queue buffers under
+     * peak cache pressure and freeze generation).  Backends without a
+     * snapshot keep the legacy fixed-fraction budget. */
+    ds4_ssd_host_memory memory;
+    if (e->backend != DS4_BACKEND_METAL &&
+        !ds4_gpu_host_memory_snapshot(&memory)) {
         return ds4_engine_configure_streaming_legacy_auto_cache(e);
     }
 
-    ds4_ssd_host_memory memory;
     if (!ds4_gpu_host_memory_snapshot(&memory)) {
         fprintf(stderr,
                 "ds4: SSD streaming auto cache: host memory snapshot unavailable; "
@@ -31672,9 +31698,9 @@ static bool ds4_engine_qwen35_metal_options_valid(
         const ds4_engine_options *opt,
         bool                      load_slice) {
     if (!e || !opt) return false;
-    if (e->backend != DS4_BACKEND_METAL) {
+    if (!ds4_backend_is_qwen_gpu(e->backend)) {
         fprintf(stderr,
-                "ds4: experimental Qwen Metal inference requires --metal\n");
+                "ds4: experimental Qwen GPU inference requires the graph backend\n");
         return false;
     }
     if (e->quality) {
@@ -31743,7 +31769,7 @@ static bool ds4_engine_qwen35_metal_options_valid(
     return true;
 }
 
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
 static bool ds4_engine_configure_qwen35_metal_streaming(ds4_engine *e) {
     if (!e || e->backend != DS4_BACKEND_METAL ||
         e->residency != DS4_RESIDENCY_SSD || !e->ssd_streaming) {
@@ -31941,15 +31967,15 @@ static bool ds4_engine_configure_qwen35_metal_streaming(ds4_engine *e) {
 }
 
 static bool ds4_engine_configure_qwen35_metal_resident(ds4_engine *e) {
-    if (!e || e->backend != DS4_BACKEND_METAL ||
+    if (!e || !ds4_backend_is_qwen_gpu(e->backend) ||
         e->residency != DS4_RESIDENCY_RESIDENT || e->ssd_streaming ||
         !e->model.map || e->model.tensor_data_pos >= e->model.size) {
         return false;
     }
 
     if (!ds4_gpu_init()) {
         fprintf(stderr,
-                "ds4: Metal backend unavailable while initializing Qwen resident mode\n");
+                "ds4: GPU backend unavailable while initializing Qwen resident mode\n");
         return false;
     }
     e->metal_ready = true;
@@ -32150,12 +32176,17 @@ int ds4_engine_open(ds4_engine **out, const ds4_engine_options *opt) {
             return 0;
         }
 
-        const char *experimental_metal =
-            getenv("DS4_QWEN_EXPERIMENTAL_METAL");
-        if (!experimental_metal || strcmp(experimental_metal, "1") != 0) {
+#ifdef DS4_ROCM_BUILD
+        const char *experimental_gate = getenv("DS4_QWEN_EXPERIMENTAL_ROCM");
+        const char *experimental_gate_name = "DS4_QWEN_EXPERIMENTAL_ROCM";
+#else
+        const char *experimental_gate = getenv("DS4_QWEN_EXPERIMENTAL_METAL");
+        const char *experimental_gate_name = "DS4_QWEN_EXPERIMENTAL_METAL";
+#endif
+        if (!experimental_gate || strcmp(experimental_gate, "1") != 0) {
             fprintf(stderr,
-                    "ds4: Qwen Metal inference requires "
-                    "DS4_QWEN_EXPERIMENTAL_METAL=1\n");
+                    "ds4: Qwen GPU inference requires %s=1\n",
+                    experimental_gate_name);
             ds4_engine_close(e);
             return 1;
         }
@@ -32164,14 +32195,22 @@ int ds4_engine_open(ds4_engine **out, const ds4_engine_options *opt) {
             ds4_engine_close(e);
             return 1;
         }
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
         if (!ds4_engine_resolve_residency(
                 e, opt, false, 0, 0, false)) {
             ds4_engine_close(e);
             return 1;
         }
         vocab_load(&e->vocab, &e->model);
         bool configured = false;
+#ifdef DS4_ROCM_BUILD
+        /* Qwen SSD streaming is not ported to ROCm; the ~19.4 GiB payload is
+         * resident on the 64 GB box, the regime this port targets. */
+        e->residency = DS4_RESIDENCY_RESIDENT;
+        e->ssd_streaming = false;
+        if (opt->warm_weights) model_warm_weights(&e->model);
+        configured = ds4_engine_configure_qwen35_metal_resident(e);
+#else
         if (e->residency == DS4_RESIDENCY_SSD && e->ssd_streaming) {
             if (opt->warm_weights) {
                 fprintf(stderr,
@@ -32185,17 +32224,18 @@ int ds4_engine_open(ds4_engine **out, const ds4_engine_options *opt) {
             configured =
                 ds4_engine_configure_qwen35_metal_resident(e);
         }
+#endif
         if (!configured) {
             fprintf(stderr,
-                    "ds4: Qwen Metal residency configuration failed for %s mode\n",
+                    "ds4: Qwen GPU residency configuration failed for %s mode\n",
                     ds4_residency_mode_name(e->residency));
             ds4_engine_close(e);
             return 1;
         }
         e->qwen_raw_runtime = true;
         e->qwen_metal_runtime = true;
         fprintf(stderr,
-                "ds4: experimental Qwen Metal %s runtime enabled\n",
+                "ds4: experimental Qwen GPU %s runtime enabled\n",
                 ds4_residency_mode_name(e->residency));
         *out = e;
         return 0;
@@ -32866,7 +32906,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
         return 1;
     }
     if (qwen35 && e->backend != DS4_BACKEND_CPU &&
-        !(e->backend == DS4_BACKEND_METAL && e->qwen_metal_runtime)) {
+        !(ds4_backend_is_qwen_gpu(e->backend) && e->qwen_metal_runtime)) {
         fprintf(stderr,
                 "ds4: Qwen session backend does not match its enabled runtime\n");
         return 1;
@@ -32940,7 +32980,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
     ds4_session *s = xcalloc(1, sizeof(*s));
     s->engine = e;
     s->ctx_size = ctx_size;
-#if defined(__APPLE__)
+#ifdef DS4_QWEN_GPU_BUILD
     if (qwen35) {
         s->prefill_cap = 1u;
         if (!qwen35_gpu_graph_alloc(
@@ -33037,7 +33077,7 @@ void ds4_session_free(ds4_session *s) {
     }
 #ifndef DS4_NO_GPU
     else if (ds4_session_is_qwen35_metal(s)) {
-#if defined(__APPLE__)
+#ifdef DS4_QWEN_GPU_BUILD
         if (getenv("DS4_METAL_MEMORY_REPORT") != NULL) {
             ds4_gpu_print_memory_report("before Qwen graph free");
         }
@@ -33772,7 +33812,7 @@ static int ds4_session_sync_qwen35_with_forward(
     return 0;
 }
 
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
 static bool ds4_session_qwen35_metal_forward_commit(
         ds4_session *s,
         int          token,
@@ -34099,7 +34139,7 @@ int ds4_session_sync(ds4_session *s, const ds4_tokens *prompt, char *err, size_t
             return ds4_session_sync_qwen35_with_forward(
                 s, prompt, err, errlen, qwen35_cpu_forward_token);
         }
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
         if (ds4_session_is_qwen35_metal(s)) {
             return ds4_session_sync_qwen35_metal(
                 s, prompt, err, errlen);
@@ -34519,7 +34559,7 @@ static int ds4_session_eval_qwen35_with_forward(
     return 0;
 }
 
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
 static int ds4_session_eval_qwen35_metal(
         ds4_session *s,
         int          token,
@@ -34584,7 +34624,7 @@ static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp,
             return ds4_session_eval_qwen35_with_forward(
                 s, token, err, errlen, qwen35_cpu_forward_token);
         }
-#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+#ifdef DS4_QWEN_GPU_BUILD
         if (ds4_session_is_qwen35_metal(s)) {
             return ds4_session_eval_qwen35_metal(
                 s, token, err, errlen);