perf(qwen35): accelerate Ampere prefill GDN

server/CMakeLists.txt

@@ -642,6 +642,18 @@ endif()
 option(DFLASH27B_TESTS "Build numerics tests" ON)
 option(DFLASH27B_SERVER "Build dflash_server + backend_ipc_daemon" ON)
 if(DFLASH27B_TESTS)
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_gdn_grouped_policy.cpp")
+        add_executable(test_gdn_grouped_policy test/test_gdn_grouped_policy.cpp)
+        target_include_directories(test_gdn_grouped_policy PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src/ggml-cuda)
+        add_test(NAME gdn_grouped_policy COMMAND test_gdn_grouped_policy)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_qwen35_gdn_broadcast.cpp")
+        add_executable(test_qwen35_gdn_broadcast test/test_qwen35_gdn_broadcast.cpp)
+        target_include_directories(test_qwen35_gdn_broadcast PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
+        target_link_libraries(test_qwen35_gdn_broadcast PRIVATE dflash_common)
+        add_test(NAME qwen35_gdn_broadcast COMMAND test_qwen35_gdn_broadcast)
+    endif()
     if(DFLASH27B_GPU_BACKEND STREQUAL "cuda" AND _dflash_cuda_min_sm GREATER_EQUAL 80 AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_flashprefill_kernels.cpp")
         add_executable(test_flashprefill_kernels test/test_flashprefill_kernels.cpp)
         set_target_properties(test_flashprefill_kernels PROPERTIES CUDA_ARCHITECTURES "${_dflash_archs}")
@@ -712,6 +724,25 @@ if(DFLASH27B_TESTS)
         target_link_libraries(test_draft_topk_cuda PRIVATE dflash_common ${DFLASH27B_GGML_BACKEND_TARGET})
         add_test(NAME draft_topk_cuda COMMAND test_draft_topk_cuda)
     endif()
+    # Focused CUDA-vs-CPU oracle for the IQ4_XS MMQ stream-k scheduler. These
+    # shapes cover the no-fixup, partial-row, fixup, two-wave, and deeper-K
+    # transitions validated for the llama.cpp #22298 backport on sm_86.
+    if(DFLASH27B_GPU_BACKEND STREQUAL "cuda")
+        add_executable(test_gdn_grouped_cuda test/test_gdn_grouped_cuda.cpp)
+        target_link_libraries(test_gdn_grouped_cuda PRIVATE
+            ggml
+            ggml-cuda
+            ggml-base)
+        add_test(NAME gdn_grouped_cuda COMMAND test_gdn_grouped_cuda)
+
+        add_executable(test_mmq_streamk_iq4_xs test/test_mmq_streamk_iq4_xs.cpp)
+        target_link_libraries(test_mmq_streamk_iq4_xs PRIVATE
+            ggml
+            ggml-cpu
+            ggml-cuda
+            ggml-base)
+        add_test(NAME mmq_streamk_iq4_xs COMMAND test_mmq_streamk_iq4_xs)
+    endif()
     # GPU port of the sample_logits chain vs the CPU reference. CUDA only:
     # geometric_sampler_cuda.cu is compiled into dflash_common solely on the cuda backend.
     if(DFLASH27B_GPU_BACKEND STREQUAL "cuda" AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_gpu_sampler_cuda.cpp")

server/deps/llama.cpp/VENDOR.md

@@ -7,7 +7,16 @@ This directory contains the ggml-only subset used by Lucebox Hub.
 - Source base commit: `6fbe72d67069136bbd370be703e1d4f441b5e942`
 - Included merged PR: `#35` (`0fe65d9354b7c5da52a7741d2e37ba85f0d0c925`)
 - Included test PR: `#37` (`0699be81480428f01b9b7ac49a09a2d51c77f8df`)
-- Reconstruction: `luce-dflash@6fbe72d67069136bbd370be703e1d4f441b5e942` plus cherry-picked PRs `#35` and `#37`
+- Included upstream backport: `llama.cpp #22298` (`9725a313be0528214c4a02fed906ddaf7b3f712e`)
+- Reconstruction: `luce-dflash@6fbe72d67069136bbd370be703e1d4f441b5e942` plus cherry-picked PRs `#35`, `#37`, and upstream `llama.cpp #22298`
 - Vendored paths: `LICENSE`, `common/jinja`, `common/log.h`, `common/unicode.*`, `ggml`, `gguf-py`
 
 Open ggml feature PRs are intentionally not included until they are merged, except for explicitly listed hub test PRs.
+
+## Hub-local deltas
+
+- `ggml/src/ggml-cuda/gated_delta_net*`: Lucebox selects the existing grouped-column
+  Gated DeltaNet kernel for Ampere prefill launches of at least 512 tokens while
+  retaining the classic kernel for decode and partial chunks. The force/disable
+  environment controls remain available for differential testing. This policy is
+  maintained locally until it is suitable for upstreaming to `lucebox-ggml`.

server/deps/llama.cpp/ggml/src/ggml-cuda/gated_delta_net.cu

@@ -1,4 +1,5 @@
 #include "gated_delta_net.cuh"
+#include "gated_delta_net_policy.h"
 #ifndef GGML_USE_HIP
 #include <cuda_fp16.h>
 #endif
@@ -471,8 +472,8 @@ static void launch_gated_delta_net(
                             && cc <  GGML_CUDA_CC_ADA_LOVELACE;
     const bool force_grouped_cols = getenv("DFLASH_GDN_FORCE_GROUPED_COLS") != nullptr;
     const bool disable_grouped_cols = getenv("DFLASH_GDN_NO_GROUPED_COLS") != nullptr;
-    const bool use_grouped_cols = force_grouped_cols ||
-        (!disable_grouped_cols && !ampere_nvidia);
+    const bool use_grouped_cols = ggml_cuda_gdn_use_grouped_columns(
+        ampere_nvidia, n_tokens, force_grouped_cols, disable_grouped_cols);
 
     switch (S_v) {
         case 16:

server/deps/llama.cpp/ggml/src/ggml-cuda/gated_delta_net_policy.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <cstdint>
+
+// Grouped columns share each Q/K load across four state columns. On Ampere,
+// that reuse wins for prefill-sized launches but its larger register footprint
+// loses at decode and other short batch sizes. Diagnostic overrides retain the
+// existing force-over-disable precedence.
+inline bool ggml_cuda_gdn_use_grouped_columns(
+        bool ampere_nvidia,
+        int64_t n_tokens,
+        bool force_grouped_columns,
+        bool disable_grouped_columns) {
+    const bool ampere_prefill = ampere_nvidia && n_tokens >= 512;
+    return force_grouped_columns ||
+        (!disable_grouped_columns && (!ampere_nvidia || ampere_prefill));
+}