From 27f00614e059a3e1f7cc4d2bc2892a11f32a9e25 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Mon, 13 Jul 2026 17:59:39 +0530 Subject: [PATCH 1/3] perf(dflash): add MMID dispatch telemetry --- .../llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu | 36 ++++++++++++++++++- .../deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu | 23 ++++++++++++ server/docs/ENVIRONMENT.md | 1 + 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu index 3aa2eb481..033b4e36b 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2564,19 +2564,36 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_TENSOR_BINARY_OP_LOCALS const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + static const bool mmid_telemetry = []() { + const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); + const int mmvq_mmid_max = ggml_is_quantized(src0->type) + ? get_mmvq_mmid_max_batch(src0->type, cc) : 0; + const auto log_dispatch = [&](const char * path) { + if (mmid_telemetry) { + std::fprintf(stderr, + "[dflash-mmid] event=dispatch name=%s type=%s ne11=%lld width=%lld pairs=%lld " + "n_experts=%lld top_k=%lld mmvq_max=%d path=%s\n", + dst->name, ggml_type_name(src0->type), (long long) ne11, (long long) ne2, + (long long) (ids->ne[0]*ne2), (long long) ne02, (long long) ids->ne[0], + mmvq_mmid_max, path); + } + }; // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { static_assert(MMVQ_MAX_BATCH_SIZE == MMVF_MAX_BATCH_SIZE); if (ne2 <= MMVQ_MAX_MOE_BATCH_SIZE) { if (ggml_is_quantized(src0->type)) { - const int mmvq_mmid_max = get_mmvq_mmid_max_batch(src0->type, cc); if (ne2 <= mmvq_mmid_max) { + log_dispatch("mmvq"); ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst); return; } } else { if (ne2 <= MMVF_MAX_BATCH_SIZE && GGML_CUDA_CC_IS_AMD(cc)) { + log_dispatch("mmvf"); ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst); return; } @@ -2584,16 +2601,20 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * } if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) { + log_dispatch("mmq"); ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst); return; } if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { + log_dispatch("mmf"); ggml_cuda_mul_mat_f(ctx, src0, src1, ids, dst); return; } } + log_dispatch("sync_fallback"); + // note: this path should not be reached when recording CUDA graphs, because it requires stream synchronization // TODO: add asserts to verify this. should work with CUDA, HIP, etc. cudaStream_t stream = ctx.stream(); @@ -3217,6 +3238,19 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { const bool mmid_mmq_ok = ggml_is_quantized(node->src[0]->type) && ggml_cuda_should_use_mmq(node->src[0]->type, cc, node->src[1]->ne[2], node->src[0]->ne[2]); + static const bool mmid_telemetry = []() { + const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); + if (mmid_telemetry) { + std::fprintf(stderr, + "[dflash-mmid] event=graph name=%s type=%s ne11=%lld width=%lld " + "mmvq_max=%d mmvq_ok=%d mmq_ok=%d node_eligible=%d\n", + node->name, ggml_type_name(node->src[0]->type), + (long long) node->src[1]->ne[1], (long long) node->ne[2], + mmvq_mmid_max, mmid_mmvq_ok, mmid_mmq_ok, + mmid_mmvq_ok || mmid_mmq_ok); + } if (!mmid_mmvq_ok && !mmid_mmq_ok) { // under these conditions, the mul_mat_id operation will need to synchronize the stream, so we cannot use CUDA graphs // TODO: figure out a way to enable for larger batch sizes, without hurting performance diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu index ec6288f97..1e6225b0f 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu @@ -4,7 +4,9 @@ #include "vecdotq.cuh" #include +#include #include +#include typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs); @@ -1642,6 +1644,10 @@ void ggml_cuda_mul_mat_vec_q( const int64_t stride_channel_y = ids ? s11 : s12; const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0; + static const bool mmid_telemetry = []() { + const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); // [TAG_MMID_GROUPED] grouped-expert path for small MUL_MAT_ID batches. if (ids && ncols_dst >= 2 && ncols_dst <= MMVQ_MAX_MOE_BATCH_SIZE && @@ -1682,10 +1688,27 @@ void ggml_cuda_mul_mat_vec_q( (int) s01, (int) stride_col_y, (int) stride_col_dst, (int) s02, (int) stride_channel_y, (int) stride_channel_dst, np, stream)) { + if (mmid_telemetry) { + std::fprintf(stderr, + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%d variant=grouped\n", + ggml_type_name(src0->type), (long long) ncols_dst, np); + } return; } } + if (mmid_telemetry && ids) { + const char * reason = ncols_dst < 2 ? "width_lt_2" : + ncols_dst > MMVQ_MAX_MOE_BATCH_SIZE ? "width_gt_16" : + (int) (nchannels_dst*ncols_dst) > MMID_GROUPED_MAX_PAIRS ? "pairs_gt_256" : + !mmid_grouped_env() ? "flag_off" : + !mmid_grouped_type_ok(src0->type) ? "unsupported_type" : "dispatch_rejected"; + std::fprintf(stderr, + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=legacy_moe reason=%s\n", + ggml_type_name(src0->type), (long long) ncols_dst, + (long long) (nchannels_dst*ncols_dst), reason); + } + mul_mat_vec_q_switch_type( src0->data, src0->type, src1_q8_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index f711a920c..91c23cc17 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -24,6 +24,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. | `DFLASH_ADAPTIVE_K_TAU` | 0 = off | Prefer the CLI: --adaptive-experts [tau]. Cumulative combine-weight threshold for per-token expert gating. | | `DFLASH_ADAPTIVE_K_DENSE` | per-model default | CSV of MoE layers kept dense under adaptive-K (DFlash capture layers). Warned-inert on families that do not thread layer indices yet. | | `DFLASH_MMID_GROUPED` | unset | Grouped MUL_MAT_ID kernel for small verify batches; candidate for CLI promotion. | +| `DFLASH_MMID_TELEMETRY` | unset | DEBUG: report MUL_MAT_ID dispatch, MMVQ variant, and per-node graph compatibility. | | `DFLASH_KVFLASH` | unset | Prefer the CLI: `--kvflash` (token count or `auto`). | ## Full inventory (generated) From 8e2b13037e1bd7384295f485e204acbc8bcc73bd Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Tue, 14 Jul 2026 22:57:27 +0530 Subject: [PATCH 2/3] fix(dflash): correct MMID telemetry labels for single-token and non-quant paths - mmvq.cu: width-1 MUL_MAT_ID now reports variant=single instead of variant=legacy_moe. The single-column MMVQ case is not the multi-token legacy MoE launch, so the old label conflated single-token decode with grouping-rejected batches. - ggml-cuda.cu: graph telemetry reports mmvq_max=0 for non-quantized nodes instead of the type-independent ceiling get_mmvq_mmid_max_batch returns, matching the mmvq_ok=0 eligibility already reported for those nodes. - ENVIRONMENT.md: add DFLASH_MMID_TELEMETRY to the generated inventory list. Diagnostics stay default-off; no inference-path behavior change. --- .../llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu | 5 +++- .../deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu | 27 ++++++++++++------- server/docs/ENVIRONMENT.md | 1 + 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu index 033b4e36b..809d5f60c 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3228,7 +3228,10 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (node->op == GGML_OP_MUL_MAT_ID) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const int mmvq_mmid_max = get_mmvq_mmid_max_batch(node->src[0]->type, cc); + // Non-quantized nodes never take the MMVQ path, so report a 0 ceiling + // instead of the type-independent value get_mmvq_mmid_max_batch returns. + const int mmvq_mmid_max = ggml_is_quantized(node->src[0]->type) + ? get_mmvq_mmid_max_batch(node->src[0]->type, cc) : 0; // Mirror ggml_cuda_mul_mat_id: the MMVQ and MMQ mul_mat_id paths // are stream-sync-free and safe to capture; only the sort-based // fallback requires a stream synchronize. diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu index 1e6225b0f..1f7c76223 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu @@ -1698,15 +1698,24 @@ void ggml_cuda_mul_mat_vec_q( } if (mmid_telemetry && ids) { - const char * reason = ncols_dst < 2 ? "width_lt_2" : - ncols_dst > MMVQ_MAX_MOE_BATCH_SIZE ? "width_gt_16" : - (int) (nchannels_dst*ncols_dst) > MMID_GROUPED_MAX_PAIRS ? "pairs_gt_256" : - !mmid_grouped_env() ? "flag_off" : - !mmid_grouped_type_ok(src0->type) ? "unsupported_type" : "dispatch_rejected"; - std::fprintf(stderr, - "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=legacy_moe reason=%s\n", - ggml_type_name(src0->type), (long long) ncols_dst, - (long long) (nchannels_dst*ncols_dst), reason); + if (ncols_dst < 2) { + // Single-token MUL_MAT_ID: the ordinary single-column MMVQ case, not + // the multi-token legacy MoE launch the grouped path falls back to. + std::fprintf(stderr, + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=single\n", + ggml_type_name(src0->type), (long long) ncols_dst, + (long long) (nchannels_dst*ncols_dst)); + } else { + const char * reason = + ncols_dst > MMVQ_MAX_MOE_BATCH_SIZE ? "width_gt_16" : + (int) (nchannels_dst*ncols_dst) > MMID_GROUPED_MAX_PAIRS ? "pairs_gt_256" : + !mmid_grouped_env() ? "flag_off" : + !mmid_grouped_type_ok(src0->type) ? "unsupported_type" : "dispatch_rejected"; + std::fprintf(stderr, + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=legacy_moe reason=%s\n", + ggml_type_name(src0->type), (long long) ncols_dst, + (long long) (nchannels_dst*ncols_dst), reason); + } } mul_mat_vec_q_switch_type( diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index 91c23cc17..77d9dc77b 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -116,6 +116,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `DFLASH_LAGUNA_VERIFY_WIDTH` - laguna_backend.cpp - `DFLASH_LAGUNA_VERIFY_WIDTH_MAX` - laguna_backend.cpp - `DFLASH_MAX_CONTEXT` - laguna_backend.cpp, qwen35moe_backend.cpp +- `DFLASH_MMID_TELEMETRY` - ggml-cuda.cu, mmvq.cu - `DFLASH_MMQ_FULL_BATCH_MIN` - moe_hybrid_ffn_eval.cpp - `DFLASH_MMQ_SUB_BATCH` - moe_hybrid_ffn_eval.cpp - `DFLASH_MODEL_CARDS_DIR` - model_card.cpp From 9af5389ebb8f12b891651ec1d45b3b1bc8b33395 Mon Sep 17 00:00:00 2001 From: cheese-cakee Date: Wed, 15 Jul 2026 03:03:33 +0530 Subject: [PATCH 3/3] fix(dflash): report actual MMVQ variant and log every graph node - mmvq.cu: the ungrouped MUL_MAT_ID path now labels the kernel that actually runs (moe / tokenwise / generic), derived from the same env flags mul_mat_vec_q_switch_type reads, instead of always variant=legacy_moe. The DFLASH_CUDA_MMVQ_MOE_TOKENWISE and DFLASH_CUDA_MMVQ_MOE_KERNEL modes are now reported accurately. - ggml-cuda.cu: the graph-compatibility scan no longer breaks on the first graph-incompatible node when telemetry is on, so event=graph is emitted for every MUL_MAT_ID node. The early exit is preserved when telemetry is off, so the graph-capture decision is unchanged either way. Diagnostics stay default-off; no inference-path behavior change. --- .../llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu | 12 +++++++----- .../deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu index 809d5f60c..d05cf68d3 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3209,6 +3209,10 @@ static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { bool use_cuda_graph = true; + static const bool mmid_telemetry = []() { + const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); // Loop over nodes in GGML graph to obtain info needed for CUDA graph for (int i = 0; i < cgraph->n_nodes; i++) { @@ -3241,10 +3245,6 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { const bool mmid_mmq_ok = ggml_is_quantized(node->src[0]->type) && ggml_cuda_should_use_mmq(node->src[0]->type, cc, node->src[1]->ne[2], node->src[0]->ne[2]); - static const bool mmid_telemetry = []() { - const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); - return value != nullptr && std::strcmp(value, "0") != 0; - }(); if (mmid_telemetry) { std::fprintf(stderr, "[dflash-mmid] event=graph name=%s type=%s ne11=%lld width=%lld " @@ -3265,7 +3265,9 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { } } - if (!use_cuda_graph) { + // With telemetry on, keep scanning so event=graph is emitted for every + // MUL_MAT_ID node; the early exit is only a performance shortcut. + if (!use_cuda_graph && !mmid_telemetry) { break; } } diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu index 1f7c76223..23effdc1e 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu @@ -1706,15 +1706,29 @@ void ggml_cuda_mul_mat_vec_q( ggml_type_name(src0->type), (long long) ncols_dst, (long long) (nchannels_dst*ncols_dst)); } else { + // Name the kernel mul_mat_vec_q_switch_type will actually run for this + // ungrouped multi-token batch, so the label is not misreported when the + // tokenwise or generic MMVQ modes are selected by env. + static const bool tokenwise_mmid = []() { + const char * e = std::getenv("DFLASH_CUDA_MMVQ_MOE_TOKENWISE"); + return e && e[0] == '1' && e[1] == '\0'; + }(); + static const bool moe_kernel = []() { + const char * e = std::getenv("DFLASH_CUDA_MMVQ_MOE_KERNEL"); + return !(e && e[0] == '0' && e[1] == '\0'); + }(); + const char * variant = + tokenwise_mmid ? "tokenwise" : + (moe_kernel || ncols_dst > MMVQ_MAX_BATCH_SIZE) ? "moe" : "generic"; const char * reason = ncols_dst > MMVQ_MAX_MOE_BATCH_SIZE ? "width_gt_16" : (int) (nchannels_dst*ncols_dst) > MMID_GROUPED_MAX_PAIRS ? "pairs_gt_256" : !mmid_grouped_env() ? "flag_off" : !mmid_grouped_type_ok(src0->type) ? "unsupported_type" : "dispatch_rejected"; std::fprintf(stderr, - "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=legacy_moe reason=%s\n", + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=%s reason=%s\n", ggml_type_name(src0->type), (long long) ncols_dst, - (long long) (nchannels_dst*ncols_dst), reason); + (long long) (nchannels_dst*ncols_dst), variant, reason); } }