diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu index 3aa2eb481..d05cf68d3 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2564,19 +2564,36 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_TENSOR_BINARY_OP_LOCALS const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + static const bool mmid_telemetry = []() { + const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); + const int mmvq_mmid_max = ggml_is_quantized(src0->type) + ? get_mmvq_mmid_max_batch(src0->type, cc) : 0; + const auto log_dispatch = [&](const char * path) { + if (mmid_telemetry) { + std::fprintf(stderr, + "[dflash-mmid] event=dispatch name=%s type=%s ne11=%lld width=%lld pairs=%lld " + "n_experts=%lld top_k=%lld mmvq_max=%d path=%s\n", + dst->name, ggml_type_name(src0->type), (long long) ne11, (long long) ne2, + (long long) (ids->ne[0]*ne2), (long long) ne02, (long long) ids->ne[0], + mmvq_mmid_max, path); + } + }; // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { static_assert(MMVQ_MAX_BATCH_SIZE == MMVF_MAX_BATCH_SIZE); if (ne2 <= MMVQ_MAX_MOE_BATCH_SIZE) { if (ggml_is_quantized(src0->type)) { - const int mmvq_mmid_max = get_mmvq_mmid_max_batch(src0->type, cc); if (ne2 <= mmvq_mmid_max) { + log_dispatch("mmvq"); ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst); return; } } else { if (ne2 <= MMVF_MAX_BATCH_SIZE && GGML_CUDA_CC_IS_AMD(cc)) { + log_dispatch("mmvf"); ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst); return; } @@ -2584,16 +2601,20 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * } if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) { + log_dispatch("mmq"); ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst); return; } if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { + log_dispatch("mmf"); ggml_cuda_mul_mat_f(ctx, src0, src1, ids, dst); return; } } + log_dispatch("sync_fallback"); + // note: this path should not be reached when recording CUDA graphs, because it requires stream synchronization // TODO: add asserts to verify this. should work with CUDA, HIP, etc. cudaStream_t stream = ctx.stream(); @@ -3188,6 +3209,10 @@ static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { bool use_cuda_graph = true; + static const bool mmid_telemetry = []() { + const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); // Loop over nodes in GGML graph to obtain info needed for CUDA graph for (int i = 0; i < cgraph->n_nodes; i++) { @@ -3207,7 +3232,10 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (node->op == GGML_OP_MUL_MAT_ID) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const int mmvq_mmid_max = get_mmvq_mmid_max_batch(node->src[0]->type, cc); + // Non-quantized nodes never take the MMVQ path, so report a 0 ceiling + // instead of the type-independent value get_mmvq_mmid_max_batch returns. + const int mmvq_mmid_max = ggml_is_quantized(node->src[0]->type) + ? get_mmvq_mmid_max_batch(node->src[0]->type, cc) : 0; // Mirror ggml_cuda_mul_mat_id: the MMVQ and MMQ mul_mat_id paths // are stream-sync-free and safe to capture; only the sort-based // fallback requires a stream synchronize. @@ -3217,6 +3245,15 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { const bool mmid_mmq_ok = ggml_is_quantized(node->src[0]->type) && ggml_cuda_should_use_mmq(node->src[0]->type, cc, node->src[1]->ne[2], node->src[0]->ne[2]); + if (mmid_telemetry) { + std::fprintf(stderr, + "[dflash-mmid] event=graph name=%s type=%s ne11=%lld width=%lld " + "mmvq_max=%d mmvq_ok=%d mmq_ok=%d node_eligible=%d\n", + node->name, ggml_type_name(node->src[0]->type), + (long long) node->src[1]->ne[1], (long long) node->ne[2], + mmvq_mmid_max, mmid_mmvq_ok, mmid_mmq_ok, + mmid_mmvq_ok || mmid_mmq_ok); + } if (!mmid_mmvq_ok && !mmid_mmq_ok) { // under these conditions, the mul_mat_id operation will need to synchronize the stream, so we cannot use CUDA graphs // TODO: figure out a way to enable for larger batch sizes, without hurting performance @@ -3228,7 +3265,9 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { } } - if (!use_cuda_graph) { + // With telemetry on, keep scanning so event=graph is emitted for every + // MUL_MAT_ID node; the early exit is only a performance shortcut. + if (!use_cuda_graph && !mmid_telemetry) { break; } } diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu index ec6288f97..23effdc1e 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu @@ -4,7 +4,9 @@ #include "vecdotq.cuh" #include +#include #include +#include typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs); @@ -1642,6 +1644,10 @@ void ggml_cuda_mul_mat_vec_q( const int64_t stride_channel_y = ids ? s11 : s12; const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0; + static const bool mmid_telemetry = []() { + const char * value = std::getenv("DFLASH_MMID_TELEMETRY"); + return value != nullptr && std::strcmp(value, "0") != 0; + }(); // [TAG_MMID_GROUPED] grouped-expert path for small MUL_MAT_ID batches. if (ids && ncols_dst >= 2 && ncols_dst <= MMVQ_MAX_MOE_BATCH_SIZE && @@ -1682,10 +1688,50 @@ void ggml_cuda_mul_mat_vec_q( (int) s01, (int) stride_col_y, (int) stride_col_dst, (int) s02, (int) stride_channel_y, (int) stride_channel_dst, np, stream)) { + if (mmid_telemetry) { + std::fprintf(stderr, + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%d variant=grouped\n", + ggml_type_name(src0->type), (long long) ncols_dst, np); + } return; } } + if (mmid_telemetry && ids) { + if (ncols_dst < 2) { + // Single-token MUL_MAT_ID: the ordinary single-column MMVQ case, not + // the multi-token legacy MoE launch the grouped path falls back to. + std::fprintf(stderr, + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=single\n", + ggml_type_name(src0->type), (long long) ncols_dst, + (long long) (nchannels_dst*ncols_dst)); + } else { + // Name the kernel mul_mat_vec_q_switch_type will actually run for this + // ungrouped multi-token batch, so the label is not misreported when the + // tokenwise or generic MMVQ modes are selected by env. + static const bool tokenwise_mmid = []() { + const char * e = std::getenv("DFLASH_CUDA_MMVQ_MOE_TOKENWISE"); + return e && e[0] == '1' && e[1] == '\0'; + }(); + static const bool moe_kernel = []() { + const char * e = std::getenv("DFLASH_CUDA_MMVQ_MOE_KERNEL"); + return !(e && e[0] == '0' && e[1] == '\0'); + }(); + const char * variant = + tokenwise_mmid ? "tokenwise" : + (moe_kernel || ncols_dst > MMVQ_MAX_BATCH_SIZE) ? "moe" : "generic"; + const char * reason = + ncols_dst > MMVQ_MAX_MOE_BATCH_SIZE ? "width_gt_16" : + (int) (nchannels_dst*ncols_dst) > MMID_GROUPED_MAX_PAIRS ? "pairs_gt_256" : + !mmid_grouped_env() ? "flag_off" : + !mmid_grouped_type_ok(src0->type) ? "unsupported_type" : "dispatch_rejected"; + std::fprintf(stderr, + "[dflash-mmid] event=mmvq type=%s width=%lld pairs=%lld variant=%s reason=%s\n", + ggml_type_name(src0->type), (long long) ncols_dst, + (long long) (nchannels_dst*ncols_dst), variant, reason); + } + } + mul_mat_vec_q_switch_type( src0->data, src0->type, src1_q8_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index f711a920c..77d9dc77b 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -24,6 +24,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. | `DFLASH_ADAPTIVE_K_TAU` | 0 = off | Prefer the CLI: --adaptive-experts [tau]. Cumulative combine-weight threshold for per-token expert gating. | | `DFLASH_ADAPTIVE_K_DENSE` | per-model default | CSV of MoE layers kept dense under adaptive-K (DFlash capture layers). Warned-inert on families that do not thread layer indices yet. | | `DFLASH_MMID_GROUPED` | unset | Grouped MUL_MAT_ID kernel for small verify batches; candidate for CLI promotion. | +| `DFLASH_MMID_TELEMETRY` | unset | DEBUG: report MUL_MAT_ID dispatch, MMVQ variant, and per-node graph compatibility. | | `DFLASH_KVFLASH` | unset | Prefer the CLI: `--kvflash` (token count or `auto`). | ## Full inventory (generated) @@ -115,6 +116,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `DFLASH_LAGUNA_VERIFY_WIDTH` - laguna_backend.cpp - `DFLASH_LAGUNA_VERIFY_WIDTH_MAX` - laguna_backend.cpp - `DFLASH_MAX_CONTEXT` - laguna_backend.cpp, qwen35moe_backend.cpp +- `DFLASH_MMID_TELEMETRY` - ggml-cuda.cu, mmvq.cu - `DFLASH_MMQ_FULL_BATCH_MIN` - moe_hybrid_ffn_eval.cpp - `DFLASH_MMQ_SUB_BATCH` - moe_hybrid_ffn_eval.cpp - `DFLASH_MODEL_CARDS_DIR` - model_card.cpp