From 3c2e000a79214a7528c6d185da3c5364eb6febf1 Mon Sep 17 00:00:00 2001 From: dusterbloom <32869278+dusterbloom@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:06:53 +0200 Subject: [PATCH] fix(hip): release Q8 memo before pool teardown --- server/CMakeLists.txt | 13 ++++ .../llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu | 7 ++ server/test/test_cuda_pool_shutdown.cpp | 67 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 server/test/test_cuda_pool_shutdown.cpp diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index dacd851bb..94f2ebd7f 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -642,6 +642,19 @@ endif() option(DFLASH27B_TESTS "Build numerics tests" ON) option(DFLASH27B_SERVER "Build dflash_server + backend_ipc_daemon" ON) if(DFLASH27B_TESTS) + if((DFLASH27B_GPU_BACKEND STREQUAL "cuda" OR DFLASH27B_GPU_BACKEND STREQUAL "hip") + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_cuda_pool_shutdown.cpp") + add_executable(test_cuda_pool_shutdown test/test_cuda_pool_shutdown.cpp) + if(DFLASH27B_GPU_BACKEND STREQUAL "hip") + set_source_files_properties(test/test_cuda_pool_shutdown.cpp PROPERTIES LANGUAGE HIP) + set_target_properties(test_cuda_pool_shutdown PROPERTIES HIP_ARCHITECTURES "${_dflash_archs}") + endif() + target_include_directories(test_cuda_pool_shutdown PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include) + target_link_libraries(test_cuda_pool_shutdown PRIVATE + ${DFLASH27B_GGML_BACKEND_TARGET} ggml ggml-base) + add_test(NAME cuda_pool_shutdown COMMAND test_cuda_pool_shutdown) + endif() if(DFLASH27B_GPU_BACKEND STREQUAL "cuda" AND _dflash_cuda_min_sm GREATER_EQUAL 80 AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_flashprefill_kernels.cpp") add_executable(test_flashprefill_kernels test/test_flashprefill_kernels.cpp) set_target_properties(test_flashprefill_kernels PROPERTIES CUDA_ARCHITECTURES "${_dflash_archs}") diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu index 3aa2eb481..33f01a10a 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu @@ -690,6 +690,13 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { std::unique_lock lock(ggml_cuda_lock); ggml_cuda_lock_cv.wait(lock, []{ return ggml_cuda_lock_counter.load(std::memory_order_relaxed) == 0; }); + // Memo entries own allocations from pools that are destroyed before the + // memo vector by member destruction order. Release them while the pools + // are still alive, preserving the LIFO discipline required by VMM pools. + while (!luce_q8_memo.empty()) { + luce_q8_memo.pop_back(); + } + if (copy_event != nullptr) { CUDA_CHECK(cudaEventDestroy(copy_event)); } diff --git a/server/test/test_cuda_pool_shutdown.cpp b/server/test/test_cuda_pool_shutdown.cpp new file mode 100644 index 000000000..20ec4423c --- /dev/null +++ b/server/test/test_cuda_pool_shutdown.cpp @@ -0,0 +1,67 @@ +#include "ggml-backend.h" +#include "ggml-cuda.h" +#include "ggml.h" + +#include +#include +#include +#include + +int main() { +#if defined(_WIN32) + _putenv_s("LUCE_Q8_MEMO", "1"); +#else + setenv("LUCE_Q8_MEMO", "1", 1); +#endif + + ggml_backend_t backend = ggml_backend_cuda_init(0); + if (!backend) { + std::fprintf(stderr, "failed to initialize CUDA/HIP backend\n"); + return 1; + } + + ggml_init_params params{}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + if (!ctx) { + ggml_backend_free(backend); + return 1; + } + + constexpr int64_t k = 256; + constexpr int64_t n = 256; + ggml_tensor * weights = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, k, n); + ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, 1); + ggml_tensor * output = ggml_mul_mat(ctx, weights, input); + ggml_set_input(input); + ggml_set_output(output); + + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, output); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (!buffer) { + ggml_free(ctx); + ggml_backend_free(backend); + return 1; + } + + std::vector weights_data(ggml_nbytes(weights), 0); + std::vector input_data((size_t) k, 1.0f); + ggml_backend_tensor_set(weights, weights_data.data(), 0, weights_data.size()); + ggml_backend_tensor_set(input, input_data.data(), 0, input_data.size() * sizeof(float)); + + const ggml_status status = ggml_backend_graph_compute(backend, graph); + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + if (status != GGML_STATUS_SUCCESS) { + ggml_backend_free(backend); + return 1; + } + + // LUCE_Q8_MEMO intentionally retains the pool allocation after compute. + // Backend teardown must release that allocation before destroying its pool. + ggml_backend_free(backend); + return 0; +}