From fc5843f2f4fd0aa1447f7613299f265b70535015 Mon Sep 17 00:00:00 2001 From: xenshinu Date: Mon, 15 Jun 2026 01:56:29 +0000 Subject: [PATCH] [fix] add ep ipc fix, deepep works without nvl_bytes=0 Signed-off-by: xenshinu --- .gitignore | 5 +- csrc/hook.cpp | 569 +++++++++++++++++- docs/vllm/direct-edits.md | 2 + python/foundry/__init__.py | 2 + python/foundry/integration/vllm/hooks.py | 22 +- recipe/experimental/README.md | 129 ++++ recipe/experimental/foundry_load.toml | 9 + recipe/experimental/foundry_load_fp8.toml | 6 + recipe/experimental/foundry_save.toml | 9 + recipe/experimental/foundry_save_fp8.toml | 6 + .../experimental/serve_qwen3-30ba3b_ipc_ep.sh | 92 +++ .../serve_qwen3-30ba3bfp8_ipc_ep.sh | 69 +++ tests/run_deepep_matrix.sh | 75 +++ tests/test_deepep_fabric.py | 57 +- 14 files changed, 1008 insertions(+), 44 deletions(-) create mode 100644 recipe/experimental/README.md create mode 100644 recipe/experimental/foundry_load.toml create mode 100644 recipe/experimental/foundry_load_fp8.toml create mode 100644 recipe/experimental/foundry_save.toml create mode 100644 recipe/experimental/foundry_save_fp8.toml create mode 100755 recipe/experimental/serve_qwen3-30ba3b_ipc_ep.sh create mode 100755 recipe/experimental/serve_qwen3-30ba3bfp8_ipc_ep.sh create mode 100755 tests/run_deepep_matrix.sh diff --git a/.gitignore b/.gitignore index d958933..31d1f0a 100644 --- a/.gitignore +++ b/.gitignore @@ -362,4 +362,7 @@ marimo/_lsp/ __marimo__/ # Streamlit -.streamlit/secrets.toml \ No newline at end of file +.streamlit/secrets.toml +tests/deepep_matrix_work/ +deepep_fabric_archive/ +hook_archive/ diff --git a/csrc/hook.cpp b/csrc/hook.cpp index d552d08..a4d5350 100644 --- a/csrc/hook.cpp +++ b/csrc/hook.cpp @@ -14,12 +14,20 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -253,6 +261,24 @@ static std::once_flag default_allocation_region_flag; static boost::unordered::concurrent_flat_map global_alloc_metadata; static boost::unordered::concurrent_flat_map global_carved_reserve_metadata; +// Defined in the VMM-IPC section below (inside the extern "C" block, hence +// the matching linkage here); closes and forgets the exported fd for a VA +// when its allocation is freed (an exported fd keeps the physical pages +// alive and would otherwise serve stale memory to importers). +extern "C" { +static void vmm_ipc_invalidate_export(CUdeviceptr dptr); +} + +// Process-global mirror of the LOAD-mode preallocated chunk (the authoritative +// copy lives in tls_storage, which the VMM-IPC export path cannot rely on: +// cuIpcGetMemHandle may be called from a different thread than the one that +// preallocated). Set by preallocate_region, cleared by +// free_preallocated_region. Chunk carves have metadata.handle == 0, so +// exporting them means exporting THIS handle plus an offset. +static std::atomic g_prealloc_handle{0}; +static std::atomic g_prealloc_base{0}; +static std::atomic g_prealloc_size{0}; + struct HookAllocationEvent { enum class Type { Alloc, Free, Reserve }; Type type; @@ -2657,6 +2683,7 @@ CUresult cuMemFree_v2(CUdeviceptr dptr) { mem_addr_free_func(metadata.ptr, metadata.size); } + vmm_ipc_invalidate_export(dptr); global_alloc_metadata.erase_if( [dptr](const std::pair& kv) { return kv.first == dptr; }); @@ -2860,8 +2887,239 @@ void* dlsym(void* handle, const char* symbol) { // VMM allocations by translating to cuMemExportToShareableHandle API // ============================================================================= -// Magic marker to identify VMM IPC handles in CUipcMemHandle -static constexpr uint32_t VMM_IPC_MAGIC = 0x564D4D49; // "VMMI" +// Magic marker to identify VMM IPC handles in CUipcMemHandle. +// v2 ("VMM2"): the blob carries {magic, exporter pid, original ptr, aligned size}. +// The POSIX fd itself is transferred via SCM_RIGHTS over a per-process abstract +// unix socket (a raw fd integer is meaningless in another process - the v1 bug +// behind "cuMemImportFromShareableHandle failed with error 999"). +static constexpr uint32_t VMM_IPC_MAGIC = 0x564D4D32; // "VMM2" + +// --------------------------------------------------------------------------- +// VMM-IPC fd transport: each exporting process runs one server thread on an +// abstract unix socket "\0foundry-vmm-ipc.". Importers connect, send the +// 8-byte exporter VA they want, and receive the exported fd via SCM_RIGHTS. +// The server does no CUDA calls: fds are exported on the cuIpcGetMemHandle +// caller's thread and parked in vmm_ipc_exported_fds. +// --------------------------------------------------------------------------- + +// exporter VA -> (allocation handle it was exported from, fd). The handle is +// kept so VA reuse after free/realloc invalidates the cached fd. +static boost::unordered::concurrent_flat_map> + vmm_ipc_exported_fds; +static std::mutex vmm_ipc_server_mutex; +static int vmm_ipc_listen_fd = -1; +// pid that owns the running server thread. fork() copies this .so's state but +// not threads, so a forked child must rebind its own socket (vLLM's default +// worker start method is fork). +static pid_t vmm_ipc_server_pid = -1; +// Per-process random token carried in the handle blob and verified by the +// server: defends against pid reuse (a recycled pid + the deterministic +// shared-base VAs would otherwise let an importer silently fetch a different +// process's allocation). +static uint64_t vmm_ipc_token = 0; + +// Wire format of a fetch request. +struct VmmIpcRequest { + uint64_t ptr; + uint64_t token; +}; + +static void vmm_ipc_set_timeouts(int fd) { + struct timeval tv = {30, 0}; + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); +} + +// Imported whole-chunk mappings (LOAD-mode exports: a buffer carved from the +// peer's preallocated chunk is shared by exporting the chunk handle plus an +// offset; cuMemMap cannot map a sub-range of an imported handle, so we map +// the entire peer chunk once and hand out interior pointers). Keyed by +// (exporter pid, exporter chunk base). Refcounted by open/close calls. +struct VmmIpcChunkMapping { + CUdeviceptr local_base; + size_t size; + CUmemGenericAllocationHandle handle; + int refcount; +}; +static std::mutex vmm_ipc_chunk_mutex; +static std::map, VmmIpcChunkMapping> vmm_ipc_chunk_mappings; +// interior pointer handed to a caller -> owning chunk key (for close) +static std::map> vmm_ipc_chunk_subptrs; + +// Dedicated VA zone for relocated peer imports, far below the foundry region +// (default 0x600000000000) and the large-reserve zone right above region end. +// Without this, the driver tends to place a relocated import immediately +// after the region reservation - exactly where the hooked large-reserve hint +// sends the NVSHMEM symmetric heap next, which silently breaks the heap's +// SAVE/LOAD address determinism (observed: heap displaced by a peer-chunk +// import landing at region_end). +static std::atomic vmm_ipc_import_hint{0x300000000000ULL}; + +static socklen_t vmm_ipc_socket_addr(pid_t pid, struct sockaddr_un* addr) { + memset(addr, 0, sizeof(*addr)); + addr->sun_family = AF_UNIX; + // Abstract namespace (sun_path[0] == '\0'): no filesystem entry, vanishes + // with the process. + int n = snprintf(addr->sun_path + 1, sizeof(addr->sun_path) - 2, "foundry-vmm-ipc.%d", (int)pid); + return (socklen_t)(offsetof(struct sockaddr_un, sun_path) + 1 + n); +} + +static void vmm_ipc_server_loop(int listen_fd) { + while (true) { + int conn = accept(listen_fd, nullptr, nullptr); + if (conn < 0) { + if (errno == EBADF || errno == EINVAL) + break; // socket gone + continue; // EINTR/ECONNABORTED/EMFILE/... are transient + } + vmm_ipc_set_timeouts(conn); + // Abstract sockets have no filesystem permissions: only serve same-uid peers. + struct ucred cred; + socklen_t cred_len = sizeof(cred); + if (getsockopt(conn, SOL_SOCKET, SO_PEERCRED, &cred, &cred_len) != 0 || cred.uid != getuid()) { + close(conn); + continue; + } + VmmIpcRequest req = {}; + int fd = -1; + if (recv(conn, &req, sizeof(req), MSG_WAITALL) == (ssize_t)sizeof(req) && + req.token == vmm_ipc_token) { + // Dup under the map lock: the export path may concurrently close and + // replace the parked fd (stale-entry refresh); the dup we send is ours. + vmm_ipc_exported_fds.visit( + (CUdeviceptr)req.ptr, + [&](const std::pair>& + kv) { fd = fcntl(kv.second.second, F_DUPFD_CLOEXEC, 0); }); + } + char status = (fd >= 0) ? 0 : 1; + struct iovec iov = {&status, 1}; + char cbuf[CMSG_SPACE(sizeof(int))] = {}; + struct msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + if (fd >= 0) { + msg.msg_control = cbuf; + msg.msg_controllen = CMSG_SPACE(sizeof(int)); + struct cmsghdr* c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(c), &fd, sizeof(int)); + } + // MSG_NOSIGNAL: a peer dying mid-handshake must not SIGPIPE-kill us. + if (sendmsg(conn, &msg, MSG_NOSIGNAL) != 1) { + fprintf(stderr, "[HOOK] WARNING: VMM-IPC fd server sendmsg failed (errno=%d)\n", errno); + } + if (fd >= 0) + close(fd); + close(conn); + } +} + +static bool vmm_ipc_ensure_server() { + std::lock_guard lock(vmm_ipc_server_mutex); + pid_t pid = getpid(); + if (vmm_ipc_server_pid == pid) { + return vmm_ipc_listen_fd >= 0; + } + // First call in this process, or first after fork (the parent's accept + // thread did not survive). Close any inherited listen fd so the parent's + // abstract name is not pinned alive by us after the parent exits. + if (vmm_ipc_listen_fd >= 0) { + close(vmm_ipc_listen_fd); + vmm_ipc_listen_fd = -1; + } + int s = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + struct sockaddr_un addr; + socklen_t len = vmm_ipc_socket_addr(pid, &addr); + if (s < 0 || bind(s, (struct sockaddr*)&addr, len) != 0 || listen(s, 64) != 0) { + fprintf(stderr, "[HOOK] ERROR: VMM-IPC fd server setup failed (errno=%d)\n", errno); + if (s >= 0) + close(s); + vmm_ipc_listen_fd = -1; + vmm_ipc_server_pid = pid; // don't retry-spam; exports in this process fail loudly + return false; + } + // Per-process token (re-derived after fork). /dev/urandom, with a clock^pid + // fallback - this is anti-accident (pid reuse), not a security boundary; + // same-uid access control is SO_PEERCRED above. + uint64_t tok = 0; + int ur = open("/dev/urandom", O_RDONLY | O_CLOEXEC); + if (ur >= 0) { + if (read(ur, &tok, sizeof(tok)) != (ssize_t)sizeof(tok)) + tok = 0; + close(ur); + } + if (tok == 0) { + struct timeval tv; + gettimeofday(&tv, nullptr); + tok = ((uint64_t)tv.tv_sec << 32) ^ (uint64_t)tv.tv_usec ^ ((uint64_t)pid << 16) ^ + 0x9e3779b97f4a7c15ULL; + } + vmm_ipc_token = tok; + vmm_ipc_listen_fd = s; + vmm_ipc_server_pid = pid; + std::thread(vmm_ipc_server_loop, s).detach(); + return true; +} + +// Importer side: fetch the fd for `original_ptr` from `exporter_pid`'s server. +// Returns a live fd in THIS process, or -1. +static int vmm_ipc_fetch_fd(pid_t exporter_pid, uint64_t token, CUdeviceptr original_ptr) { + int s = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + if (s < 0) + return -1; + vmm_ipc_set_timeouts(s); + struct sockaddr_un addr; + socklen_t len = vmm_ipc_socket_addr(exporter_pid, &addr); + if (connect(s, (struct sockaddr*)&addr, len) != 0) { + fprintf(stderr, + "[HOOK] ERROR: VMM-IPC connect to exporter pid %d failed (errno=%d) - exporter dead " + "or hook version mismatch\n", + (int)exporter_pid, errno); + close(s); + return -1; + } + VmmIpcRequest req = {(uint64_t)original_ptr, token}; + if (send(s, &req, sizeof(req), MSG_NOSIGNAL) != (ssize_t)sizeof(req)) { + close(s); + return -1; + } + char status = 1; + struct iovec iov = {&status, 1}; + char cbuf[CMSG_SPACE(sizeof(int))] = {}; + struct msghdr msg = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = cbuf; + msg.msg_controllen = sizeof(cbuf); + ssize_t r = recvmsg(s, &msg, MSG_CMSG_CLOEXEC); + close(s); + if (r != 1 || status != 0) + return -1; + for (struct cmsghdr* c = CMSG_FIRSTHDR(&msg); c != nullptr; c = CMSG_NXTHDR(&msg, c)) { + if (c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS) { + int fd = -1; + memcpy(&fd, CMSG_DATA(c), sizeof(int)); + return fd; + } + } + return -1; +} + +// Close and forget the parked exported fd for a VA (called from the free +// path). Without this, the fd keeps the freed allocation's physical pages +// alive and the server would hand importers a stale mapping. +static void vmm_ipc_invalidate_export(CUdeviceptr dptr) { + vmm_ipc_exported_fds.erase_if( + [dptr](const std::pair>& kv) { + if (kv.first != dptr) + return false; + close(kv.second.second); + return true; + }); +} // Hook for cuIpcGetMemHandle - intercept Driver API to support VMM allocations CUresult cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) { @@ -2878,7 +3136,37 @@ CUresult cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) { metadata = kv.second; }); - if (found && metadata.handle != 0) { + // Decide what to export: the allocation's own handle (SAVE-mode slow-path + // allocs), or the whole preallocated chunk plus an offset (LOAD-mode fast + // path carves, which have no individual handle). cuMemMap cannot map a + // sub-range of an imported handle, so chunk carves are shared by exporting + // the chunk handle; the importer maps the entire chunk and returns an + // interior pointer. + CUmemGenericAllocationHandle export_handle = 0; + CUdeviceptr export_key = dptr; // fd-registry key == blob lookup key + uint64_t chunk_base = 0; + uint64_t chunk_size = 0; + if (found) { + export_handle = metadata.handle; + if (export_handle == 0 && metadata.from_preallocation) { + chunk_base = g_prealloc_base.load(); + chunk_size = g_prealloc_size.load(); + export_handle = (CUmemGenericAllocationHandle)g_prealloc_handle.load(); + export_key = (CUdeviceptr)chunk_base; + if (export_handle == 0 || dptr < chunk_base || dptr >= chunk_base + chunk_size) { + fprintf(stderr, + "[HOOK] ERROR: cuIpcGetMemHandle: carved ptr=0x%llx outside live preallocated " + "chunk [0x%llx, +%llu) - cannot export\n", + (unsigned long long)dptr, (unsigned long long)chunk_base, + (unsigned long long)chunk_size); + export_handle = 0; + chunk_base = 0; + chunk_size = 0; + } + } + } + + if (export_handle != 0) { // This is a VMM allocation - export via shareable handle typedef CUresult (*cuMemExportToShareableHandle_t)( void*, CUmemGenericAllocationHandle, CUmemAllocationHandleType, unsigned long long); @@ -2890,32 +3178,79 @@ CUresult cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) { return CUDA_ERROR_NOT_SUPPORTED; } - int fd = -1; - CUresult result = - export_func(&fd, metadata.handle, CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, 0); + // The server also owns the per-process token packed into the blob, so it + // must exist before we hand out any handle. + if (!vmm_ipc_ensure_server()) { + return CUDA_ERROR_NOT_SUPPORTED; + } - if (result != CUDA_SUCCESS) { - fprintf(stderr, - "[HOOK] ERROR: cuMemExportToShareableHandle failed with error %d for ptr=0x%llx\n", - result, (unsigned long long)dptr); - return result; + // Export once per allocation and park the fd for the server thread. + // The cached entry is keyed by VA and validated against the allocation + // handle, so VA reuse after free/realloc re-exports instead of serving a + // stale fd (the free path also eagerly invalidates via + // vmm_ipc_invalidate_export). + int fd = -1; + vmm_ipc_exported_fds.visit( + export_key, + [&](const std::pair>& kv) { + if (kv.second.first == export_handle) + fd = kv.second.second; + }); + if (fd < 0) { + int new_fd = -1; + CUresult result = + export_func(&new_fd, export_handle, CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, 0); + if (result != CUDA_SUCCESS) { + fprintf(stderr, + "[HOOK] ERROR: cuMemExportToShareableHandle failed with error %d for ptr=0x%llx\n", + result, (unsigned long long)dptr); + return result; + } + // Parked fds must not leak into forked children (each inherited copy + // pins the allocation's physical memory). SCM_RIGHTS transfer is + // unaffected by CLOEXEC. + fcntl(new_fd, F_SETFD, FD_CLOEXEC); + vmm_ipc_exported_fds.insert_or_visit( + std::make_pair(export_key, std::make_pair(export_handle, new_fd)), + [&](std::pair>& kv) { + // Entry exists: either a racing thread won (same handle - drop + // ours) or it is stale from a freed allocation (replace). + if (kv.second.first == export_handle) { + close(new_fd); + new_fd = kv.second.second; + } else { + close(kv.second.second); + kv.second = std::make_pair(export_handle, new_fd); + } + }); + fd = new_fd; } - // Pack our custom data into the handle structure - // CUipcMemHandle has 64 reserved bytes - we use them to store our info: - // - Magic marker (4 bytes) to identify VMM handles - // - File descriptor (4 bytes) - // - Original pointer address (8 bytes) - critical for CUDA graph replay! - // - Size (8 bytes) + // Pack our custom data into the handle structure. + // CUipcMemHandle has 64 reserved bytes: + // - Magic marker (4 bytes, "VMM2") + // - Exporter pid (4 bytes) - importer fetches the fd from this process's + // VMM-IPC socket via SCM_RIGHTS (a raw fd integer is not portable) + // - Original pointer address (8 bytes) - fd lookup key + placement hint + // - Aligned size (8 bytes) + // - Per-process token (8 bytes) - server rejects mismatches (pid reuse) + // - Chunk base + chunk size (8+8 bytes) - nonzero iff the pointer is a + // carve from the preallocated chunk; the importer then maps the whole + // chunk and returns base + (ptr - chunk_base) memset(pHandle, 0, sizeof(CUipcMemHandle)); + uint32_t pid_u32 = (uint32_t)getpid(); memcpy(pHandle->reserved, &VMM_IPC_MAGIC, sizeof(uint32_t)); - memcpy(pHandle->reserved + 4, &fd, sizeof(int)); + memcpy(pHandle->reserved + 4, &pid_u32, sizeof(uint32_t)); memcpy(pHandle->reserved + 8, &dptr, sizeof(CUdeviceptr)); memcpy(pHandle->reserved + 16, &metadata.size, sizeof(size_t)); + memcpy(pHandle->reserved + 24, &vmm_ipc_token, sizeof(uint64_t)); + memcpy(pHandle->reserved + 32, &chunk_base, sizeof(uint64_t)); + memcpy(pHandle->reserved + 40, &chunk_size, sizeof(uint64_t)); #ifdef HOOK_DEBUG - fprintf(stderr, "[HOOK] cuIpcGetMemHandle: VMM ptr=0x%llx, fd=%d, size=%zu\n", + fprintf(stderr, + "[HOOK] cuIpcGetMemHandle: VMM ptr=0x%llx, fd=%d (served via socket), size=%zu\n", (unsigned long long)dptr, fd, metadata.size); #endif return CUDA_SUCCESS; @@ -2940,19 +3275,61 @@ CUresult cuIpcOpenMemHandle(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned if (magic == VMM_IPC_MAGIC) { // This is a VMM IPC handle - extract the packed data - int fd; + uint32_t exporter_pid_u32; CUdeviceptr original_ptr; size_t size; + uint64_t token; + uint64_t chunk_base; + uint64_t chunk_size; - memcpy(&fd, handle.reserved + 4, sizeof(int)); + memcpy(&exporter_pid_u32, handle.reserved + 4, sizeof(uint32_t)); memcpy(&original_ptr, handle.reserved + 8, sizeof(CUdeviceptr)); memcpy(&size, handle.reserved + 16, sizeof(size_t)); + memcpy(&token, handle.reserved + 24, sizeof(uint64_t)); + memcpy(&chunk_base, handle.reserved + 32, sizeof(uint64_t)); + memcpy(&chunk_size, handle.reserved + 40, sizeof(uint64_t)); + pid_t exporter_pid = (pid_t)exporter_pid_u32; + // For chunk carves the fd registry on the exporter is keyed by the chunk + // base, and what we import/map is the whole chunk. + CUdeviceptr fetch_key = (chunk_base != 0) ? (CUdeviceptr)chunk_base : original_ptr; + size_t map_size = (chunk_base != 0) ? (size_t)chunk_size : size; + + if (chunk_base != 0) { + // Fast path: peer chunk already mapped -> hand out an interior pointer. + std::lock_guard lock(vmm_ipc_chunk_mutex); + auto it = vmm_ipc_chunk_mappings.find({exporter_pid, chunk_base}); + if (it != vmm_ipc_chunk_mappings.end()) { + it->second.refcount++; + CUdeviceptr mapped = it->second.local_base + (original_ptr - (CUdeviceptr)chunk_base); + vmm_ipc_chunk_subptrs[mapped] = {exporter_pid, chunk_base}; + *pdptr = mapped; + return CUDA_SUCCESS; + } + } #ifdef HOOK_DEBUG - fprintf(stderr, "[HOOK] cuIpcOpenMemHandle: VMM fd=%d, original_ptr=0x%llx, size=%zu\n", fd, - (unsigned long long)original_ptr, size); + fprintf(stderr, + "[HOOK] cuIpcOpenMemHandle: VMM exporter_pid=%d, original_ptr=0x%llx, size=%zu\n", + (int)exporter_pid, (unsigned long long)original_ptr, size); #endif + // Obtain a live fd in THIS process: dup from our own registry for + // same-process opens, SCM_RIGHTS fetch from the exporter otherwise. + int fd = -1; + if (exporter_pid == getpid() && token == vmm_ipc_token) { + vmm_ipc_exported_fds.visit( + fetch_key, + [&](const std::pair>& + kv) { fd = fcntl(kv.second.second, F_DUPFD_CLOEXEC, 0); }); + } else { + fd = vmm_ipc_fetch_fd(exporter_pid, token, fetch_key); + } + if (fd < 0) { + fprintf(stderr, "[HOOK] ERROR: VMM-IPC could not obtain fd for ptr=0x%llx from pid %d\n", + (unsigned long long)fetch_key, (int)exporter_pid); + return CUDA_ERROR_INVALID_VALUE; + } + // Import the allocation handle from file descriptor typedef CUresult (*cuMemImportFromShareableHandle_t)(CUmemGenericAllocationHandle*, void*, CUmemAllocationHandleType); @@ -2961,12 +3338,14 @@ CUresult cuIpcOpenMemHandle(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned if (import_func == nullptr) { fprintf(stderr, "[HOOK] ERROR: cuMemImportFromShareableHandle not found\n"); + close(fd); return CUDA_ERROR_NOT_SUPPORTED; } CUmemGenericAllocationHandle imported_handle; CUresult result = import_func(&imported_handle, (void*)(intptr_t)fd, CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR); + close(fd); // the driver does not take ownership of our fd copy if (result != CUDA_SUCCESS) { fprintf(stderr, "[HOOK] ERROR: cuMemImportFromShareableHandle failed with error %d\n", @@ -2974,16 +3353,58 @@ CUresult cuIpcOpenMemHandle(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned return result; } - // Map to the SAME address as original (critical for CUDA graph replay!) + typedef CUresult (*cuMemAddressReserve_t)(CUdeviceptr*, size_t, size_t, CUdeviceptr, + unsigned long long); + auto real_reserve_func = (cuMemAddressReserve_t)CUDA_DRIVER_CALL( + cuda_driver_entry_table, CUDA_ENTRY_cuMemAddressReserve); + typedef CUresult (*cuMemRelease_t)(CUmemGenericAllocationHandle); + auto mem_release_func = + (cuMemRelease_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemRelease); + typedef CUresult (*cuMemAddressFree_t)(CUdeviceptr, size_t); + auto real_address_free_func = + (cuMemAddressFree_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemAddressFree); + + // Reserve our own VA range for the peer mapping (real driver call, NOT the + // hooked wrapper - peer imports must never advance the deterministic + // cursor). The exporter's VA is passed as a hint: with per-rank disjoint + // regions it is honored and the mapping is address-stable; with today's + // shared-base symmetric layouts that range is occupied by our own region + // reservation, the driver picks elsewhere, and the caller gets a valid but + // relocated peer pointer. That is correct for table-indirect consumers + // (DeepEP buffer_ptrs_gpu, custom-allreduce RankData contents): peer VAs + // are never baked into captured graph nodes on those paths. + // First preference: the exporter's own VA (address-stable whenever it is + // locally free, e.g. future per-rank disjoint-base configs). + CUdeviceptr mapped_ptr = 0; + result = real_reserve_func(&mapped_ptr, map_size, 0, fetch_key, 0); + if (result == CUDA_SUCCESS && mapped_ptr != fetch_key) { + // Hint not honored. Do NOT keep the driver's fallback choice - it tends + // to sit immediately after existing reservations, i.e. on the NVSHMEM + // heap hint. Re-reserve inside the dedicated import zone. + real_address_free_func(mapped_ptr, map_size); + uint64_t zone_hint = + vmm_ipc_import_hint.fetch_add((map_size + ((1ULL << 30) - 1)) & ~((1ULL << 30) - 1)); + mapped_ptr = 0; + result = real_reserve_func(&mapped_ptr, map_size, 0, (CUdeviceptr)zone_hint, 0); + } + if (result != CUDA_SUCCESS) { + fprintf(stderr, "[HOOK] ERROR: cuMemAddressReserve for IPC import failed with error %d\n", + result); + mem_release_func(imported_handle); + return result; + } + typedef CUresult (*cuMemMap_t)(CUdeviceptr, size_t, size_t, CUmemGenericAllocationHandle, unsigned long long); auto map_func = (cuMemMap_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemMap); - result = map_func(original_ptr, size, 0, imported_handle, 0); + result = map_func(mapped_ptr, map_size, 0, imported_handle, 0); if (result != CUDA_SUCCESS) { fprintf(stderr, "[HOOK] ERROR: cuMemMap for IPC failed with error %d at addr=0x%llx size=%zu\n", - result, (unsigned long long)original_ptr, size); + result, (unsigned long long)mapped_ptr, map_size); + real_address_free_func(mapped_ptr, map_size); + mem_release_func(imported_handle); return result; } @@ -3003,27 +3424,67 @@ CUresult cuIpcOpenMemHandle(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned auto set_access_func = (cuMemSetAccess_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemSetAccess); - result = set_access_func(original_ptr, size, &accessDesc, 1); + result = set_access_func(mapped_ptr, map_size, &accessDesc, 1); if (result != CUDA_SUCCESS) { fprintf(stderr, "[HOOK] ERROR: cuMemSetAccess for IPC failed with error %d\n", result); + typedef CUresult (*cuMemUnmap_t)(CUdeviceptr, size_t); + auto mem_unmap_func = + (cuMemUnmap_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemUnmap); + mem_unmap_func(mapped_ptr, map_size); + real_address_free_func(mapped_ptr, map_size); + mem_release_func(imported_handle); return result; } - *pdptr = original_ptr; + if (mapped_ptr != fetch_key) { + fprintf(stderr, + "[HOOK] INFO: VMM-IPC import relocated: exporter VA 0x%llx -> local VA 0x%llx " + "(expected with shared region bases; peer tables must be refreshed by the caller)\n", + (unsigned long long)fetch_key, (unsigned long long)mapped_ptr); + } + + if (chunk_base != 0) { + // Register the whole-chunk mapping and hand out the interior pointer. + CUdeviceptr interior = mapped_ptr + (original_ptr - (CUdeviceptr)chunk_base); + std::lock_guard lock(vmm_ipc_chunk_mutex); + auto key = std::make_pair(exporter_pid, chunk_base); + auto it = vmm_ipc_chunk_mappings.find(key); + if (it != vmm_ipc_chunk_mappings.end()) { + // Lost a race to a concurrent open of the same peer chunk: keep the + // winner's mapping, drop ours. + typedef CUresult (*cuMemUnmap_t)(CUdeviceptr, size_t); + auto mem_unmap_func = + (cuMemUnmap_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemUnmap); + mem_unmap_func(mapped_ptr, map_size); + real_address_free_func(mapped_ptr, map_size); + mem_release_func(imported_handle); + it->second.refcount++; + interior = it->second.local_base + (original_ptr - (CUdeviceptr)chunk_base); + } else { + vmm_ipc_chunk_mappings[key] = VmmIpcChunkMapping{mapped_ptr, map_size, imported_handle, 1}; + } + vmm_ipc_chunk_subptrs[interior] = key; + *pdptr = interior; + return CUDA_SUCCESS; + } - // Track this imported allocation + *pdptr = mapped_ptr; + + // Track this imported allocation (close path unmaps, releases, and frees + // the reservation we created above) AllocMetadata alloc_metadata; - alloc_metadata.ptr = original_ptr; + alloc_metadata.ptr = mapped_ptr; alloc_metadata.size = size; alloc_metadata.handle = imported_handle; alloc_metadata.region_base = 0; // region_base == 0 indicates IPC-imported alloc_metadata.from_preallocation = false; - global_alloc_metadata.emplace(original_ptr, alloc_metadata); + global_alloc_metadata.emplace(mapped_ptr, alloc_metadata); #ifdef HOOK_DEBUG fprintf(stderr, - "[HOOK] cuIpcOpenMemHandle: Successfully mapped VMM fd=%d -> ptr=0x%llx, size=%zu\n", - fd, (unsigned long long)*pdptr, size); + "[HOOK] cuIpcOpenMemHandle: Successfully mapped exporter ptr=0x%llx -> ptr=0x%llx, " + "size=%zu\n", + (unsigned long long)original_ptr, (unsigned long long)*pdptr, size); #endif return CUDA_SUCCESS; } @@ -3041,6 +3502,34 @@ CUresult cuIpcCloseMemHandle(CUdeviceptr dptr) { auto real_func = (cuIpcCloseMemHandle_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuIpcCloseMemHandle); + // Interior pointer into an imported peer chunk? Refcounted: the chunk + // mapping is torn down when its last interior pointer closes. + { + std::lock_guard lock(vmm_ipc_chunk_mutex); + auto sub_it = vmm_ipc_chunk_subptrs.find(dptr); + if (sub_it != vmm_ipc_chunk_subptrs.end()) { + auto key = sub_it->second; + vmm_ipc_chunk_subptrs.erase(sub_it); + auto it = vmm_ipc_chunk_mappings.find(key); + if (it != vmm_ipc_chunk_mappings.end() && --it->second.refcount == 0) { + typedef CUresult (*cuMemUnmap_t)(CUdeviceptr, size_t); + auto mem_unmap_func = + (cuMemUnmap_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemUnmap); + typedef CUresult (*cuMemRelease_t)(CUmemGenericAllocationHandle); + auto mem_release_func = + (cuMemRelease_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemRelease); + typedef CUresult (*cuMemAddressFree_t)(CUdeviceptr, size_t); + auto addr_free_func = (cuMemAddressFree_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, + CUDA_ENTRY_cuMemAddressFree); + mem_unmap_func(it->second.local_base, it->second.size); + mem_release_func(it->second.handle); + addr_free_func(it->second.local_base, it->second.size); + vmm_ipc_chunk_mappings.erase(it); + } + return CUDA_SUCCESS; + } + } + AllocMetadata metadata; bool found = false; @@ -3059,8 +3548,14 @@ CUresult cuIpcCloseMemHandle(CUdeviceptr dptr) { auto mem_release_func = (cuMemRelease_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemRelease); + typedef CUresult (*cuMemAddressFree_t)(CUdeviceptr, size_t); + auto real_address_free_func = + (cuMemAddressFree_t)CUDA_DRIVER_CALL(cuda_driver_entry_table, CUDA_ENTRY_cuMemAddressFree); + mem_unmap_func(dptr, metadata.size); mem_release_func(metadata.handle); + // The import path owns a dedicated VA reservation for this mapping. + real_address_free_func(dptr, metadata.size); global_alloc_metadata.erase_if( [dptr](const std::pair& kv) { return kv.first == dptr; }); @@ -3268,6 +3763,9 @@ bool preallocate_region(size_t size) { tls_storage.preallocated_start_addr = target_addr; tls_storage.preallocated_end_addr = target_addr + aligned_size; tls_storage.has_preallocation = true; + g_prealloc_handle.store((unsigned long long)allocHandle); + g_prealloc_base.store((uint64_t)target_addr); + g_prealloc_size.store((uint64_t)aligned_size); // Note: we do NOT advance current_alloc_base_addr here. // The alloc calls will advance it as they consume the preallocated memory. @@ -3293,6 +3791,11 @@ void free_preallocated_region() { mem_unmap_func(tls_storage.preallocated_start_addr, preallocated_size); mem_release_func(tls_storage.preallocated_handle); + vmm_ipc_invalidate_export((CUdeviceptr)tls_storage.preallocated_start_addr); + g_prealloc_handle.store(0); + g_prealloc_base.store(0); + g_prealloc_size.store(0); + tls_storage.preallocated_handle = 0; tls_storage.preallocated_start_addr = 0; tls_storage.preallocated_end_addr = 0; diff --git a/docs/vllm/direct-edits.md b/docs/vllm/direct-edits.md index a1e63f1..a18cf1e 100644 --- a/docs/vllm/direct-edits.md +++ b/docs/vllm/direct-edits.md @@ -55,6 +55,8 @@ class CompilationConfig: This is the earliest activation point in the parent process. As soon as a `CompilationConfig` is fully constructed, foundry's runtime patches are in place. +> Note: vLLM folds `graph_extension_config_path` into `CompilationConfig.compute_hash()` (the torch.compile cache key) even though it never affects codegen. This is harmless **as long as every SAVE/LOAD phase passes the same path string** — run all phases from one canonical CWD (or always an absolute path). If pass 1 and pass 2 see the path via different mount aliases (`/home/...` vs `/data/...`), pass 2 misses pass 1's warm cache and inductor recompiles inside the cuda-graph capture window → `cudaErrorStreamCaptureInvalidated`. We keep vLLM unpatched and rely on consistent invocation instead; the EP recipe scripts are launched from the workspace root. (An optional `ignored_factors` exclusion would harden this at the source but is not applied.) + ## 3. `vllm/v1/engine/core.py` (~6 lines) ```python diff --git a/python/foundry/__init__.py b/python/foundry/__init__.py index bb88e00..226db5e 100644 --- a/python/foundry/__init__.py +++ b/python/foundry/__init__.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the Foundry project +# The ops wildcard must come FIRST: .graph's CUDAGraph (and the +# allocation_region helpers) intentionally override the raw pybind names. from .allocation_region import ( allocation_region, free_preallocated_region, diff --git a/python/foundry/integration/vllm/hooks.py b/python/foundry/integration/vllm/hooks.py index f776863..b330663 100644 --- a/python/foundry/integration/vllm/hooks.py +++ b/python/foundry/integration/vllm/hooks.py @@ -695,12 +695,22 @@ def _patch_deepep() -> None: def patched(self, *args, **kwargs): out = orig(self, *args, **kwargs) if isinstance(out, dict) and get_graph_extension_mode() != CUDAGraphExtensionMode.NONE: - # Force VMM-fabric buffers - # so the foundry cuMem hook tracks them, and disable the - # NVLink buffer (LL-mode uses RDMA) to avoid fabric-cuMemCreate - # collisions with preallocated region. - out["use_fabric"] = True - out["num_nvl_bytes"] = 0 + if os.environ.get("FOUNDRY_DEEPEP_NVL_IPC", "0") == "1": + # Keep upstream's NVLink buffer (legacy cudaIpc sharing). The + # hook's VMM-IPC layer transports the fds via SCM_RIGHTS and + # maps peers at relocated VAs, which DeepEP absorbs through + # its buffer_ptrs_gpu device table - peer VAs are never baked + # into captured graphs. use_fabric stays off: fabric + # cuMemCreate needs IMEX, and exercising the IPC path is the + # point of this mode. + out["use_fabric"] = False + else: + # Default: force VMM-fabric buffers + # so the foundry cuMem hook tracks them, and disable the + # NVLink buffer (LL-mode uses RDMA) to avoid fabric-cuMemCreate + # collisions with preallocated region. + out["use_fabric"] = True + out["num_nvl_bytes"] = 0 return out cls._make_all2all_kwargs = patched diff --git a/recipe/experimental/README.md b/recipe/experimental/README.md new file mode 100644 index 0000000..d0ccada --- /dev/null +++ b/recipe/experimental/README.md @@ -0,0 +1,129 @@ +# Foundry recipe — experimental (DeepEP legacy CUDA-IPC NVL buffer) + +End-to-end SAVE / LOAD recipe for **Qwen3-30B-A3B** expert-parallel where DeepEP's +intranode NVLink buffer stays on the **legacy CUDA-IPC** path +(`cudaIpcGetMemHandle` / `cudaIpcOpenMemHandle`) under foundry, instead of the +default fabric/NVSHMEM-only path used by `recipe/vllm/serve_qwen3-30ba3b_ep.sh`. + +This exercises foundry's **VMM-IPC translation layer**: DeepEP's NVL buffer is a +foundry VMM (`cuMemCreate`) allocation that legacy IPC can't share, so the hook +exports it as a POSIX fd and transports that fd to the peer rank via +`SCM_RIGHTS` over a per-process unix socket, then maps it into the importer's +own VA range. This is the path that lets foundry SAVE/LOAD work on machines +**without fabric / IMEX**, where DeepEP would otherwise fail at `Buffer.sync` +with `cuMemImportFromShareableHandle ... error 999`. + +``` +recipe/experimental/ +├── README.md # this file +├── foundry_save.toml # SAVE config (workspace_root = "foundry_archive_ipc") +├── foundry_load.toml # LOAD config (same workspace_root) +└── serve_qwen3-30ba3b_ipc_ep.sh # Qwen3-30B-A3B (MoE) EP, DeepEP NVL/IPC +``` + +It is the standard `recipe/vllm` EP recipe plus one switch — `FOUNDRY_DEEPEP_NVL_IPC=1` +— so read [`../vllm/README.md`](../vllm/README.md) first for installation, the +two-pass SAVE workflow, the archive layout, and the shared EP flags. Only the +IPC-specific deltas are documented here. + +## Required code + +The IPC path needs only **Foundry** changes beyond the standard install — no +vLLM edit: + +- **Foundry** — `foundry/csrc/hook.cpp`: the SCM_RIGHTS VMM-IPC fd transport + + whole-chunk peer mapping (handles LOAD-mode buffers carved from the + preallocated chunk). **C++ change → rebuild required:** + `uv pip install -e . --no-build-isolation` in `foundry/`. +- **Foundry** — `foundry/python/foundry/integration/vllm/hooks.py`: the + `FOUNDRY_DEEPEP_NVL_IPC` env knob in `_patch_deepep` (Python, no rebuild). + +### Run all phases from one consistent path (no vLLM edit needed) + +vLLM folds the foundry TOML path (`graph_extension_config_path`) into its +torch.compile cache key even though the path never affects codegen. If SAVE +pass 1 and pass 2 see that path with *different spellings* — e.g. two mount +aliases of the same dir (`/data/...` vs `/home/...`), or you move the TOML +between passes — pass 2 misses pass 1's warm compile cache and inductor +recompiles **inside** the cuda-graph capture window, where its combo-kernel +benchmark does an illegal `torch.randn` → `cudaErrorStreamCaptureInvalidated`. + +The fix is operational, not code: **invoke the script the same way for every +phase** (same shell / same `cd`, or always an absolute path), so SAVE pass 1, +SAVE pass 2, and LOAD all pass the identical path string → identical cache hash +→ pass 2 reuses pass 1's compiled kernels. Run from the workspace root, e.g.: + +```bash +cd # one canonical path for all three phases +bash foundry/recipe/experimental/serve_qwen3-30ba3b_ipc_ep.sh 2 --save +bash foundry/recipe/experimental/serve_qwen3-30ba3b_ipc_ep.sh 2 --save +bash foundry/recipe/experimental/serve_qwen3-30ba3b_ipc_ep.sh 2 --load +``` + +## Workflow + +Same two-pass SAVE → LOAD as the base recipe; `` is the first arg: + +```bash +# 0. Fresh start (distinct workspace from the base recipe) +rm -rf foundry_archive_ipc + +# 1. SAVE pass 1 — memory profile + capture +bash serve_qwen3-30ba3b_ipc_ep.sh 2 --save +# wait for "Application startup complete", then Ctrl-C + +# 2. SAVE pass 2 — deterministic re-capture +bash serve_qwen3-30ba3b_ipc_ep.sh 2 --save +# wait for "Application startup complete", then Ctrl-C + +# 3. LOAD — preallocate, re-import IPC buffers, replay graphs +bash serve_qwen3-30ba3b_ipc_ep.sh 2 --load +# leave running + +# 4. Query (separate shell) +bash ../../../experimental/query.sh 12000 Qwen/Qwen3-30B-A3B +``` + +Uncomment `nvshmem_host_path` in both TOMLs first (EP still needs NVSHMEM for the +DeepEP RDMA buffer; the IPC path only changes the NVL buffer). + +## What `FOUNDRY_DEEPEP_NVL_IPC=1` does + +`_patch_deepep` (`foundry/python/foundry/integration/vllm/hooks.py`) normally +forces `use_fabric=True, num_nvl_bytes=0` so the only cross-GPU buffer is the +NVSHMEM symmetric heap. With `FOUNDRY_DEEPEP_NVL_IPC=1` it instead keeps +upstream's nonzero `num_nvl_bytes` with `use_fabric=False`, so the DeepEP Buffer +allocates the NVLink buffer via `cudaMalloc` + `cudaIpcGetMemHandle` on **both** +SAVE and LOAD. The hook then: + +- **SAVE**: exports each VMM-backed NVL buffer as a POSIX fd, served to peers + over `\0foundry-vmm-ipc.` via `SCM_RIGHTS` (same-uid `SO_PEERCRED` check, + per-process token vs PID reuse). +- **LOAD**: buffers are carved from the preallocated chunk (no individual + handle), so the hook exports the **whole chunk** fd + offset; the peer maps the + entire chunk once and returns an interior pointer. +- Peer mappings land at a **relocated** VA (logged `[HOOK] INFO: VMM-IPC import + relocated`) — correct, because DeepEP resolves peers through its device-side + `buffer_ptrs_gpu` table (refreshed by `Buffer.sync`), never through addresses + baked into captured graphs. + +A healthy run logs two `VMM-IPC import relocated` lines per phase (one per peer) +and **no** `error 999`. + +## Validation status + +SAVE → SAVE → LOAD → query verified on Qwen3-30B-A3B EP=2 (2× H200, no IMEX): +per-rank `final_alloc_offset` identical across passes, LOAD replays at the saved +offset, query returns coherent completions. LOAD reaches a serving server in +~27 s; the IPC import itself is sub-second (inside the ~7.5 s `load_model`) and +has no steady-state serving cost — peer addresses are resolved once at init via +the device pointer table, not per token. + +## IPC-specific troubleshooting + +| Symptom | Likely cause | +|---|---| +| `[HOOK] ERROR: cuMemImportFromShareableHandle failed with error 999` at `deep_ep.cpp` `Buffer::sync` | Foundry hook not rebuilt with the SCM_RIGHTS transport — it's still packing a raw fd. Rebuild `foundry/` (`uv pip install -e . --no-build-isolation`). | +| `operation failed due to a previous error during capture` on SAVE **pass 2** | vLLM compile-cache over-keying not applied (`graph_extension_config_path` still hashed) → recompile-in-capture. Apply the `compilation.py` `ignored_factors` edit, or set `FOUNDRY_DISABLE_COMBO_KERNELS=1` (opt-in belt-and-suspenders, wired in the experimental serve script under `experimental/expert-parallel/`). | +| LOAD `illegal memory access` at replay with an NVL buffer present | Relocated peer import collided with the NVSHMEM heap hint — ensure the hook build includes the dedicated import-VA zone (`0x300000000000`). | +| `error 999` only with `--deepep-mode auto`/`normal` | That's expected for non-LL modes here; this recipe pins `deepep_low_latency`. | diff --git a/recipe/experimental/foundry_load.toml b/recipe/experimental/foundry_load.toml new file mode 100644 index 0000000..09677bb --- /dev/null +++ b/recipe/experimental/foundry_load.toml @@ -0,0 +1,9 @@ +mode = "load" +base_addr = 0x600000000000 +region_size = "256GB" +workspace_root = "foundry_archive_ipc" +scratch_space_size = "4096MB" +# REQUIRED for the EP/IPC recipe — DeepEP low-latency still allocates its RDMA +# buffer on the NVSHMEM symmetric heap (the NVL buffer is the cudaIpc path). +# Point this at the libnvshmem_host.so built by vllm/tools/ep_kernels. +nvshmem_host_path = "/data/liuxs/workarea/foundry-org/vllm/tools/ep_kernels/ep_kernels_workspace/nvshmem/lib/libnvshmem_host.so" diff --git a/recipe/experimental/foundry_load_fp8.toml b/recipe/experimental/foundry_load_fp8.toml new file mode 100644 index 0000000..a035b6a --- /dev/null +++ b/recipe/experimental/foundry_load_fp8.toml @@ -0,0 +1,6 @@ +mode = "load" +base_addr = 0x600000000000 +region_size = "256GB" +workspace_root = "foundry_archive_ipc_fp8" +scratch_space_size = "4096MB" +nvshmem_host_path = "/data/liuxs/workarea/foundry-org/vllm/tools/ep_kernels/ep_kernels_workspace/nvshmem/lib/libnvshmem_host.so" diff --git a/recipe/experimental/foundry_save.toml b/recipe/experimental/foundry_save.toml new file mode 100644 index 0000000..da07d1a --- /dev/null +++ b/recipe/experimental/foundry_save.toml @@ -0,0 +1,9 @@ +mode = "save" +base_addr = 0x600000000000 +region_size = "256GB" +workspace_root = "foundry_archive_ipc" +scratch_space_size = "4096MB" +# REQUIRED for the EP/IPC recipe — DeepEP low-latency still allocates its RDMA +# buffer on the NVSHMEM symmetric heap (the NVL buffer is the cudaIpc path). +# Point this at the libnvshmem_host.so built by vllm/tools/ep_kernels. +nvshmem_host_path = "/data/liuxs/workarea/foundry-org/vllm/tools/ep_kernels/ep_kernels_workspace/nvshmem/lib/libnvshmem_host.so" diff --git a/recipe/experimental/foundry_save_fp8.toml b/recipe/experimental/foundry_save_fp8.toml new file mode 100644 index 0000000..74a8adc --- /dev/null +++ b/recipe/experimental/foundry_save_fp8.toml @@ -0,0 +1,6 @@ +mode = "save" +base_addr = 0x600000000000 +region_size = "256GB" +workspace_root = "foundry_archive_ipc_fp8" +scratch_space_size = "4096MB" +nvshmem_host_path = "/data/liuxs/workarea/foundry-org/vllm/tools/ep_kernels/ep_kernels_workspace/nvshmem/lib/libnvshmem_host.so" diff --git a/recipe/experimental/serve_qwen3-30ba3b_ipc_ep.sh b/recipe/experimental/serve_qwen3-30ba3b_ipc_ep.sh new file mode 100755 index 0000000..949f18f --- /dev/null +++ b/recipe/experimental/serve_qwen3-30ba3b_ipc_ep.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Usage: bash serve_qwen3-30ba3b_ipc_ep.sh [--save|--load] +# +# EXPERIMENTAL: DeepEP expert-parallel with the legacy CUDA-IPC NVLink buffer +# (cudaIpcGet/OpenMemHandle) kept ON under foundry, instead of the default +# fabric/NVSHMEM-only path. This exercises foundry's VMM-IPC translation layer +# (SCM_RIGHTS fd transport + whole-chunk peer mapping) — the path that lets +# foundry SAVE/LOAD work on machines without fabric/IMEX, where DeepEP shares +# its intranode NVL buffers via legacy IPC. +# +# Differs from recipe/vllm/serve_qwen3-30ba3b_ep.sh by exactly one env var: +# FOUNDRY_DEEPEP_NVL_IPC=1 -> _patch_deepep keeps upstream's nonzero +# num_nvl_bytes with use_fabric=False (legacy cudaIpc NVL buffer on SAVE and +# LOAD), instead of forcing use_fabric=True / num_nvl_bytes=0. +# +# Requires the foundry hook rebuilt with the SCM_RIGHTS VMM-IPC transport and +# the vLLM compile-cache fix (see this directory's README "Required code"). +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +EP_SIZE=${1:?Usage: $0 [--save|--load]} +MODEL_NAME="Qwen/Qwen3-30B-A3B" +HOST="0.0.0.0" +PORT=12000 +GPU_MEMORY_UTILIZATION=0.8 + +FOUNDRY_ARGS=() +if [[ "$2" == "--save" ]]; then + FOUNDRY_ARGS+=( --compilation-config.graph_extension_config_path "${SCRIPT_DIR}/foundry_save.toml" ) + # Foundry pins NCCL to the IPC/ring transports — the CUMEM P2P and NVLS + # multicast fast paths cuMemMap with driver-capability flags the foundry + # VMM region doesn't carry. + export NCCL_CUMEM_ENABLE=0 + export NCCL_NVLS_ENABLE=0 + # Foundry only patches the V1 model runner; pin V1 explicitly so vLLM + # doesn't quietly route Qwen3ForCausalLM-class architectures to V2. + export VLLM_USE_V2_MODEL_RUNNER=0 + # Keep DeepEP's NVLink buffer on the legacy cudaIpc path under foundry. + # Must be identical on SAVE and LOAD (it changes the comm-buffer alloc + # trajectory and the captured graphs). + export FOUNDRY_DEEPEP_NVL_IPC=1 + echo "Using foundry SAVE (DeepEP NVL/IPC): ${SCRIPT_DIR}/foundry_save.toml" +elif [[ "$2" == "--load" ]]; then + FOUNDRY_ARGS+=( --compilation-config.graph_extension_config_path "${SCRIPT_DIR}/foundry_load.toml" ) + export NCCL_CUMEM_ENABLE=0 + export NCCL_NVLS_ENABLE=0 + export VLLM_USE_V2_MODEL_RUNNER=0 + export FOUNDRY_DEEPEP_NVL_IPC=1 + echo "Using foundry LOAD (DeepEP NVL/IPC): ${SCRIPT_DIR}/foundry_load.toml" +elif [[ -n "$2" ]]; then + echo "Usage: $0 [--save|--load]" + exit 1 +else + echo "Running without foundry (baseline vLLM)" +fi + +# LD_PRELOAD of libcuda_hook.so / libnvshmem_host.so is set by foundry's +# setup_ld_preload_env at worker spawn time (uses the paths in the TOML +# config). Baseline runs don't need either preloaded by the shell. + +# Foundry only patches the V1 model runner. vLLM defaults certain +# architectures (e.g. Qwen3ForCausalLM) to the V2 runner, which our +# patches don't touch — pin V1 here. +export VLLM_USE_V2_MODEL_RUNNER=0 + +export VLLM_USE_FLASHINFER_SAMPLER=1 +export VLLM_DISABLE_SHARED_EXPERTS_STREAM=1 + +CUDAGRAPH_CAPTURE_SIZES=($(seq 1 256)) + +ARGS=( + --trust-remote-code + --host "$HOST" + --port "$PORT" + --tensor-parallel-size 1 + --data-parallel-size "$EP_SIZE" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --distributed-executor-backend uni + --enable-expert-parallel + --all2all-backend deepep_low_latency + --no-enable-prefix-caching + # max-num-batched-tokens == max-num-seqs satisfies vLLM's batched>=seqs check + # and keeps DeepEP's (tokens+1)*2 <= NVSHMEM_QP_DEPTH(=1024) — no QP override. + --max-num-batched-tokens 256 + --max-num-seqs 256 + --attention-config.backend FLASH_ATTN + --compilation-config.cudagraph_mode FULL_DECODE_ONLY + --compilation-config.cudagraph_num_of_warmups 0 + --chat-template-content-format string + --cudagraph-capture-sizes ${CUDAGRAPH_CAPTURE_SIZES[@]} +) + +vllm serve "$MODEL_NAME" "${ARGS[@]}" "${FOUNDRY_ARGS[@]}" diff --git a/recipe/experimental/serve_qwen3-30ba3bfp8_ipc_ep.sh b/recipe/experimental/serve_qwen3-30ba3bfp8_ipc_ep.sh new file mode 100755 index 0000000..c5da14d --- /dev/null +++ b/recipe/experimental/serve_qwen3-30ba3bfp8_ipc_ep.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Usage: bash serve_qwen3-30ba3bfp8_ipc_ep.sh [--save|--load] +# +# EXPERIMENTAL: FP8 Qwen3-30B-A3B with DeepGEMM MoE + DeepEP expert-parallel, +# keeping the legacy CUDA-IPC NVLink buffer ON under foundry (FOUNDRY_DEEPEP_NVL_IPC=1). +# Same as serve_qwen3-30ba3b_ipc_ep.sh but: FP8 model + VLLM_USE_DEEP_GEMM=1 +# (blockscale FP8 MoE), and its own archive (foundry_archive_ipc_fp8). +# +# Run every phase from one consistent path (or absolute path) so SAVE pass 1/2 +# and LOAD pass the identical graph_extension_config_path — see README. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +EP_SIZE=${1:?Usage: $0 [--save|--load]} +MODEL_NAME="Qwen/Qwen3-30B-A3B-FP8" +HOST="0.0.0.0" +PORT=12000 +GPU_MEMORY_UTILIZATION=0.8 + +FOUNDRY_ARGS=() +if [[ "$2" == "--save" ]]; then + FOUNDRY_ARGS+=( --compilation-config.graph_extension_config_path "${SCRIPT_DIR}/foundry_save_fp8.toml" ) + export NCCL_CUMEM_ENABLE=0 + export NCCL_NVLS_ENABLE=0 + export VLLM_USE_V2_MODEL_RUNNER=0 + export FOUNDRY_DEEPEP_NVL_IPC=1 + echo "Using foundry SAVE (FP8/DeepGEMM, DeepEP NVL/IPC): ${SCRIPT_DIR}/foundry_save_fp8.toml" +elif [[ "$2" == "--load" ]]; then + FOUNDRY_ARGS+=( --compilation-config.graph_extension_config_path "${SCRIPT_DIR}/foundry_load_fp8.toml" ) + export NCCL_CUMEM_ENABLE=0 + export NCCL_NVLS_ENABLE=0 + export VLLM_USE_V2_MODEL_RUNNER=0 + export FOUNDRY_DEEPEP_NVL_IPC=1 + echo "Using foundry LOAD (FP8/DeepGEMM, DeepEP NVL/IPC): ${SCRIPT_DIR}/foundry_load_fp8.toml" +elif [[ -n "$2" ]]; then + echo "Usage: $0 [--save|--load]" + exit 1 +else + echo "Running without foundry (baseline vLLM)" +fi + +export VLLM_USE_V2_MODEL_RUNNER=0 +export VLLM_USE_FLASHINFER_SAMPLER=1 +export VLLM_DISABLE_SHARED_EXPERTS_STREAM=1 +# FP8 blockscale MoE via DeepGEMM. +export VLLM_USE_DEEP_GEMM=1 + +CUDAGRAPH_CAPTURE_SIZES=($(seq 1 256)) + +ARGS=( + --trust-remote-code + --host "$HOST" + --port "$PORT" + --tensor-parallel-size 1 + --data-parallel-size "$EP_SIZE" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --distributed-executor-backend uni + --enable-expert-parallel + --all2all-backend deepep_low_latency + --no-enable-prefix-caching + --max-num-batched-tokens 256 + --max-num-seqs 256 + --attention-config.backend FLASH_ATTN + --compilation-config.cudagraph_mode FULL_DECODE_ONLY + --compilation-config.cudagraph_num_of_warmups 0 + --chat-template-content-format string + --cudagraph-capture-sizes ${CUDAGRAPH_CAPTURE_SIZES[@]} +) + +vllm serve "$MODEL_NAME" "${ARGS[@]}" "${FOUNDRY_ARGS[@]}" diff --git a/tests/run_deepep_matrix.sh b/tests/run_deepep_matrix.sh new file mode 100755 index 0000000..b6ac4c9 --- /dev/null +++ b/tests/run_deepep_matrix.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Matrix driver for tests/test_deepep_fabric.py — standalone DeepEP save/load probes +# (no vLLM/sglang init; needs ~4 GB free on each of GPUs 0,1, NOT a fully empty GPU). +# +# ll_nofabric pure low-latency (NVSHMEM heap, FD handles), NCCL fast paths off +# -> the "EP without fabric" baseline; expected PASS +# ll_ncclcumem same, but NCCL_CUMEM_ENABLE=1 -> probe: can NCCL cumem coexist? +# ll_ncclnvls same, but CUMEM=1 + NVLS=1 -> probe: can NCCL NVLS coexist? +# nvl_ipc + 64 MB NVL buffer, use_fabric=0 -> legacy cudaIpc path through the +# hook's VMM-IPC translation (SCM_RIGHTS fd transport; peer mappings +# relocate under shared region bases); expected PASS +# nvl_fabric + 64 MB NVL buffer, use_fabric=1 -> fabric cuMemCreate on a no-IMEX +# machine; documents the fabric dependency; expected FAIL +# +# Usage: run_deepep_matrix.sh +set -euo pipefail + +ROOT=/data/liuxs/workarea/foundry-org +PY=$ROOT/foundry_env/bin/python +TEST=$ROOT/foundry/tests/test_deepep_fabric.py +NVSHMEM_SO=$ROOT/vllm/tools/ep_kernels/ep_kernels_workspace/nvshmem/lib/libnvshmem_host.so +HOOK_SO=$($PY -c "import foundry.ops, pathlib; print(pathlib.Path(foundry.ops.__file__).parent / 'libcuda_hook.so')") + +CASE=${1:?usage: run_deepep_matrix.sh } +PHASE=${2:?usage: run_deepep_matrix.sh } + +export CUDA_VISIBLE_DEVICES=0,1 +# Pin NVSHMEM heap chunks to POSIX FD handles — explicit "no fabric handles anywhere". +# (Auto-detect picks FD on non-MNNVL machines anyway; pinning removes the variable.) +export NVSHMEM_CUMEM_HANDLE_TYPE=FILE_DESCRIPTOR + +case "$CASE" in + ll_nofabric) export TEST_USE_FABRIC=0 TEST_NVL_BYTES_MB=0 NCCL_CUMEM_ENABLE=0 NCCL_NVLS_ENABLE=0; PORT=29611 ;; + ll_ncclcumem) export TEST_USE_FABRIC=0 TEST_NVL_BYTES_MB=0 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=0; PORT=29621 ;; + ll_ncclcumem_preinit) export TEST_USE_FABRIC=0 TEST_NVL_BYTES_MB=0 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=0 TEST_NCCL_WARMUP_PRE_REGION=1; PORT=29661 ;; + ll_ncclnvls) export TEST_USE_FABRIC=0 TEST_NVL_BYTES_MB=0 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1; PORT=29631 ;; + ll_ncclnvls_preinit) export TEST_USE_FABRIC=0 TEST_NVL_BYTES_MB=0 NCCL_CUMEM_ENABLE=1 NCCL_NVLS_ENABLE=1 TEST_NCCL_WARMUP_PRE_REGION=1; PORT=29671 ;; + nvl_ipc) export TEST_USE_FABRIC=0 TEST_NVL_BYTES_MB=64 NCCL_CUMEM_ENABLE=0 NCCL_NVLS_ENABLE=0; PORT=29641 ;; + nvl_fabric) export TEST_USE_FABRIC=1 TEST_NVL_BYTES_MB=64 NCCL_CUMEM_ENABLE=0 NCCL_NVLS_ENABLE=0; PORT=29651 ;; + nvl_ipc_prealloc) export TEST_USE_FABRIC=0 TEST_NVL_BYTES_MB=64 NCCL_CUMEM_ENABLE=0 NCCL_NVLS_ENABLE=0 TEST_LOAD_PREALLOC_MB=1024; PORT=29681 ;; + *) echo "unknown case: $CASE"; exit 2 ;; +esac + +# GPUs are shared with other users — refuse to run without headroom. +for i in 0 1; do + free=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits -i $i) + if [ "$free" -lt 6000 ]; then + echo "GPU $i has only ${free} MiB free — aborting (need ~6 GB headroom)"; exit 3 + fi +done + +# main() overwrites TEST_USE_FABRIC from the CLI flag, so pass the flag too. +FABRIC_ARG="" +[ "$TEST_USE_FABRIC" = "0" ] && FABRIC_ARG="--no-fabric" + +WORK=$ROOT/foundry/tests/deepep_matrix_work/$CASE +LOGDIR=$ROOT/logs/deepep_matrix +mkdir -p "$WORK" "$LOGDIR" +cd "$WORK" # deepep_fabric_archive/ is CWD-relative + +export LD_PRELOAD=$NVSHMEM_SO:$HOOK_SO${LD_PRELOAD:+:$LD_PRELOAD} + +run_phase() { + local p=$1 + # distinct rendezvous port per phase to dodge TIME_WAIT + if [ "$p" = save ]; then export MASTER_PORT=$PORT; else export MASTER_PORT=$((PORT + 1)); fi + echo "=== case=$CASE phase=$p $(date '+%F %T') ===" + echo "env: TEST_USE_FABRIC=$TEST_USE_FABRIC TEST_NVL_BYTES_MB=$TEST_NVL_BYTES_MB" \ + "NCCL_CUMEM_ENABLE=$NCCL_CUMEM_ENABLE NCCL_NVLS_ENABLE=$NCCL_NVLS_ENABLE" \ + "NVSHMEM_CUMEM_HANDLE_TYPE=$NVSHMEM_CUMEM_HANDLE_TYPE MASTER_PORT=$MASTER_PORT" + if [ "$p" = save ]; then rm -rf deepep_fabric_archive; fi + $PY "$TEST" --$p $FABRIC_ARG --num-processes=2 2>&1 | tee "$LOGDIR/${CASE}_${p}.log" +} + +if [ "$PHASE" = both ]; then run_phase save; run_phase load; else run_phase "$PHASE"; fi diff --git a/tests/test_deepep_fabric.py b/tests/test_deepep_fabric.py index be6be72..b139558 100644 --- a/tests/test_deepep_fabric.py +++ b/tests/test_deepep_fabric.py @@ -77,6 +77,27 @@ def _init_dist(local_rank: int, num_local_ranks: int): ) +def _maybe_warm_nccl_pre_region(group): + """Diagnostic knob (TEST_NCCL_WARMUP_PRE_REGION=1): force NCCL transport + setup BEFORE the foundry region exists, so NCCL's cuMem allocations + (NCCL_CUMEM_ENABLE=1) pass through untracked. Localizes whether the + NCCL-CUMEM incompatibility is "NCCL cuMem while region enabled" only. + The warmup tensor's caching-allocator segment is drained afterwards so + post-region allocations don't reuse an untracked (non-deterministic) + segment. + """ + if os.environ.get("TEST_NCCL_WARMUP_PRE_REGION", "0") != "1": + return + t = torch.ones(1024, 1024, device="cuda") + dist.all_reduce(t) # default group: ring/p2p transport setup + dist.all_reduce(t, group=group) # subgroup used by the DeepEP Buffer + dist.all_gather_object([None] * dist.get_world_size(), 0) # object-coll path + torch.cuda.synchronize() + del t + torch.cuda.empty_cache() # legal here: region not set yet, no captures + print("[TEST] NCCL pre-region warmup done (transports up before region)", flush=True) + + def _create_deterministic_inputs( rank: int, num_tokens: int, hidden: int, num_experts: int, num_topk: int ): @@ -263,6 +284,8 @@ def _run_save(local_rank: int, num_processes: int): num_experts = num_ranks * 4 # 4 experts per rank num_topk = 2 + _maybe_warm_nccl_pre_region(group) + # Setup allocation region region_size = fdry.parse_size(REGION_SIZE_STR) print(f"[Rank {rank}] SAVE: Setting up allocation region at {hex(BASE_ADDR)}", flush=True) @@ -487,11 +510,22 @@ def _run_load(local_rank: int, num_processes: int): print(f"[Rank {rank}] LOAD: Loading CUDA modules from {rank_archive}", flush=True) fdry.load_cuda_modules_and_libraries(rank_archive) + _maybe_warm_nccl_pre_region(group) + # Step 2: Setup allocation region region_size = fdry.parse_size(REGION_SIZE_STR) print(f"[Rank {rank}] LOAD: Setting up allocation region at {hex(BASE_ADDR)}", flush=True) fdry.set_allocation_region(BASE_ADDR, region_size) + # Optional (TEST_LOAD_PREALLOC_MB=N): reproduce the production LOAD shape - + # one upfront-preallocated chunk that subsequent allocations (including the + # DeepEP NVL buffer) carve from with NO individual cuMem handle. This + # exercises the hook's whole-chunk VMM-IPC export path. + prealloc_mb = int(os.environ.get("TEST_LOAD_PREALLOC_MB", "0")) + if prealloc_mb > 0: + assert fdry.preallocate_region(prealloc_mb * 1024 * 1024), "preallocate_region failed" + print(f"[Rank {rank}] LOAD: preallocated {prealloc_mb} MB chunk", flush=True) + # Step 3: Create DeepEP buffer (initializes NVSHMEM runtime) num_local_experts = num_experts // num_ranks num_rdma_bytes = deep_ep.Buffer.get_low_latency_rdma_size_hint( @@ -499,13 +533,17 @@ def _run_load(local_rank: int, num_processes: int): ) use_fabric = os.environ.get("TEST_USE_FABRIC", "1") == "1" + # Must match SAVE: an NVL buffer allocated on SAVE but not LOAD (or vice versa) + # diverges the allocation trajectory and the peer-pointer setup. + num_nvl_bytes = int(os.environ.get("TEST_NVL_BYTES_MB", "0")) * 1024 * 1024 print( - f"[Rank {rank}] LOAD: Creating DeepEP buffer with use_fabric={use_fabric}, size={num_rdma_bytes / 1e6:.2f} MB", + f"[Rank {rank}] LOAD: Creating DeepEP buffer with use_fabric={use_fabric}, num_nvl_bytes={num_nvl_bytes / 1e6:.2f} MB, num_rdma_bytes={num_rdma_bytes / 1e6:.2f} MB", flush=True, ) buffer = deep_ep.Buffer( group, + num_nvl_bytes=num_nvl_bytes, num_rdma_bytes=num_rdma_bytes, low_latency_mode=True, num_qps_per_rank=num_local_experts, @@ -593,10 +631,21 @@ def _run_load(local_rank: int, num_processes: int): flush=True, ) - # Load graph from per-rank archive + # Load graph from per-rank archive. + # Default: the production path (start_graph_builds + finish_graph_loads), + # same machinery the vLLM/sglang integrations use. TEST_LOAD_API=single + # selects the legacy CUDAGraph.load path — KNOWN BROKEN for DeepEP graphs: + # it does not merge root-level common_kernel_node_attrs, so factored-out + # cooperative/clusterDim attrs are dropped and the dispatch kernel faults + # with "unspecified launch failure" at replay. graph_json = os.path.join(rank_archive, "deepep_dispatch_graph.json") - print(f"[Rank {rank}] LOAD: Loading graph from {graph_json}", flush=True) - graph, output_tensors = fdry.CUDAGraph.load(graph_json) + load_api = os.environ.get("TEST_LOAD_API", "parallel") + print(f"[Rank {rank}] LOAD: Loading graph from {graph_json} (api={load_api})", flush=True) + if load_api == "single": + graph, output_tensors = fdry.CUDAGraph.load(graph_json) + else: + pending = fdry.CUDAGraph.start_graph_builds([graph_json]) + ((graph, output_tensors),) = fdry.CUDAGraph.finish_graph_loads(pending) print(f"[Rank {rank}] LOAD: Graph loaded successfully", flush=True) # Print loaded output tensor info