diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dfc4860592..000677e622 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -148,6 +148,22 @@ else() message(STATUS "Found cuFile Version API: ${cuFile_VERSION_API_FOUND}") endif() +# cuObject (libcuobjclient) powers the optional S3-over-RDMA data plane. The header and library ship +# with the CUDA Toolkit. KvikIO never links them directly; instead a small C-ABI shim +# (libkvikio_cuobj_shim.so) is built when they are present and loaded at runtime via dlopen, keeping +# cuObject an optional runtime dependency (same model as cuFile). +find_path( + cuObj_INCLUDE_DIR cuobjclient.h HINTS ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_TARGET_DIR}/include +) +find_library(cuObj_LIBRARY cuobjclient HINTS ${CUDAToolkit_LIBRARY_DIR} ${CUDAToolkit_TARGET_DIR}/lib64) +if(cuObj_INCLUDE_DIR AND cuObj_LIBRARY) + set(cuObj_FOUND 1) + message(STATUS "Found cuObject: ${cuObj_LIBRARY}") +else() + set(cuObj_FOUND 0) + message(STATUS "Cannot find cuObject - KvikIO will build without S3-over-RDMA support") +endif() + include(cmake/thirdparty/get_thread_pool.cmake) # ################################################################################################## @@ -193,6 +209,7 @@ if(KvikIO_REMOTE_SUPPORT) "src/detail/remote_callback.cpp" "src/detail/tls.cpp" "src/detail/url.cpp" + "src/shim/cuobj.cpp" "src/shim/libcurl.cpp" ) endif() @@ -290,6 +307,25 @@ install( EXPORT kvikio-exports ) +# Optional C-ABI shim wrapping cuObject's C++ client. Built only when cuObject is available and +# loaded at runtime via dlopen (see src/shim/cuobj.cpp); KvikIO itself never links cuObject. +if(KvikIO_REMOTE_SUPPORT AND cuObj_FOUND) + add_library(kvikio_cuobj_shim SHARED "cuobj_shim/cuobj_shim.cpp") + target_include_directories( + kvikio_cuobj_shim PRIVATE ${cuObj_INCLUDE_DIR} ${CUDAToolkit_INCLUDE_DIRS} + ) + target_link_libraries(kvikio_cuobj_shim PRIVATE ${cuObj_LIBRARY}) + set_target_properties( + kvikio_cuobj_shim + PROPERTIES BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + ) + install(TARGETS kvikio_cuobj_shim DESTINATION ${lib_dir}) +endif() + install(DIRECTORY include/kvikio/ DESTINATION include/kvikio) install(FILES ${KvikIO_BINARY_DIR}/include/kvikio/version_config.hpp DESTINATION include/kvikio) diff --git a/cpp/cuobj_shim/cuobj_shim.cpp b/cpp/cuobj_shim/cuobj_shim.cpp new file mode 100644 index 0000000000..28b480995a --- /dev/null +++ b/cpp/cuobj_shim/cuobj_shim.cpp @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +// extern "C" shim over NVIDIA cuObject (libcuobjclient), implementing the +// "manual RDMA token" pattern (cuObjClient API spec section 1.12.4). cuObject +// is a C++ library whose client object requires an I/O-ops callback table at +// construction, so it cannot be loaded directly through KvikIO's dlopen shim +// (which resolves plain C symbols, like cuFileAPI does for libcufile). This +// thunk exposes a small C ABI that KvikIO's cuObjAPI dlopens at runtime; only +// this translation unit is compiled against cuobjclient.h and links +// libcuobjclient, so the main KvikIO library keeps cuObject as an optional +// runtime dependency. +// +// Built into libkvikio_cuobj_shim.so only when cuobjclient.h is found (see +// cpp/CMakeLists.txt). Point KVIKIO_CUOBJ_SHIM at the result, or install it on +// the loader path. + +#include + +#include +#include +#include + +namespace { + +// cuObject get/put callbacks are unused in the manual-token pattern (KvikIO +// drives the S3 request out of band over libcurl), but the cuObjClient +// constructor requires an ops table. Provide stubs. +ssize_t cuobj_stub_get(const void* /*handle*/, + char* /*ptr*/, + size_t /*size*/, + loff_t /*offset*/, + const cufileRDMAInfo_t*) +{ + return -EOPNOTSUPP; +} + +ssize_t cuobj_stub_put(const void* /*handle*/, + const char* /*ptr*/, + size_t /*size*/, + loff_t /*offset*/, + const cufileRDMAInfo_t*) +{ + return -EOPNOTSUPP; +} + +cuObjClient* get_client() +{ + static CUObjOps_t ops = {cuobj_stub_get, cuobj_stub_put}; + static cuObjClient client(ops); + return &client; +} + +// Descriptors returned by cuMemObjGetRDMAToken are owned by cuObject and must +// be released via cuMemObjPutRDMAToken. Keep the original pointer keyed by +// descriptor value so the caller can free it by the C string it received, +// without owning the lifetime. +std::mutex& token_mutex() +{ + static std::mutex m; + return m; +} + +std::unordered_map& token_registry() +{ + static std::unordered_map r; + return r; +} + +} // namespace + +extern "C" { + +int kvikio_cuobj_available() { return get_client()->isConnected() ? 1 : 0; } + +int kvikio_cuobj_register_buffer(void* ptr, size_t size) +{ + return get_client()->cuMemObjGetDescriptor(ptr, size) == CU_OBJ_SUCCESS ? 0 : -1; +} + +int kvikio_cuobj_deregister_buffer(void* ptr) +{ + return get_client()->cuMemObjPutDescriptor(ptr) == CU_OBJ_SUCCESS ? 0 : -1; +} + +// Returns the descriptor string for [offset, offset+size) of the registered +// buffer, or nullptr on failure. is_put selects PUT (1) vs GET (0). The caller +// must release it via kvikio_cuobj_put_rdma_token after the request completes. +const char* kvikio_cuobj_get_rdma_token(void* ptr, size_t size, size_t offset, int is_put) +{ + char* desc = nullptr; + cuObjOpType_t op = is_put ? CUOBJ_PUT : CUOBJ_GET; + cuObjErr_t const stat = get_client()->cuMemObjGetRDMAToken(ptr, size, offset, op, &desc); + if (stat != CU_OBJ_SUCCESS || desc == nullptr) { return nullptr; } + std::lock_guard lock(token_mutex()); + token_registry()[std::string(desc)] = desc; + return desc; +} + +int kvikio_cuobj_put_rdma_token(const char* token) +{ + if (token == nullptr) { return -1; } + char* desc = nullptr; + { + std::lock_guard lock(token_mutex()); + auto it = token_registry().find(std::string(token)); + if (it == token_registry().end()) { return 0; } + desc = it->second; + token_registry().erase(it); + } + return get_client()->cuMemObjPutRDMAToken(desc) == CU_OBJ_SUCCESS ? 0 : -1; +} + +} // extern "C" diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index a14d3a1415..3ea37c7998 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -1,6 +1,6 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 + * reserved. reserved. reserved. reserved. reserved. reserved. SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -136,6 +136,35 @@ class RemoteEndpoint { * @return The type of the remote file. */ [[nodiscard]] RemoteEndpointType remote_endpoint_type() const noexcept; + + /** + * @brief Whether this endpoint can use the cuObject S3-over-RDMA data plane. + * + * Default is `false`. An endpoint returns `true` only when RDMA is both + * requested and usable (an RDMA-capable server plus a working cuObject + * runtime). When `true`, `RemoteHandle::read()` registers the destination + * buffer with cuObject and transfers the payload directly over RDMA instead + * of streaming the HTTP body. + * + * @return Boolean answer. + */ + [[nodiscard]] virtual bool supports_rdma() const noexcept; + + /** + * @brief Configure a curl handle for an RDMA range request. + * + * Sets the same connection options as `setopt()` plus the signed + * `x-amz-rdma-token` header carrying `rdma_token`. The token must be signed + * with the rest of the request (SigV4 signs `x-amz-*` headers), so it is + * added before signing. + * + * @param curl The curl handle. + * @param rdma_token The cuObject RDMA descriptor (formatted + * `::`). + * @return An owned `curl_slist` that the caller must free with + * `curl_slist_free_all()` after the transfer completes. + */ + virtual curl_slist* setopt_rdma(CurlHandle& curl, std::string const& rdma_token); }; /** @@ -266,6 +295,15 @@ class S3Endpoint : public RemoteEndpoint { std::size_t get_file_size() override; void setup_range_request(CurlHandle& curl, std::size_t file_offset, std::size_t size) override; + /** + * @brief Whether S3-over-RDMA is usable for this endpoint. + * + * Returns `true` when the `KVIKIO_REMOTE_RDMA` environment variable is set to + * a truthy value and the cuObject runtime is available (`is_cuobj_available()`). + */ + [[nodiscard]] bool supports_rdma() const noexcept override; + curl_slist* setopt_rdma(CurlHandle& curl, std::string const& rdma_token) override; + /** * @brief Whether the given URL is valid for S3 endpoints (excluding presigned URL). * diff --git a/cpp/include/kvikio/shim/cuobj.hpp b/cpp/include/kvikio/shim/cuobj.hpp new file mode 100644 index 0000000000..bd5e96c3a2 --- /dev/null +++ b/cpp/include/kvikio/shim/cuobj.hpp @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +#include + +namespace kvikio { + +/** + * @brief Shim layer over the NVIDIA cuObject C-ABI used for S3-over-RDMA. + * + * Singleton that `dlopen`s ``libkvikio_cuobj_shim.so`` on construction and + * resolves its C entry points. The shim wraps the cuObject ``cuObjClient`` C++ + * class (see ``cpp/cuobj_shim/cuobj_shim.cpp``); loading it at runtime keeps + * cuObject an optional dependency, mirroring how :class:`cuFileAPI` loads + * ``libcufile.so.0``. The library path can be overridden with the + * ``KVIKIO_CUOBJ_SHIM`` environment variable. + */ +class cuObjAPI { + public: + int (*Available)(){nullptr}; + int (*RegisterBuffer)(void*, std::size_t){nullptr}; + int (*DeregisterBuffer)(void*){nullptr}; + char const* (*GetRDMAToken)(void*, std::size_t, std::size_t, int){nullptr}; + int (*PutRDMAToken)(char const*){nullptr}; + + private: + cuObjAPI(); + + public: + cuObjAPI(cuObjAPI const&) = delete; + cuObjAPI& operator=(cuObjAPI const&) = delete; + + KVIKIO_EXPORT static cuObjAPI& instance(); +}; + +/** + * @brief Whether cuObject S3-over-RDMA is usable. + * + * ``true`` only when the shim library loads and a cuObject client connection + * can be established (RDMA-capable NIC and a reachable RDMA S3 endpoint). + */ +[[nodiscard]] bool is_cuobj_available() noexcept; + +} // namespace kvikio diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 82d5cfec4f..b98747e2c5 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -1,13 +1,17 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 + * reserved. reserved. reserved. reserved. reserved. reserved. SPDX-License-Identifier: Apache-2.0 */ #include #include #include +#include #include +#include +#include #include +#include #include #include #include @@ -27,6 +31,7 @@ #include #include #include +#include #include #include @@ -322,6 +327,14 @@ RemoteEndpointType RemoteEndpoint::remote_endpoint_type() const noexcept return _remote_endpoint_type; } +bool RemoteEndpoint::supports_rdma() const noexcept { return false; } + +curl_slist* RemoteEndpoint::setopt_rdma(CurlHandle& /*curl*/, std::string const& /*rdma_token*/) +{ + KVIKIO_FAIL("This remote endpoint does not support RDMA", std::runtime_error); + return nullptr; +} + HttpEndpoint::HttpEndpoint(std::string url) : RemoteEndpoint{RemoteEndpointType::HTTP}, _url{std::move(url)} { @@ -365,6 +378,36 @@ void S3Endpoint::setopt(CurlHandle& curl) if (_curl_header_list) { curl.setopt(CURLOPT_HTTPHEADER, _curl_header_list); } } +bool S3Endpoint::supports_rdma() const noexcept +{ + char const* env = std::getenv("KVIKIO_REMOTE_RDMA"); + if (env == nullptr) { return false; } + std::string value{env}; + std::transform( + value.begin(), value.end(), value.begin(), [](unsigned char c) { return std::tolower(c); }); + bool const enabled = (value == "1" || value == "on" || value == "true" || value == "yes"); + return enabled && is_cuobj_available(); +} + +curl_slist* S3Endpoint::setopt_rdma(CurlHandle& curl, std::string const& rdma_token) +{ + // Reuse the standard connection options (URL, SigV4, userpwd, base headers), + // then replace the header list with a fresh copy that also carries the signed + // RDMA token. A per-call list keeps this thread-safe: a single endpoint is + // shared across the worker threads of a parallel `pread()`. + setopt(curl); + + curl_slist* list = nullptr; + for (curl_slist* node = _curl_header_list; node != nullptr; node = node->next) { + list = curl_slist_append(list, node->data); + } + std::string const header = "x-amz-rdma-token: " + rdma_token; + list = curl_slist_append(list, header.c_str()); + KVIKIO_EXPECT(list != nullptr, "Failed to build curl header for RDMA token", std::runtime_error); + curl.setopt(CURLOPT_HTTPHEADER, list); + return list; +} + std::string S3Endpoint::url_from_bucket_and_object(std::string bucket_name, std::string object_name, std::optional aws_region, @@ -754,6 +797,99 @@ std::size_t callback_device_memory(char* data, std::size_t size, std::size_t nme ctx->offset += nbytes; return nbytes; } + +// In the cuObject RDMA data plane the payload is delivered out of band straight +// into the registered buffer; only an error body (e.g. an S3 XML error) should +// ever reach this write callback, so it is discarded. +std::size_t callback_rdma_discard(char* /*data*/, std::size_t size, std::size_t nmemb, void*) +{ + return size * nmemb; +} + +// Captures the `x-amz-rdma-reply` response header so the caller can confirm the +// endpoint honored the RDMA request. +struct RdmaReplyContext { + std::string reply; +}; + +std::size_t callback_rdma_reply(char* buffer, std::size_t size, std::size_t nitems, void* userdata) +{ + std::size_t const nbytes = size * nitems; + auto* ctx = reinterpret_cast(userdata); + std::string line{buffer, nbytes}; + auto const colon = line.find(':'); + if (colon != std::string::npos) { + std::string name = line.substr(0, colon); + std::transform( + name.begin(), name.end(), name.begin(), [](unsigned char c) { return std::tolower(c); }); + if (name == "x-amz-rdma-reply") { + std::string value = line.substr(colon + 1); + auto const begin = value.find_first_not_of(" \t"); + auto const end = value.find_last_not_of(" \t\r\n"); + ctx->reply = (begin == std::string::npos) ? "" : value.substr(begin, end - begin + 1); + } + } + return nbytes; +} + +// Single-shot RDMA read: register `buf` with cuObject, mint a descriptor for +// `[0, size)`, issue a body-less range GET carrying the signed +// `x-amz-rdma-token`, and let the endpoint RDMA-write the payload directly into +// `buf`. Works for both host and device buffers (cuObject accepts either). +std::size_t read_rdma( + RemoteEndpoint& endpoint, void* buf, std::size_t size, std::size_t file_offset, bool is_host_mem) +{ + KVIKIO_NVTX_FUNC_RANGE(size); + auto& api = cuObjAPI::instance(); + + KVIKIO_EXPECT(api.RegisterBuffer(buf, size) == 0, + "cuObject failed to register the destination buffer for RDMA", + std::runtime_error); + + char const* descriptor = nullptr; + curl_slist* header_list = nullptr; + try { + descriptor = api.GetRDMAToken(buf, size, 0, /*is_put=*/0); + KVIKIO_EXPECT( + descriptor != nullptr, "cuObject failed to mint an RDMA token", std::runtime_error); + + auto const addr = reinterpret_cast(buf); + std::stringstream token; + token << descriptor << ":" << std::hex << std::setw(16) << std::setfill('0') << addr << ":" + << std::setw(16) << std::setfill('0') << size; + + auto curl = create_curl_handle(); + header_list = endpoint.setopt_rdma(curl, token.str()); + endpoint.setup_range_request(curl, file_offset, size); + + RdmaReplyContext reply{}; + curl.setopt(CURLOPT_WRITEFUNCTION, callback_rdma_discard); + curl.setopt(CURLOPT_HEADERFUNCTION, callback_rdma_reply); + curl.setopt(CURLOPT_HEADERDATA, &reply); + + if (is_host_mem) { + curl.perform(); + } else { + PushAndPopContext c(get_context_from_pointer(buf)); + curl.perform(); + } + + KVIKIO_EXPECT(!reply.reply.empty() && reply.reply != "501", + "S3 endpoint declined RDMA (x-amz-rdma-reply='" + reply.reply + + "'); the endpoint is not RDMA-capable. Unset KVIKIO_REMOTE_RDMA to use the " + "standard data plane.", + std::runtime_error); + } catch (...) { + if (descriptor != nullptr) { api.PutRDMAToken(descriptor); } + if (header_list != nullptr) { curl_slist_free_all(header_list); } + api.DeregisterBuffer(buf); + throw; + } + api.PutRDMAToken(descriptor); + curl_slist_free_all(header_list); + api.DeregisterBuffer(buf); + return size; +} } // namespace detail std::size_t RemoteHandle::read(void* buf, std::size_t size, std::size_t file_offset) @@ -769,7 +905,12 @@ std::size_t RemoteHandle::read(void* buf, std::size_t size, std::size_t file_off KVIKIO_FAIL(ss.str(), std::invalid_argument); } bool const is_host_mem = is_host_memory(buf); - auto curl = create_curl_handle(); + + if (_endpoint->supports_rdma()) { + return detail::read_rdma(*_endpoint, buf, size, file_offset, is_host_mem); + } + + auto curl = create_curl_handle(); _endpoint->setopt(curl); _endpoint->setup_range_request(curl, file_offset, size); diff --git a/cpp/src/shim/cuobj.cpp b/cpp/src/shim/cuobj.cpp new file mode 100644 index 0000000000..50ca6cdc5a --- /dev/null +++ b/cpp/src/shim/cuobj.cpp @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include + +namespace kvikio { + +namespace { +std::string cuobj_shim_library_name() +{ + if (char const* env = std::getenv("KVIKIO_CUOBJ_SHIM"); env != nullptr && *env != '\0') { + return std::string{env}; + } + return "libkvikio_cuobj_shim.so"; +} +} // namespace + +cuObjAPI::cuObjAPI() +{ + void* lib = load_library(cuobj_shim_library_name()); + get_symbol(Available, lib, "kvikio_cuobj_available"); + get_symbol(RegisterBuffer, lib, "kvikio_cuobj_register_buffer"); + get_symbol(DeregisterBuffer, lib, "kvikio_cuobj_deregister_buffer"); + get_symbol(GetRDMAToken, lib, "kvikio_cuobj_get_rdma_token"); + get_symbol(PutRDMAToken, lib, "kvikio_cuobj_put_rdma_token"); +} + +cuObjAPI& cuObjAPI::instance() +{ + static cuObjAPI _instance; + return _instance; +} + +bool is_cuobj_available() noexcept +{ + try { + return cuObjAPI::instance().Available() != 0; + } catch (std::runtime_error const&) { + return false; + } +} + +} // namespace kvikio diff --git a/docs/source/remote_file.rst b/docs/source/remote_file.rst index e6d038035e..7dbd2a1254 100644 --- a/docs/source/remote_file.rst +++ b/docs/source/remote_file.rst @@ -9,6 +9,25 @@ Example .. literalinclude:: ../../python/kvikio/examples/http_io.py :language: python +S3-over-RDMA (NVIDIA cuObject) +------------------------------ + +For RDMA-capable S3 endpoints (such as MinIO AIStor), KvikIO can transfer object payloads directly +into the destination buffer over RDMA using NVIDIA cuObject, bypassing the HTTP body and the CPU copy. +This is an opt-in data plane for the S3 endpoint: + +- Build KvikIO on a host where the cuObject SDK (``cuobjclient.h`` and ``libcuobjclient``, shipped with + the CUDA Toolkit) is present. CMake then builds ``libkvikio_cuobj_shim.so`` automatically. +- At runtime, set ``KVIKIO_REMOTE_RDMA=on``. If the shim cannot be found on the loader path, point + ``KVIKIO_CUOBJ_SHIM`` at it. cuObject also needs a cuFile/cuObject JSON config describing the RDMA NIC + (``CUFILE_ENV_PATH_JSON``). + +When enabled and a cuObject connection is established, ``RemoteHandle.read``/``pread`` register the +destination buffer (host or device memory) with cuObject and issue a body-less range request carrying +the signed ``x-amz-rdma-token`` header; the endpoint RDMA-writes the payload into the buffer. If the +endpoint does not honor the request (``x-amz-rdma-reply: 501``), the read fails rather than silently +falling back, so only enable it against an RDMA-capable endpoint. + AWS S3 object naming requirement --------------------------------