From c73ae840f442dfe661efa832fb9ebd47c72c2272 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 26 Jun 2026 12:04:56 -0700 Subject: [PATCH] Add optional S3-over-RDMA (NVIDIA cuObject) data plane to remote I/O Add an opt-in RDMA data plane for the S3 remote endpoint, backed by NVIDIA cuObject (libcuobjclient). When enabled, RemoteHandle::read() registers the destination buffer (host or device memory) with cuObject, mints an RDMA descriptor, and issues a body-less range GET carrying the SigV4-signed x-amz-rdma-token header; the RDMA-capable endpoint transfers the payload directly into the buffer, bypassing the HTTP body and the CPU copy. The HTTP control plane (range, auth) continues to flow through the existing libcurl path, so the change is additive. cuObject's client is a C++ class, so it is not loaded directly through the dlopen shim mechanism the way libcufile is. Instead a small C-ABI shim (cpp/cuobj_shim/cuobj_shim.cpp) wraps it and is built into the optional libkvikio_cuobj_shim.so only when cuobjclient.h is found; the cuObjAPI singleton dlopens that shim at runtime (mirroring cuFileAPI), so cuObject remains an optional runtime dependency and KvikIO links neither it nor the shim. Activation is gated by the KVIKIO_REMOTE_RDMA environment variable plus a successful cuObject connection (S3Endpoint::supports_rdma()), so existing behavior is unchanged when unset. A 501 reply (x-amz-rdma-reply) fails loudly rather than silently falling back, since RDMA must be explicit. This is host-staged-capable today and validated against an RDMA-capable endpoint; device-memory (GPUDirect) registration uses the same code path and is compile-validated, with hardware end-to-end validation pending. Test Plan: - Build on a host with the cuObject SDK; confirm libkvikio_cuobj_shim.so builds and KvikIO loads without it when absent. - Host-memory round trip against an RDMA MinIO AIStor endpoint with KVIKIO_REMOTE_RDMA=on: RemoteHandle::read()/pread() returns byte-identical data and x-amz-rdma-reply is 200. - With KVIKIO_REMOTE_RDMA unset, the standard libcurl data plane is used (existing tests unaffected). --- cpp/CMakeLists.txt | 36 +++++++ cpp/cuobj_shim/cuobj_shim.cpp | 116 +++++++++++++++++++++ cpp/include/kvikio/remote_handle.hpp | 40 +++++++- cpp/include/kvikio/shim/cuobj.hpp | 50 +++++++++ cpp/src/remote_handle.cpp | 145 ++++++++++++++++++++++++++- cpp/src/shim/cuobj.cpp | 50 +++++++++ docs/source/remote_file.rst | 19 ++++ 7 files changed, 453 insertions(+), 3 deletions(-) create mode 100644 cpp/cuobj_shim/cuobj_shim.cpp create mode 100644 cpp/include/kvikio/shim/cuobj.hpp create mode 100644 cpp/src/shim/cuobj.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dfc4860592..000677e622 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -148,6 +148,22 @@ else() message(STATUS "Found cuFile Version API: ${cuFile_VERSION_API_FOUND}") endif() +# cuObject (libcuobjclient) powers the optional S3-over-RDMA data plane. The header and library ship +# with the CUDA Toolkit. KvikIO never links them directly; instead a small C-ABI shim +# (libkvikio_cuobj_shim.so) is built when they are present and loaded at runtime via dlopen, keeping +# cuObject an optional runtime dependency (same model as cuFile). +find_path( + cuObj_INCLUDE_DIR cuobjclient.h HINTS ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_TARGET_DIR}/include +) +find_library(cuObj_LIBRARY cuobjclient HINTS ${CUDAToolkit_LIBRARY_DIR} ${CUDAToolkit_TARGET_DIR}/lib64) +if(cuObj_INCLUDE_DIR AND cuObj_LIBRARY) + set(cuObj_FOUND 1) + message(STATUS "Found cuObject: ${cuObj_LIBRARY}") +else() + set(cuObj_FOUND 0) + message(STATUS "Cannot find cuObject - KvikIO will build without S3-over-RDMA support") +endif() + include(cmake/thirdparty/get_thread_pool.cmake) # ################################################################################################## @@ -193,6 +209,7 @@ if(KvikIO_REMOTE_SUPPORT) "src/detail/remote_callback.cpp" "src/detail/tls.cpp" "src/detail/url.cpp" + "src/shim/cuobj.cpp" "src/shim/libcurl.cpp" ) endif() @@ -290,6 +307,25 @@ install( EXPORT kvikio-exports ) +# Optional C-ABI shim wrapping cuObject's C++ client. Built only when cuObject is available and +# loaded at runtime via dlopen (see src/shim/cuobj.cpp); KvikIO itself never links cuObject. +if(KvikIO_REMOTE_SUPPORT AND cuObj_FOUND) + add_library(kvikio_cuobj_shim SHARED "cuobj_shim/cuobj_shim.cpp") + target_include_directories( + kvikio_cuobj_shim PRIVATE ${cuObj_INCLUDE_DIR} ${CUDAToolkit_INCLUDE_DIRS} + ) + target_link_libraries(kvikio_cuobj_shim PRIVATE ${cuObj_LIBRARY}) + set_target_properties( + kvikio_cuobj_shim + PROPERTIES BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + ) + install(TARGETS kvikio_cuobj_shim DESTINATION ${lib_dir}) +endif() + install(DIRECTORY include/kvikio/ DESTINATION include/kvikio) install(FILES ${KvikIO_BINARY_DIR}/include/kvikio/version_config.hpp DESTINATION include/kvikio) diff --git a/cpp/cuobj_shim/cuobj_shim.cpp b/cpp/cuobj_shim/cuobj_shim.cpp new file mode 100644 index 0000000000..28b480995a --- /dev/null +++ b/cpp/cuobj_shim/cuobj_shim.cpp @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +// extern "C" shim over NVIDIA cuObject (libcuobjclient), implementing the +// "manual RDMA token" pattern (cuObjClient API spec section 1.12.4). cuObject +// is a C++ library whose client object requires an I/O-ops callback table at +// construction, so it cannot be loaded directly through KvikIO's dlopen shim +// (which resolves plain C symbols, like cuFileAPI does for libcufile). This +// thunk exposes a small C ABI that KvikIO's cuObjAPI dlopens at runtime; only +// this translation unit is compiled against cuobjclient.h and links +// libcuobjclient, so the main KvikIO library keeps cuObject as an optional +// runtime dependency. +// +// Built into libkvikio_cuobj_shim.so only when cuobjclient.h is found (see +// cpp/CMakeLists.txt). Point KVIKIO_CUOBJ_SHIM at the result, or install it on +// the loader path. + +#include + +#include +#include +#include + +namespace { + +// cuObject get/put callbacks are unused in the manual-token pattern (KvikIO +// drives the S3 request out of band over libcurl), but the cuObjClient +// constructor requires an ops table. Provide stubs. +ssize_t cuobj_stub_get(const void* /*handle*/, + char* /*ptr*/, + size_t /*size*/, + loff_t /*offset*/, + const cufileRDMAInfo_t*) +{ + return -EOPNOTSUPP; +} + +ssize_t cuobj_stub_put(const void* /*handle*/, + const char* /*ptr*/, + size_t /*size*/, + loff_t /*offset*/, + const cufileRDMAInfo_t*) +{ + return -EOPNOTSUPP; +} + +cuObjClient* get_client() +{ + static CUObjOps_t ops = {cuobj_stub_get, cuobj_stub_put}; + static cuObjClient client(ops); + return &client; +} + +// Descriptors returned by cuMemObjGetRDMAToken are owned by cuObject and must +// be released via cuMemObjPutRDMAToken. Keep the original pointer keyed by +// descriptor value so the caller can free it by the C string it received, +// without owning the lifetime. +std::mutex& token_mutex() +{ + static std::mutex m; + return m; +} + +std::unordered_map& token_registry() +{ + static std::unordered_map r; + return r; +} + +} // namespace + +extern "C" { + +int kvikio_cuobj_available() { return get_client()->isConnected() ? 1 : 0; } + +int kvikio_cuobj_register_buffer(void* ptr, size_t size) +{ + return get_client()->cuMemObjGetDescriptor(ptr, size) == CU_OBJ_SUCCESS ? 0 : -1; +} + +int kvikio_cuobj_deregister_buffer(void* ptr) +{ + return get_client()->cuMemObjPutDescriptor(ptr) == CU_OBJ_SUCCESS ? 0 : -1; +} + +// Returns the descriptor string for [offset, offset+size) of the registered +// buffer, or nullptr on failure. is_put selects PUT (1) vs GET (0). The caller +// must release it via kvikio_cuobj_put_rdma_token after the request completes. +const char* kvikio_cuobj_get_rdma_token(void* ptr, size_t size, size_t offset, int is_put) +{ + char* desc = nullptr; + cuObjOpType_t op = is_put ? CUOBJ_PUT : CUOBJ_GET; + cuObjErr_t const stat = get_client()->cuMemObjGetRDMAToken(ptr, size, offset, op, &desc); + if (stat != CU_OBJ_SUCCESS || desc == nullptr) { return nullptr; } + std::lock_guard lock(token_mutex()); + token_registry()[std::string(desc)] = desc; + return desc; +} + +int kvikio_cuobj_put_rdma_token(const char* token) +{ + if (token == nullptr) { return -1; } + char* desc = nullptr; + { + std::lock_guard lock(token_mutex()); + auto it = token_registry().find(std::string(token)); + if (it == token_registry().end()) { return 0; } + desc = it->second; + token_registry().erase(it); + } + return get_client()->cuMemObjPutRDMAToken(desc) == CU_OBJ_SUCCESS ? 0 : -1; +} + +} // extern "C" diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index a14d3a1415..3ea37c7998 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -1,6 +1,6 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 + * reserved. reserved. reserved. reserved. reserved. reserved. SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -136,6 +136,35 @@ class RemoteEndpoint { * @return The type of the remote file. */ [[nodiscard]] RemoteEndpointType remote_endpoint_type() const noexcept; + + /** + * @brief Whether this endpoint can use the cuObject S3-over-RDMA data plane. + * + * Default is `false`. An endpoint returns `true` only when RDMA is both + * requested and usable (an RDMA-capable server plus a working cuObject + * runtime). When `true`, `RemoteHandle::read()` registers the destination + * buffer with cuObject and transfers the payload directly over RDMA instead + * of streaming the HTTP body. + * + * @return Boolean answer. + */ + [[nodiscard]] virtual bool supports_rdma() const noexcept; + + /** + * @brief Configure a curl handle for an RDMA range request. + * + * Sets the same connection options as `setopt()` plus the signed + * `x-amz-rdma-token` header carrying `rdma_token`. The token must be signed + * with the rest of the request (SigV4 signs `x-amz-*` headers), so it is + * added before signing. + * + * @param curl The curl handle. + * @param rdma_token The cuObject RDMA descriptor (formatted + * `::`). + * @return An owned `curl_slist` that the caller must free with + * `curl_slist_free_all()` after the transfer completes. + */ + virtual curl_slist* setopt_rdma(CurlHandle& curl, std::string const& rdma_token); }; /** @@ -266,6 +295,15 @@ class S3Endpoint : public RemoteEndpoint { std::size_t get_file_size() override; void setup_range_request(CurlHandle& curl, std::size_t file_offset, std::size_t size) override; + /** + * @brief Whether S3-over-RDMA is usable for this endpoint. + * + * Returns `true` when the `KVIKIO_REMOTE_RDMA` environment variable is set to + * a truthy value and the cuObject runtime is available (`is_cuobj_available()`). + */ + [[nodiscard]] bool supports_rdma() const noexcept override; + curl_slist* setopt_rdma(CurlHandle& curl, std::string const& rdma_token) override; + /** * @brief Whether the given URL is valid for S3 endpoints (excluding presigned URL). * diff --git a/cpp/include/kvikio/shim/cuobj.hpp b/cpp/include/kvikio/shim/cuobj.hpp new file mode 100644 index 0000000000..bd5e96c3a2 --- /dev/null +++ b/cpp/include/kvikio/shim/cuobj.hpp @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +#include + +namespace kvikio { + +/** + * @brief Shim layer over the NVIDIA cuObject C-ABI used for S3-over-RDMA. + * + * Singleton that `dlopen`s ``libkvikio_cuobj_shim.so`` on construction and + * resolves its C entry points. The shim wraps the cuObject ``cuObjClient`` C++ + * class (see ``cpp/cuobj_shim/cuobj_shim.cpp``); loading it at runtime keeps + * cuObject an optional dependency, mirroring how :class:`cuFileAPI` loads + * ``libcufile.so.0``. The library path can be overridden with the + * ``KVIKIO_CUOBJ_SHIM`` environment variable. + */ +class cuObjAPI { + public: + int (*Available)(){nullptr}; + int (*RegisterBuffer)(void*, std::size_t){nullptr}; + int (*DeregisterBuffer)(void*){nullptr}; + char const* (*GetRDMAToken)(void*, std::size_t, std::size_t, int){nullptr}; + int (*PutRDMAToken)(char const*){nullptr}; + + private: + cuObjAPI(); + + public: + cuObjAPI(cuObjAPI const&) = delete; + cuObjAPI& operator=(cuObjAPI const&) = delete; + + KVIKIO_EXPORT static cuObjAPI& instance(); +}; + +/** + * @brief Whether cuObject S3-over-RDMA is usable. + * + * ``true`` only when the shim library loads and a cuObject client connection + * can be established (RDMA-capable NIC and a reachable RDMA S3 endpoint). + */ +[[nodiscard]] bool is_cuobj_available() noexcept; + +} // namespace kvikio diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 82d5cfec4f..b98747e2c5 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -1,13 +1,17 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 + * reserved. reserved. reserved. reserved. reserved. reserved. SPDX-License-Identifier: Apache-2.0 */ #include #include #include +#include #include +#include +#include #include +#include #include #include #include @@ -27,6 +31,7 @@ #include #include #include +#include #include #include @@ -322,6 +327,14 @@ RemoteEndpointType RemoteEndpoint::remote_endpoint_type() const noexcept return _remote_endpoint_type; } +bool RemoteEndpoint::supports_rdma() const noexcept { return false; } + +curl_slist* RemoteEndpoint::setopt_rdma(CurlHandle& /*curl*/, std::string const& /*rdma_token*/) +{ + KVIKIO_FAIL("This remote endpoint does not support RDMA", std::runtime_error); + return nullptr; +} + HttpEndpoint::HttpEndpoint(std::string url) : RemoteEndpoint{RemoteEndpointType::HTTP}, _url{std::move(url)} { @@ -365,6 +378,36 @@ void S3Endpoint::setopt(CurlHandle& curl) if (_curl_header_list) { curl.setopt(CURLOPT_HTTPHEADER, _curl_header_list); } } +bool S3Endpoint::supports_rdma() const noexcept +{ + char const* env = std::getenv("KVIKIO_REMOTE_RDMA"); + if (env == nullptr) { return false; } + std::string value{env}; + std::transform( + value.begin(), value.end(), value.begin(), [](unsigned char c) { return std::tolower(c); }); + bool const enabled = (value == "1" || value == "on" || value == "true" || value == "yes"); + return enabled && is_cuobj_available(); +} + +curl_slist* S3Endpoint::setopt_rdma(CurlHandle& curl, std::string const& rdma_token) +{ + // Reuse the standard connection options (URL, SigV4, userpwd, base headers), + // then replace the header list with a fresh copy that also carries the signed + // RDMA token. A per-call list keeps this thread-safe: a single endpoint is + // shared across the worker threads of a parallel `pread()`. + setopt(curl); + + curl_slist* list = nullptr; + for (curl_slist* node = _curl_header_list; node != nullptr; node = node->next) { + list = curl_slist_append(list, node->data); + } + std::string const header = "x-amz-rdma-token: " + rdma_token; + list = curl_slist_append(list, header.c_str()); + KVIKIO_EXPECT(list != nullptr, "Failed to build curl header for RDMA token", std::runtime_error); + curl.setopt(CURLOPT_HTTPHEADER, list); + return list; +} + std::string S3Endpoint::url_from_bucket_and_object(std::string bucket_name, std::string object_name, std::optional aws_region, @@ -754,6 +797,99 @@ std::size_t callback_device_memory(char* data, std::size_t size, std::size_t nme ctx->offset += nbytes; return nbytes; } + +// In the cuObject RDMA data plane the payload is delivered out of band straight +// into the registered buffer; only an error body (e.g. an S3 XML error) should +// ever reach this write callback, so it is discarded. +std::size_t callback_rdma_discard(char* /*data*/, std::size_t size, std::size_t nmemb, void*) +{ + return size * nmemb; +} + +// Captures the `x-amz-rdma-reply` response header so the caller can confirm the +// endpoint honored the RDMA request. +struct RdmaReplyContext { + std::string reply; +}; + +std::size_t callback_rdma_reply(char* buffer, std::size_t size, std::size_t nitems, void* userdata) +{ + std::size_t const nbytes = size * nitems; + auto* ctx = reinterpret_cast(userdata); + std::string line{buffer, nbytes}; + auto const colon = line.find(':'); + if (colon != std::string::npos) { + std::string name = line.substr(0, colon); + std::transform( + name.begin(), name.end(), name.begin(), [](unsigned char c) { return std::tolower(c); }); + if (name == "x-amz-rdma-reply") { + std::string value = line.substr(colon + 1); + auto const begin = value.find_first_not_of(" \t"); + auto const end = value.find_last_not_of(" \t\r\n"); + ctx->reply = (begin == std::string::npos) ? "" : value.substr(begin, end - begin + 1); + } + } + return nbytes; +} + +// Single-shot RDMA read: register `buf` with cuObject, mint a descriptor for +// `[0, size)`, issue a body-less range GET carrying the signed +// `x-amz-rdma-token`, and let the endpoint RDMA-write the payload directly into +// `buf`. Works for both host and device buffers (cuObject accepts either). +std::size_t read_rdma( + RemoteEndpoint& endpoint, void* buf, std::size_t size, std::size_t file_offset, bool is_host_mem) +{ + KVIKIO_NVTX_FUNC_RANGE(size); + auto& api = cuObjAPI::instance(); + + KVIKIO_EXPECT(api.RegisterBuffer(buf, size) == 0, + "cuObject failed to register the destination buffer for RDMA", + std::runtime_error); + + char const* descriptor = nullptr; + curl_slist* header_list = nullptr; + try { + descriptor = api.GetRDMAToken(buf, size, 0, /*is_put=*/0); + KVIKIO_EXPECT( + descriptor != nullptr, "cuObject failed to mint an RDMA token", std::runtime_error); + + auto const addr = reinterpret_cast(buf); + std::stringstream token; + token << descriptor << ":" << std::hex << std::setw(16) << std::setfill('0') << addr << ":" + << std::setw(16) << std::setfill('0') << size; + + auto curl = create_curl_handle(); + header_list = endpoint.setopt_rdma(curl, token.str()); + endpoint.setup_range_request(curl, file_offset, size); + + RdmaReplyContext reply{}; + curl.setopt(CURLOPT_WRITEFUNCTION, callback_rdma_discard); + curl.setopt(CURLOPT_HEADERFUNCTION, callback_rdma_reply); + curl.setopt(CURLOPT_HEADERDATA, &reply); + + if (is_host_mem) { + curl.perform(); + } else { + PushAndPopContext c(get_context_from_pointer(buf)); + curl.perform(); + } + + KVIKIO_EXPECT(!reply.reply.empty() && reply.reply != "501", + "S3 endpoint declined RDMA (x-amz-rdma-reply='" + reply.reply + + "'); the endpoint is not RDMA-capable. Unset KVIKIO_REMOTE_RDMA to use the " + "standard data plane.", + std::runtime_error); + } catch (...) { + if (descriptor != nullptr) { api.PutRDMAToken(descriptor); } + if (header_list != nullptr) { curl_slist_free_all(header_list); } + api.DeregisterBuffer(buf); + throw; + } + api.PutRDMAToken(descriptor); + curl_slist_free_all(header_list); + api.DeregisterBuffer(buf); + return size; +} } // namespace detail std::size_t RemoteHandle::read(void* buf, std::size_t size, std::size_t file_offset) @@ -769,7 +905,12 @@ std::size_t RemoteHandle::read(void* buf, std::size_t size, std::size_t file_off KVIKIO_FAIL(ss.str(), std::invalid_argument); } bool const is_host_mem = is_host_memory(buf); - auto curl = create_curl_handle(); + + if (_endpoint->supports_rdma()) { + return detail::read_rdma(*_endpoint, buf, size, file_offset, is_host_mem); + } + + auto curl = create_curl_handle(); _endpoint->setopt(curl); _endpoint->setup_range_request(curl, file_offset, size); diff --git a/cpp/src/shim/cuobj.cpp b/cpp/src/shim/cuobj.cpp new file mode 100644 index 0000000000..50ca6cdc5a --- /dev/null +++ b/cpp/src/shim/cuobj.cpp @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include + +namespace kvikio { + +namespace { +std::string cuobj_shim_library_name() +{ + if (char const* env = std::getenv("KVIKIO_CUOBJ_SHIM"); env != nullptr && *env != '\0') { + return std::string{env}; + } + return "libkvikio_cuobj_shim.so"; +} +} // namespace + +cuObjAPI::cuObjAPI() +{ + void* lib = load_library(cuobj_shim_library_name()); + get_symbol(Available, lib, "kvikio_cuobj_available"); + get_symbol(RegisterBuffer, lib, "kvikio_cuobj_register_buffer"); + get_symbol(DeregisterBuffer, lib, "kvikio_cuobj_deregister_buffer"); + get_symbol(GetRDMAToken, lib, "kvikio_cuobj_get_rdma_token"); + get_symbol(PutRDMAToken, lib, "kvikio_cuobj_put_rdma_token"); +} + +cuObjAPI& cuObjAPI::instance() +{ + static cuObjAPI _instance; + return _instance; +} + +bool is_cuobj_available() noexcept +{ + try { + return cuObjAPI::instance().Available() != 0; + } catch (std::runtime_error const&) { + return false; + } +} + +} // namespace kvikio diff --git a/docs/source/remote_file.rst b/docs/source/remote_file.rst index e6d038035e..7dbd2a1254 100644 --- a/docs/source/remote_file.rst +++ b/docs/source/remote_file.rst @@ -9,6 +9,25 @@ Example .. literalinclude:: ../../python/kvikio/examples/http_io.py :language: python +S3-over-RDMA (NVIDIA cuObject) +------------------------------ + +For RDMA-capable S3 endpoints (such as MinIO AIStor), KvikIO can transfer object payloads directly +into the destination buffer over RDMA using NVIDIA cuObject, bypassing the HTTP body and the CPU copy. +This is an opt-in data plane for the S3 endpoint: + +- Build KvikIO on a host where the cuObject SDK (``cuobjclient.h`` and ``libcuobjclient``, shipped with + the CUDA Toolkit) is present. CMake then builds ``libkvikio_cuobj_shim.so`` automatically. +- At runtime, set ``KVIKIO_REMOTE_RDMA=on``. If the shim cannot be found on the loader path, point + ``KVIKIO_CUOBJ_SHIM`` at it. cuObject also needs a cuFile/cuObject JSON config describing the RDMA NIC + (``CUFILE_ENV_PATH_JSON``). + +When enabled and a cuObject connection is established, ``RemoteHandle.read``/``pread`` register the +destination buffer (host or device memory) with cuObject and issue a body-less range request carrying +the signed ``x-amz-rdma-token`` header; the endpoint RDMA-writes the payload into the buffer. If the +endpoint does not honor the request (``x-amz-rdma-reply: 501``), the read fails rather than silently +falling back, so only enable it against an RDMA-capable endpoint. + AWS S3 object naming requirement --------------------------------