Add optional S3-over-RDMA (NVIDIA cuObject) data plane to remote I/O

cpp/CMakeLists.txt

@@ -148,6 +148,22 @@ else()
   message(STATUS "Found cuFile Version API: ${cuFile_VERSION_API_FOUND}")
 endif()
 
+# cuObject (libcuobjclient) powers the optional S3-over-RDMA data plane. The header and library ship
+# with the CUDA Toolkit. KvikIO never links them directly; instead a small C-ABI shim
+# (libkvikio_cuobj_shim.so) is built when they are present and loaded at runtime via dlopen, keeping
+# cuObject an optional runtime dependency (same model as cuFile).
+find_path(
+  cuObj_INCLUDE_DIR cuobjclient.h HINTS ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_TARGET_DIR}/include
+)
+find_library(cuObj_LIBRARY cuobjclient HINTS ${CUDAToolkit_LIBRARY_DIR} ${CUDAToolkit_TARGET_DIR}/lib64)
+if(cuObj_INCLUDE_DIR AND cuObj_LIBRARY)
+  set(cuObj_FOUND 1)
+  message(STATUS "Found cuObject: ${cuObj_LIBRARY}")
+else()
+  set(cuObj_FOUND 0)
+  message(STATUS "Cannot find cuObject - KvikIO will build without S3-over-RDMA support")
+endif()
+
 include(cmake/thirdparty/get_thread_pool.cmake)
 
 # ##################################################################################################
@@ -193,6 +209,7 @@ if(KvikIO_REMOTE_SUPPORT)
     "src/detail/remote_callback.cpp"
     "src/detail/tls.cpp"
     "src/detail/url.cpp"
+    "src/shim/cuobj.cpp"
     "src/shim/libcurl.cpp"
   )
 endif()
@@ -290,6 +307,25 @@ install(
   EXPORT kvikio-exports
 )
 
+# Optional C-ABI shim wrapping cuObject's C++ client. Built only when cuObject is available and
+# loaded at runtime via dlopen (see src/shim/cuobj.cpp); KvikIO itself never links cuObject.
+if(KvikIO_REMOTE_SUPPORT AND cuObj_FOUND)
+  add_library(kvikio_cuobj_shim SHARED "cuobj_shim/cuobj_shim.cpp")
+  target_include_directories(
+    kvikio_cuobj_shim PRIVATE ${cuObj_INCLUDE_DIR} ${CUDAToolkit_INCLUDE_DIRS}
+  )
+  target_link_libraries(kvikio_cuobj_shim PRIVATE ${cuObj_LIBRARY})
+  set_target_properties(
+    kvikio_cuobj_shim
+    PROPERTIES BUILD_RPATH "\$ORIGIN"
+               INSTALL_RPATH "\$ORIGIN"
+               CXX_STANDARD 20
+               CXX_STANDARD_REQUIRED ON
+               POSITION_INDEPENDENT_CODE ON
+  )
+  install(TARGETS kvikio_cuobj_shim DESTINATION ${lib_dir})
+endif()
+
 install(DIRECTORY include/kvikio/ DESTINATION include/kvikio)
 install(FILES ${KvikIO_BINARY_DIR}/include/kvikio/version_config.hpp DESTINATION include/kvikio)
 

cpp/cuobj_shim/cuobj_shim.cpp

@@ -0,0 +1,116 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// extern "C" shim over NVIDIA cuObject (libcuobjclient), implementing the
+// "manual RDMA token" pattern (cuObjClient API spec section 1.12.4). cuObject
+// is a C++ library whose client object requires an I/O-ops callback table at
+// construction, so it cannot be loaded directly through KvikIO's dlopen shim
+// (which resolves plain C symbols, like cuFileAPI does for libcufile). This
+// thunk exposes a small C ABI that KvikIO's cuObjAPI dlopens at runtime; only
+// this translation unit is compiled against cuobjclient.h and links
+// libcuobjclient, so the main KvikIO library keeps cuObject as an optional
+// runtime dependency.
+//
+// Built into libkvikio_cuobj_shim.so only when cuobjclient.h is found (see
+// cpp/CMakeLists.txt). Point KVIKIO_CUOBJ_SHIM at the result, or install it on
+// the loader path.
+
+#include <cuobjclient.h>
+
+#include <mutex>
+#include <string>
+#include <unordered_map>
+
+namespace {
+
+// cuObject get/put callbacks are unused in the manual-token pattern (KvikIO
+// drives the S3 request out of band over libcurl), but the cuObjClient
+// constructor requires an ops table. Provide stubs.
+ssize_t cuobj_stub_get(const void* /*handle*/,
+                       char* /*ptr*/,
+                       size_t /*size*/,
+                       loff_t /*offset*/,
+                       const cufileRDMAInfo_t*)
+{
+  return -EOPNOTSUPP;
+}
+
+ssize_t cuobj_stub_put(const void* /*handle*/,
+                       const char* /*ptr*/,
+                       size_t /*size*/,
+                       loff_t /*offset*/,
+                       const cufileRDMAInfo_t*)
+{
+  return -EOPNOTSUPP;
+}
+
+cuObjClient* get_client()
+{
+  static CUObjOps_t ops = {cuobj_stub_get, cuobj_stub_put};
+  static cuObjClient client(ops);
+  return &client;
+}
+
+// Descriptors returned by cuMemObjGetRDMAToken are owned by cuObject and must
+// be released via cuMemObjPutRDMAToken. Keep the original pointer keyed by
+// descriptor value so the caller can free it by the C string it received,
+// without owning the lifetime.
+std::mutex& token_mutex()
+{
+  static std::mutex m;
+  return m;
+}
+
+std::unordered_map<std::string, char*>& token_registry()
+{
+  static std::unordered_map<std::string, char*> r;
+  return r;
+}
+
+}  // namespace
+
+extern "C" {
+
+int kvikio_cuobj_available() { return get_client()->isConnected() ? 1 : 0; }
+
+int kvikio_cuobj_register_buffer(void* ptr, size_t size)
+{
+  return get_client()->cuMemObjGetDescriptor(ptr, size) == CU_OBJ_SUCCESS ? 0 : -1;
+}
+
+int kvikio_cuobj_deregister_buffer(void* ptr)
+{
+  return get_client()->cuMemObjPutDescriptor(ptr) == CU_OBJ_SUCCESS ? 0 : -1;
+}
+
+// Returns the descriptor string for [offset, offset+size) of the registered
+// buffer, or nullptr on failure. is_put selects PUT (1) vs GET (0). The caller
+// must release it via kvikio_cuobj_put_rdma_token after the request completes.
+const char* kvikio_cuobj_get_rdma_token(void* ptr, size_t size, size_t offset, int is_put)
+{
+  char* desc            = nullptr;
+  cuObjOpType_t op      = is_put ? CUOBJ_PUT : CUOBJ_GET;
+  cuObjErr_t const stat = get_client()->cuMemObjGetRDMAToken(ptr, size, offset, op, &desc);
+  if (stat != CU_OBJ_SUCCESS || desc == nullptr) { return nullptr; }
+  std::lock_guard<std::mutex> lock(token_mutex());
+  token_registry()[std::string(desc)] = desc;
+  return desc;
+}
+
+int kvikio_cuobj_put_rdma_token(const char* token)
+{
+  if (token == nullptr) { return -1; }
+  char* desc = nullptr;
+  {
+    std::lock_guard<std::mutex> lock(token_mutex());
+    auto it = token_registry().find(std::string(token));
+    if (it == token_registry().end()) { return 0; }
+    desc = it->second;
+    token_registry().erase(it);
+  }
+  return get_client()->cuMemObjPutRDMAToken(desc) == CU_OBJ_SUCCESS ? 0 : -1;
+}
+
+}  // extern "C"

cpp/include/kvikio/remote_handle.hpp

@@ -1,6 +1,6 @@
 /*
  * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION.
- * SPDX-License-Identifier: Apache-2.0
+ * reserved. reserved. reserved. reserved. reserved. reserved. SPDX-License-Identifier: Apache-2.0
  */
 #pragma once
 
@@ -136,6 +136,35 @@ class RemoteEndpoint {
    * @return The type of the remote file.
    */
   [[nodiscard]] RemoteEndpointType remote_endpoint_type() const noexcept;
+
+  /**
+   * @brief Whether this endpoint can use the cuObject S3-over-RDMA data plane.
+   *
+   * Default is `false`. An endpoint returns `true` only when RDMA is both
+   * requested and usable (an RDMA-capable server plus a working cuObject
+   * runtime). When `true`, `RemoteHandle::read()` registers the destination
+   * buffer with cuObject and transfers the payload directly over RDMA instead
+   * of streaming the HTTP body.
+   *
+   * @return Boolean answer.
+   */
+  [[nodiscard]] virtual bool supports_rdma() const noexcept;
+
+  /**
+   * @brief Configure a curl handle for an RDMA range request.
+   *
+   * Sets the same connection options as `setopt()` plus the signed
+   * `x-amz-rdma-token` header carrying `rdma_token`. The token must be signed
+   * with the rest of the request (SigV4 signs `x-amz-*` headers), so it is
+   * added before signing.
+   *
+   * @param curl The curl handle.
+   * @param rdma_token The cuObject RDMA descriptor (formatted
+   * `<descriptor>:<hex addr>:<hex size>`).
+   * @return An owned `curl_slist` that the caller must free with
+   * `curl_slist_free_all()` after the transfer completes.
+   */
+  virtual curl_slist* setopt_rdma(CurlHandle& curl, std::string const& rdma_token);
 };
 
 /**
@@ -266,6 +295,15 @@ class S3Endpoint : public RemoteEndpoint {
   std::size_t get_file_size() override;
   void setup_range_request(CurlHandle& curl, std::size_t file_offset, std::size_t size) override;
 
+  /**
+   * @brief Whether S3-over-RDMA is usable for this endpoint.
+   *
+   * Returns `true` when the `KVIKIO_REMOTE_RDMA` environment variable is set to
+   * a truthy value and the cuObject runtime is available (`is_cuobj_available()`).
+   */
+  [[nodiscard]] bool supports_rdma() const noexcept override;
+  curl_slist* setopt_rdma(CurlHandle& curl, std::string const& rdma_token) override;
+
   /**
    * @brief Whether the given URL is valid for S3 endpoints (excluding presigned URL).
    *

cpp/include/kvikio/shim/cuobj.hpp

@@ -0,0 +1,50 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#pragma once
+
+#include <cstddef>
+#include <string>
+
+#include <kvikio/shim/utils.hpp>
+
+namespace kvikio {
+
+/**
+ * @brief Shim layer over the NVIDIA cuObject C-ABI used for S3-over-RDMA.
+ *
+ * Singleton that `dlopen`s ``libkvikio_cuobj_shim.so`` on construction and
+ * resolves its C entry points. The shim wraps the cuObject ``cuObjClient`` C++
+ * class (see ``cpp/cuobj_shim/cuobj_shim.cpp``); loading it at runtime keeps
+ * cuObject an optional dependency, mirroring how :class:`cuFileAPI` loads
+ * ``libcufile.so.0``. The library path can be overridden with the
+ * ``KVIKIO_CUOBJ_SHIM`` environment variable.
+ */
+class cuObjAPI {
+ public:
+  int (*Available)(){nullptr};
+  int (*RegisterBuffer)(void*, std::size_t){nullptr};
+  int (*DeregisterBuffer)(void*){nullptr};
+  char const* (*GetRDMAToken)(void*, std::size_t, std::size_t, int){nullptr};
+  int (*PutRDMAToken)(char const*){nullptr};
+
+ private:
+  cuObjAPI();
+
+ public:
+  cuObjAPI(cuObjAPI const&)            = delete;
+  cuObjAPI& operator=(cuObjAPI const&) = delete;
+
+  KVIKIO_EXPORT static cuObjAPI& instance();
+};
+
+/**
+ * @brief Whether cuObject S3-over-RDMA is usable.
+ *
+ * ``true`` only when the shim library loads and a cuObject client connection
+ * can be established (RDMA-capable NIC and a reachable RDMA S3 endpoint).
+ */
+[[nodiscard]] bool is_cuobj_available() noexcept;
+
+}  // namespace kvikio