From c8d1816bcce41bfbd0914bd6d6fccec82125a2cd Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 17 Nov 2025 06:47:57 +0000 Subject: [PATCH 01/31] Add per-file-handle pool --- .../kvikio/detail/parallel_operation.hpp | 29 ++++++++------- cpp/include/kvikio/file_handle.hpp | 2 ++ cpp/src/file_handle.cpp | 35 +++++++++++++++---- cpp/src/mmap.cpp | 1 + cpp/src/remote_handle.cpp | 3 +- 5 files changed, 50 insertions(+), 20 deletions(-) diff --git a/cpp/include/kvikio/detail/parallel_operation.hpp b/cpp/include/kvikio/detail/parallel_operation.hpp index a4489da8e5..b395e6b20d 100644 --- a/cpp/include/kvikio/detail/parallel_operation.hpp +++ b/cpp/include/kvikio/detail/parallel_operation.hpp @@ -18,6 +18,7 @@ #include #include #include +#include "kvikio/threadpool_wrapper.hpp" namespace kvikio { @@ -75,8 +76,9 @@ std::future submit_task(F op, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset, - std::uint64_t nvtx_payload = 0ull, - nvtx_color_type nvtx_color = NvtxManager::default_color()) + BS_thread_pool* thread_pool = &defaults::thread_pool(), + std::uint64_t nvtx_payload = 0ull, + nvtx_color_type nvtx_color = NvtxManager::default_color()) { static_assert(std::is_invocable_r_v submit_task(F op, decltype(file_offset), decltype(devPtr_offset)>); - return defaults::thread_pool().submit_task([=] { + return thread_pool->submit_task([=] { KVIKIO_NVTX_SCOPED_RANGE("task", nvtx_payload, nvtx_color); return op(buf, size, file_offset, devPtr_offset); }); @@ -101,12 +103,13 @@ std::future submit_task(F op, template std::future submit_move_only_task( F op_move_only, - std::uint64_t nvtx_payload = 0ull, - nvtx_color_type nvtx_color = NvtxManager::default_color()) + BS_thread_pool* thread_pool = &defaults::thread_pool(), + std::uint64_t nvtx_payload = 0ull, + nvtx_color_type nvtx_color = NvtxManager::default_color()) { static_assert(std::is_invocable_r_v); auto op_copyable = make_copyable_lambda(std::move(op_move_only)); - return defaults::thread_pool().submit_task([=] { + return thread_pool->submit_task([=] { KVIKIO_NVTX_SCOPED_RANGE("task", nvtx_payload, nvtx_color); return op_copyable(); }); @@ -133,8 +136,9 @@ std::future parallel_io(F op, std::size_t file_offset, std::size_t task_size, std::size_t devPtr_offset, - std::uint64_t call_idx = 0, - nvtx_color_type nvtx_color = NvtxManager::default_color()) + BS_thread_pool* thread_pool = &defaults::thread_pool(), + std::uint64_t call_idx = 0, + nvtx_color_type nvtx_color = NvtxManager::default_color()) { KVIKIO_EXPECT(task_size > 0, "`task_size` must be positive", std::invalid_argument); static_assert(std::is_invocable_r_v parallel_io(F op, // Single-task guard if (task_size >= size || get_page_size() >= size) { - return detail::submit_task(op, buf, size, file_offset, devPtr_offset, call_idx, nvtx_color); + return detail::submit_task( + op, buf, size, file_offset, devPtr_offset, thread_pool, call_idx, nvtx_color); } std::vector> tasks; @@ -154,8 +159,8 @@ std::future parallel_io(F op, // 1) Submit all tasks but the last one. These are all `task_size` sized tasks. while (size > task_size) { - tasks.push_back( - detail::submit_task(op, buf, task_size, file_offset, devPtr_offset, call_idx, nvtx_color)); + tasks.push_back(detail::submit_task( + op, buf, task_size, file_offset, devPtr_offset, thread_pool, call_idx, nvtx_color)); file_offset += task_size; devPtr_offset += task_size; size -= task_size; @@ -170,7 +175,7 @@ std::future parallel_io(F op, } return ret; }; - return detail::submit_move_only_task(std::move(last_task), call_idx, nvtx_color); + return detail::submit_move_only_task(std::move(last_task), thread_pool, call_idx, nvtx_color); } } // namespace kvikio diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index e74b8e3e20..fd49657c4d 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -40,6 +40,8 @@ class FileHandle { CompatModeManager _compat_mode_manager; friend class CompatModeManager; + std::unique_ptr _thread_pool; + public: // 644 is a common setting of Unix file permissions: read and write for owner, read-only for group // and others. diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index 30f1cf335a..06724a7017 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace kvikio { @@ -29,6 +31,7 @@ FileHandle::FileHandle(std::string const& file_path, : _initialized{true}, _compat_mode_manager{file_path, flags, mode, compat_mode, this} { KVIKIO_NVTX_FUNC_RANGE(); + _thread_pool = std::make_unique(defaults::thread_pool_nthreads()); } FileHandle::FileHandle(FileHandle&& o) noexcept @@ -37,7 +40,8 @@ FileHandle::FileHandle(FileHandle&& o) noexcept _initialized{std::exchange(o._initialized, false)}, _nbytes{std::exchange(o._nbytes, 0)}, _cufile_handle{std::exchange(o._cufile_handle, {})}, - _compat_mode_manager{std::move(o._compat_mode_manager)} + _compat_mode_manager{std::move(o._compat_mode_manager)}, + _thread_pool{std::move(o._thread_pool)} { } @@ -49,6 +53,7 @@ FileHandle& FileHandle::operator=(FileHandle&& o) noexcept _nbytes = std::exchange(o._nbytes, 0); _cufile_handle = std::exchange(o._cufile_handle, {}); _compat_mode_manager = std::move(o._compat_mode_manager); + _thread_pool = std::move(o._thread_pool); return *this; } @@ -162,7 +167,8 @@ std::future FileHandle::pread(void* buf, _file_direct_off.fd(), buf, size, file_offset, _file_direct_on.fd()); }; - return parallel_io(op, buf, size, file_offset, task_size, 0, call_idx, nvtx_color); + return parallel_io( + op, buf, size, file_offset, task_size, 0, _thread_pool.get(), call_idx, nvtx_color); } CUcontext ctx = get_context_from_pointer(buf); @@ -192,8 +198,15 @@ std::future FileHandle::pread(void* buf, return read(devPtr_base, size, file_offset, devPtr_offset, /* sync_default_stream = */ false); }; auto [devPtr_base, base_size, devPtr_offset] = get_alloc_info(buf, &ctx); - return parallel_io( - task, devPtr_base, size, file_offset, task_size, devPtr_offset, call_idx, nvtx_color); + return parallel_io(task, + devPtr_base, + size, + file_offset, + task_size, + devPtr_offset, + _thread_pool.get(), + call_idx, + nvtx_color); } std::future FileHandle::pwrite(void const* buf, @@ -215,7 +228,8 @@ std::future FileHandle::pwrite(void const* buf, _file_direct_off.fd(), buf, size, file_offset, _file_direct_on.fd()); }; - return parallel_io(op, buf, size, file_offset, task_size, 0, call_idx, nvtx_color); + return parallel_io( + op, buf, size, file_offset, task_size, 0, _thread_pool.get(), call_idx, nvtx_color); } CUcontext ctx = get_context_from_pointer(buf); @@ -245,8 +259,15 @@ std::future FileHandle::pwrite(void const* buf, return write(devPtr_base, size, file_offset, devPtr_offset, /* sync_default_stream = */ false); }; auto [devPtr_base, base_size, devPtr_offset] = get_alloc_info(buf, &ctx); - return parallel_io( - op, devPtr_base, size, file_offset, task_size, devPtr_offset, call_idx, nvtx_color); + return parallel_io(op, + devPtr_base, + size, + file_offset, + task_size, + devPtr_offset, + _thread_pool.get(), + call_idx, + nvtx_color); } void FileHandle::read_async(void* devPtr_base, diff --git a/cpp/src/mmap.cpp b/cpp/src/mmap.cpp index a720fa8929..f1e801f943 100644 --- a/cpp/src/mmap.cpp +++ b/cpp/src/mmap.cpp @@ -448,6 +448,7 @@ std::future MmapHandle::pread(void* buf, offset, task_size, 0, // dst buffer offset initial value + &defaults::thread_pool(), call_idx, nvtx_color); } diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 6004515b76..7217294422 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -819,7 +819,8 @@ std::future RemoteHandle::pread(void* buf, std::size_t devPtr_offset) -> std::size_t { return read(static_cast(devPtr_base) + devPtr_offset, size, file_offset); }; - return parallel_io(task, buf, size, file_offset, task_size, 0, call_idx, nvtx_color); + return parallel_io( + task, buf, size, file_offset, task_size, 0, &defaults::thread_pool(), call_idx, nvtx_color); } } // namespace kvikio From f65776cc41f392f3ce9d26ba0f5d5c6d6c491e44 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 18 Nov 2025 15:54:58 -0500 Subject: [PATCH 02/31] Add seq read benchmark --- cpp/benchmarks/CMakeLists.txt | 3 +++ cpp/benchmarks/local/posix_benchmark.cpp | 8 ++++++++ 2 files changed, 11 insertions(+) create mode 100644 cpp/benchmarks/local/posix_benchmark.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index c6e227558e..61f98ba939 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -60,3 +60,6 @@ function(kvikio_add_benchmark) endfunction() kvikio_add_benchmark(NAME THREADPOOL_BENCHMARK SOURCES "threadpool/threadpool_benchmark.cpp") + +kvikio_add_benchmark(NAME POSIX_BENCHMARK SOURCES "local/posix_benchmark.cpp") + diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp new file mode 100644 index 0000000000..b993156e2a --- /dev/null +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +int main() { return 0; } \ No newline at end of file From 8375846f21435447d135fe060eab52cea042dfe0 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Wed, 19 Nov 2025 15:56:03 -0500 Subject: [PATCH 03/31] Add seq read benchmark --- cpp/benchmarks/CMakeLists.txt | 3 +- cpp/benchmarks/local/posix_benchmark.cpp | 257 ++++++++++++++++++++++- cpp/benchmarks/local/posix_benchmark.hpp | 46 ++++ cpp/benchmarks/utils.cpp | 9 + cpp/benchmarks/utils.hpp | 7 + 5 files changed, 318 insertions(+), 4 deletions(-) create mode 100644 cpp/benchmarks/local/posix_benchmark.hpp create mode 100644 cpp/benchmarks/utils.cpp create mode 100644 cpp/benchmarks/utils.hpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 61f98ba939..36a4593aa2 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -61,5 +61,4 @@ endfunction() kvikio_add_benchmark(NAME THREADPOOL_BENCHMARK SOURCES "threadpool/threadpool_benchmark.cpp") -kvikio_add_benchmark(NAME POSIX_BENCHMARK SOURCES "local/posix_benchmark.cpp") - +kvikio_add_benchmark(NAME POSIX_BENCHMARK SOURCES "local/posix_benchmark.cpp" "utils.cpp") diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index b993156e2a..ad780aca4f 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -3,6 +3,259 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include +#include "posix_benchmark.hpp" -int main() { return 0; } \ No newline at end of file +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "kvikio/error.hpp" + +namespace kvikio::benchmark { + +// Helper to parse size strings like "1G", "500M" +std::size_t parse_size(std::string const& str) +{ + if (str.empty()) { throw std::invalid_argument("Empty size string"); } + + // Parse the numeric part + std::size_t pos{}; + double value{}; + try { + value = std::stod(str, &pos); + } catch (std::exception const& e) { + throw std::invalid_argument("Invalid size format: " + str); + } + + if (value < 0) { throw std::invalid_argument("Size cannot be negative"); } + + // Extract suffix (everything after the number) + auto suffix = str.substr(pos); + + // No suffix means raw bytes + if (suffix.empty()) { return static_cast(value); } + + // Normalize to uppercase for case-insensitive comparison + std::transform( + suffix.begin(), suffix.end(), suffix.begin(), [](unsigned char c) { return std::tolower(c); }); + + // All multipliers use 1024 (binary), not 1000 + std::size_t multiplier{1}; + + // Support both K/Ki, M/Mi, etc. as synonyms (all 1024-based) + if (suffix == "K" || suffix == "KI") { + multiplier = 1024ULL; + } else if (suffix == "M" || suffix == "MI") { + multiplier = 1024ULL * 1024; + } else if (suffix == "G" || suffix == "GI") { + multiplier = 1024ULL * 1024 * 1024; + } else if (suffix == "T" || suffix == "TI") { + multiplier = 1024ULL * 1024 * 1024 * 1024; + } else { + throw std::invalid_argument("Invalid size suffix: '" + suffix + + "' (use K/Ki, M/Mi, G/Gi, or T/Ti)"); + } + + return static_cast(value * multiplier); +} + +Config Config::parse_args(int argc, char** argv) +{ + Config config; + + static struct option long_options[] = {{"file", required_argument, 0, 'f'}, + {"size", required_argument, 0, 's'}, + {"threads", required_argument, 0, 't'}, + {"repetitions", required_argument, 0, 'r'}, + {"no-direct", no_argument, 0, 'D'}, + {"no-align", no_argument, 0, 'A'}, + {"drop-cache", no_argument, 0, 'c'}, + {"overwrite", no_argument, 0, 'w'}, + {"open-once", no_argument, 0, 'o'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0}}; + + int opt; + int option_index = 0; + + while ((opt = getopt_long(argc, argv, "f:s:t:r:DAcwoh", long_options, &option_index)) != -1) { + switch (opt) { + case 'f': config.filepaths.push_back(optarg); break; + case 's': + config.num_bytes = parse_size(optarg); // Helper to parse "1G", "500M", etc. + break; + case 't': config.num_threads = std::stoul(optarg); break; + case 'r': config.repetition = std::stoi(optarg); break; + case 'D': config.o_direct = false; break; + case 'A': config.align_buffer = false; break; + case 'c': config.drop_file_cache = true; break; + case 'w': config.overwrite_file = true; break; + case 'o': config.open_file_once = true; break; + case 'h': config.print_usage(argv[0]); std::exit(0); + default: config.print_usage(argv[0]); std::exit(1); + } + } + + // Validation + if (config.filepaths.empty()) { throw std::invalid_argument("--file is required"); } + + return config; +} + +void Config::print_usage(std::string const& program_name) +{ + std::cout << "Usage: " << program_name << " [OPTIONS]\n\n" + << "Options:\n" + << " -f, --file PATH File path to benchmark (required)\n" + << " -s, --size SIZE Number of bytes to read (default: 4G)\n" + << " Supports suffixes: K, M, G, T\n" + << " -t, --threads NUM Number of threads (default: 1)\n" + << " -r, --repetitions NUM Number of repetitions (default: 5)\n" + << " -D, --no-direct Disable O_DIRECT (use buffered I/O)\n" + << " -A, --no-align Disable buffer alignment\n" + << " -c, --drop-cache Drop page cache before each run\n" + << " -w, --overwrite Overwrite existing file\n" + << " -o, --open-once Open file once (not per iteration)\n" + << " -h, --help Show this help message\n\n" + << "Examples:\n" + << " " << program_name << " -f /mnt/nvme/test.bin -s 1G -t 4\n" + << " " << program_name + << " --file /dev/nvme0n1 --size 10G --threads 16 --drop-cache\n"; +} + +BenchmarkManager::BenchmarkManager(Config const& config) : _config(config) +{ + kvikio::defaults::set_thread_pool_nthreads(_config.num_threads); + + for (auto const& filepath : _config.filepaths) { + // Initialize buffer + void* buf{}; + + if (_config.align_buffer) { + auto const page_size = get_page_size(); + auto const aligned_size = kvikio::detail::align_up(_config.num_bytes, page_size); + buf = std::aligned_alloc(page_size, aligned_size); + } else { + buf = std::malloc(_config.num_bytes); + } + + std::memset(buf, 0, _config.num_bytes); + + _bufs.push_back(buf); + + // Initialize file + // Create the file if the overwrite flag is on, or if the file does not exist. + if (_config.overwrite_file || access(filepath.c_str(), F_OK) != 0) { + kvikio::FileHandle file_handle( + filepath, "w", kvikio::FileHandle::m644, kvikio::CompatMode::ON); + auto fut = file_handle.pwrite(buf, _config.num_bytes); + fut.get(); + } + } +} + +BenchmarkManager::~BenchmarkManager() +{ + for (auto&& buf : _bufs) { + std::free(buf); + } +} + +void BenchmarkManager::run() +{ + if (_config.open_file_once) { initialize(); } + + decltype(_config.repetition) count{0}; + double time_elapsed_total_us{0.0}; + for (decltype(_config.repetition) idx = 0; idx < _config.repetition; ++idx) { + if (_config.drop_file_cache) { kvikio::clear_page_cache(); } + + if (!_config.open_file_once) { initialize(); } + + auto start = std::chrono::steady_clock::now(); + run_target(); + auto end = std::chrono::steady_clock::now(); + + std::chrono::duration time_elapsed = end - start; + double time_elapsed_us = time_elapsed.count(); + if (idx > 0) { + ++count; + time_elapsed_total_us += time_elapsed_us; + double bandwidth = _config.num_bytes / time_elapsed_us * 1e6 / 1024.0 / 1024.0; + std::cout << std::string(4, ' ') << std::left << std::setw(4) << idx << std::setw(10) + << bandwidth << " [MiB/s]" << std::endl; + } + + if (!_config.open_file_once) { cleanup(); } + } + double average_bandwidth = + _config.num_bytes * count / time_elapsed_total_us * 1e6 / 1024.0 / 1024.0; + std::cout << std::string(4, ' ') << "Average bandwidth: " << std::setw(10) << average_bandwidth + << " [MiB/s]" << std::endl; + + if (_config.open_file_once) { cleanup(); } +} + +void BenchmarkManager::initialize() +{ + _file_handles.clear(); + + for (auto const& filepath : _config.filepaths) { + auto p = std::make_unique( + filepath, "r", kvikio::FileHandle::m644, kvikio::CompatMode::ON); + + if (_config.o_direct) { + auto file_status_flags = fcntl(p->fd(), F_GETFL); + SYSCALL_CHECK(file_status_flags); + SYSCALL_CHECK(fcntl(p->fd(), F_SETFL, file_status_flags | O_DIRECT)); + } + + _file_handles.push_back(std::move(p)); + } +} + +void BenchmarkManager::cleanup() +{ + for (auto&& file_handle : _file_handles) { + file_handle->close(); + } +} + +void BenchmarkManager::run_target() +{ + std::vector> futs; + + for (std::size_t i = 0; i < _file_handles.size(); ++i) { + auto& file_handle = _file_handles[i]; + auto* buf = _bufs[i]; + auto fut = file_handle->pread(buf, _config.num_bytes); + futs.push_back(std::move(fut)); + } + + for (auto&& fut : futs) { + fut.get(); + } +} +} // namespace kvikio::benchmark + +int main(int argc, char* argv[]) +{ + try { + auto config = kvikio::benchmark::Config::parse_args(argc, argv); + kvikio::benchmark::BenchmarkManager bench_manager(config); + bench_manager.run(); + } catch (std::exception const& e) { + std::cerr << "Error: " << e.what() << std::endl; + return 1; + } + return 0; +} diff --git a/cpp/benchmarks/local/posix_benchmark.hpp b/cpp/benchmarks/local/posix_benchmark.hpp new file mode 100644 index 0000000000..adb5695c9c --- /dev/null +++ b/cpp/benchmarks/local/posix_benchmark.hpp @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +#include + +namespace kvikio::benchmark { + +struct Config { + std::size_t num_bytes{4ull * 1024ull * 1024ull * 1024ull}; + std::vector filepaths; + bool overwrite_file{false}; + bool align_buffer{true}; + bool o_direct{true}; + bool drop_file_cache{false}; + bool compat_mode{true}; + unsigned int num_threads{1}; + bool open_file_once{false}; + int repetition{5}; + + static Config parse_args(int argc, char** argv); + static void print_usage(std::string const& program_name); +}; + +class BenchmarkManager { + private: + Config const& _config; + std::vector> _file_handles; + std::vector _bufs; + + void initialize(); + void cleanup(); + void run_target(); + + public: + BenchmarkManager(Config const& config); + ~BenchmarkManager(); + void run(); +}; +} // namespace kvikio::benchmark diff --git a/cpp/benchmarks/utils.cpp b/cpp/benchmarks/utils.cpp new file mode 100644 index 0000000000..1a6acb79ed --- /dev/null +++ b/cpp/benchmarks/utils.cpp @@ -0,0 +1,9 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "utils.hpp" + +namespace kvikio::benchmark { +} // namespace kvikio::benchmark diff --git a/cpp/benchmarks/utils.hpp b/cpp/benchmarks/utils.hpp new file mode 100644 index 0000000000..a4e6e4aea9 --- /dev/null +++ b/cpp/benchmarks/utils.hpp @@ -0,0 +1,7 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +namespace kvikio::benchmark { +} // namespace kvikio::benchmark From 60d6a63de2f6ccdab8cfa66ea3a792475ad357bc Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Wed, 19 Nov 2025 15:59:47 -0500 Subject: [PATCH 04/31] Update --- cpp/benchmarks/local/posix_benchmark.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index ad780aca4f..7641da1018 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -12,13 +12,13 @@ #include #include #include +#include #include #include +#include #include #include -#include -#include "kvikio/error.hpp" namespace kvikio::benchmark { From a938ebdbf1c628d7c750477b542176675a5ed7ac Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Wed, 19 Nov 2025 16:03:47 -0500 Subject: [PATCH 05/31] Update --- cpp/benchmarks/local/posix_benchmark.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index 7641da1018..9e20ed65c5 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -52,14 +52,18 @@ std::size_t parse_size(std::string const& str) std::size_t multiplier{1}; // Support both K/Ki, M/Mi, etc. as synonyms (all 1024-based) + std::size_t constexpr one_Ki{1024ULL}; + std::size_t constexpr one_Mi{1024ULL * one_Ki}; + std::size_t constexpr one_Gi{1024ULL * one_Mi}; + std::size_t constexpr one_Ti{1024ULL * one_Gi}; if (suffix == "K" || suffix == "KI") { - multiplier = 1024ULL; + multiplier = one_Ki; } else if (suffix == "M" || suffix == "MI") { - multiplier = 1024ULL * 1024; + multiplier = one_Mi; } else if (suffix == "G" || suffix == "GI") { - multiplier = 1024ULL * 1024 * 1024; + multiplier = one_Gi; } else if (suffix == "T" || suffix == "TI") { - multiplier = 1024ULL * 1024 * 1024 * 1024; + multiplier = one_Ti; } else { throw std::invalid_argument("Invalid size suffix: '" + suffix + "' (use K/Ki, M/Mi, G/Gi, or T/Ti)"); From 2ed42d542bea648a8761c50b2decd6b5f206ef69 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 20 Nov 2025 14:33:46 -0500 Subject: [PATCH 06/31] Update --- cpp/benchmarks/local/posix_benchmark.cpp | 202 ++++++----------------- cpp/benchmarks/local/posix_benchmark.hpp | 39 ++--- cpp/benchmarks/utils.cpp | 160 ++++++++++++++++++ cpp/benchmarks/utils.hpp | 75 +++++++++ 4 files changed, 299 insertions(+), 177 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index 9e20ed65c5..eeedb1ac33 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -4,15 +4,14 @@ */ #include "posix_benchmark.hpp" +#include "../utils.hpp" #include -#include -#include -#include -#include -#include + #include -#include +#include +#include +#include #include #include @@ -21,125 +20,52 @@ #include namespace kvikio::benchmark { - -// Helper to parse size strings like "1G", "500M" -std::size_t parse_size(std::string const& str) -{ - if (str.empty()) { throw std::invalid_argument("Empty size string"); } - - // Parse the numeric part - std::size_t pos{}; - double value{}; - try { - value = std::stod(str, &pos); - } catch (std::exception const& e) { - throw std::invalid_argument("Invalid size format: " + str); - } - - if (value < 0) { throw std::invalid_argument("Size cannot be negative"); } - - // Extract suffix (everything after the number) - auto suffix = str.substr(pos); - - // No suffix means raw bytes - if (suffix.empty()) { return static_cast(value); } - - // Normalize to uppercase for case-insensitive comparison - std::transform( - suffix.begin(), suffix.end(), suffix.begin(), [](unsigned char c) { return std::tolower(c); }); - - // All multipliers use 1024 (binary), not 1000 - std::size_t multiplier{1}; - - // Support both K/Ki, M/Mi, etc. as synonyms (all 1024-based) - std::size_t constexpr one_Ki{1024ULL}; - std::size_t constexpr one_Mi{1024ULL * one_Ki}; - std::size_t constexpr one_Gi{1024ULL * one_Mi}; - std::size_t constexpr one_Ti{1024ULL * one_Gi}; - if (suffix == "K" || suffix == "KI") { - multiplier = one_Ki; - } else if (suffix == "M" || suffix == "MI") { - multiplier = one_Mi; - } else if (suffix == "G" || suffix == "GI") { - multiplier = one_Gi; - } else if (suffix == "T" || suffix == "TI") { - multiplier = one_Ti; - } else { - throw std::invalid_argument("Invalid size suffix: '" + suffix + - "' (use K/Ki, M/Mi, G/Gi, or T/Ti)"); - } - - return static_cast(value * multiplier); -} - -Config Config::parse_args(int argc, char** argv) +void PosixConfig::parse_args(int argc, char** argv) { - Config config; - - static struct option long_options[] = {{"file", required_argument, 0, 'f'}, - {"size", required_argument, 0, 's'}, - {"threads", required_argument, 0, 't'}, - {"repetitions", required_argument, 0, 'r'}, - {"no-direct", no_argument, 0, 'D'}, - {"no-align", no_argument, 0, 'A'}, - {"drop-cache", no_argument, 0, 'c'}, - {"overwrite", no_argument, 0, 'w'}, - {"open-once", no_argument, 0, 'o'}, - {"help", no_argument, 0, 'h'}, - {0, 0, 0, 0}}; - - int opt; - int option_index = 0; - - while ((opt = getopt_long(argc, argv, "f:s:t:r:DAcwoh", long_options, &option_index)) != -1) { + Config::parse_args(argc, argv); + static option long_options[] = { + {"overwrite", no_argument, nullptr, 'w'}, {0, 0, 0, 0} + // Sentinel to mark the end of the array. Needed by getopt_long() + }; + + int opt{0}; + int option_index{-1}; + + // "f:"" means "-f" takes an argument + // "c" means "-c" does not take an argument + while ((opt = getopt_long(argc, argv, ":w", long_options, &option_index)) != -1) { switch (opt) { - case 'f': config.filepaths.push_back(optarg); break; - case 's': - config.num_bytes = parse_size(optarg); // Helper to parse "1G", "500M", etc. + case 'w': { + overwrite_file = true; + break; + } + case ':': { + // The parsed option has missing argument + std::stringstream ss; + ss << "Missing argument for option " << argv[optind - 1] << " (-" + << static_cast(optopt) << ")"; + throw std::runtime_error(ss.str()); break; - case 't': config.num_threads = std::stoul(optarg); break; - case 'r': config.repetition = std::stoi(optarg); break; - case 'D': config.o_direct = false; break; - case 'A': config.align_buffer = false; break; - case 'c': config.drop_file_cache = true; break; - case 'w': config.overwrite_file = true; break; - case 'o': config.open_file_once = true; break; - case 'h': config.print_usage(argv[0]); std::exit(0); - default: config.print_usage(argv[0]); std::exit(1); + } + default: { + // Unknown option is deferred to subsequent parsing, if any + break; + } } } - // Validation - if (config.filepaths.empty()) { throw std::invalid_argument("--file is required"); } - - return config; + // Reset getopt state for second pass in the future + optind = 1; } -void Config::print_usage(std::string const& program_name) +void PosixConfig::print_usage(std::string const& program_name) { - std::cout << "Usage: " << program_name << " [OPTIONS]\n\n" - << "Options:\n" - << " -f, --file PATH File path to benchmark (required)\n" - << " -s, --size SIZE Number of bytes to read (default: 4G)\n" - << " Supports suffixes: K, M, G, T\n" - << " -t, --threads NUM Number of threads (default: 1)\n" - << " -r, --repetitions NUM Number of repetitions (default: 5)\n" - << " -D, --no-direct Disable O_DIRECT (use buffered I/O)\n" - << " -A, --no-align Disable buffer alignment\n" - << " -c, --drop-cache Drop page cache before each run\n" - << " -w, --overwrite Overwrite existing file\n" - << " -o, --open-once Open file once (not per iteration)\n" - << " -h, --help Show this help message\n\n" - << "Examples:\n" - << " " << program_name << " -f /mnt/nvme/test.bin -s 1G -t 4\n" - << " " << program_name - << " --file /dev/nvme0n1 --size 10G --threads 16 --drop-cache\n"; + Config::print_usage(program_name); + std::cout << " -w, --overwrite Overwrite existing file\n"; } -BenchmarkManager::BenchmarkManager(Config const& config) : _config(config) +PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config)) { - kvikio::defaults::set_thread_pool_nthreads(_config.num_threads); - for (auto const& filepath : _config.filepaths) { // Initialize buffer void* buf{}; @@ -167,49 +93,14 @@ BenchmarkManager::BenchmarkManager(Config const& config) : _config(config) } } -BenchmarkManager::~BenchmarkManager() +PosixBenchmark::~PosixBenchmark() { for (auto&& buf : _bufs) { std::free(buf); } } -void BenchmarkManager::run() -{ - if (_config.open_file_once) { initialize(); } - - decltype(_config.repetition) count{0}; - double time_elapsed_total_us{0.0}; - for (decltype(_config.repetition) idx = 0; idx < _config.repetition; ++idx) { - if (_config.drop_file_cache) { kvikio::clear_page_cache(); } - - if (!_config.open_file_once) { initialize(); } - - auto start = std::chrono::steady_clock::now(); - run_target(); - auto end = std::chrono::steady_clock::now(); - - std::chrono::duration time_elapsed = end - start; - double time_elapsed_us = time_elapsed.count(); - if (idx > 0) { - ++count; - time_elapsed_total_us += time_elapsed_us; - double bandwidth = _config.num_bytes / time_elapsed_us * 1e6 / 1024.0 / 1024.0; - std::cout << std::string(4, ' ') << std::left << std::setw(4) << idx << std::setw(10) - << bandwidth << " [MiB/s]" << std::endl; - } - - if (!_config.open_file_once) { cleanup(); } - } - double average_bandwidth = - _config.num_bytes * count / time_elapsed_total_us * 1e6 / 1024.0 / 1024.0; - std::cout << std::string(4, ' ') << "Average bandwidth: " << std::setw(10) << average_bandwidth - << " [MiB/s]" << std::endl; - - if (_config.open_file_once) { cleanup(); } -} - -void BenchmarkManager::initialize() +void PosixBenchmark::initialize_impl() { _file_handles.clear(); @@ -227,14 +118,14 @@ void BenchmarkManager::initialize() } } -void BenchmarkManager::cleanup() +void PosixBenchmark::cleanup_impl() { for (auto&& file_handle : _file_handles) { file_handle->close(); } } -void BenchmarkManager::run_target() +void PosixBenchmark::run_target_impl() { std::vector> futs; @@ -254,9 +145,10 @@ void BenchmarkManager::run_target() int main(int argc, char* argv[]) { try { - auto config = kvikio::benchmark::Config::parse_args(argc, argv); - kvikio::benchmark::BenchmarkManager bench_manager(config); - bench_manager.run(); + kvikio::benchmark::PosixConfig config; + config.parse_args(argc, argv); + kvikio::benchmark::PosixBenchmark bench(std::move(config)); + bench.run(); } catch (std::exception const& e) { std::cerr << "Error: " << e.what() << std::endl; return 1; diff --git a/cpp/benchmarks/local/posix_benchmark.hpp b/cpp/benchmarks/local/posix_benchmark.hpp index adb5695c9c..0649c92c80 100644 --- a/cpp/benchmarks/local/posix_benchmark.hpp +++ b/cpp/benchmarks/local/posix_benchmark.hpp @@ -3,6 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ +#pragma once + +#include "../utils.hpp" + #include #include #include @@ -12,35 +16,26 @@ namespace kvikio::benchmark { -struct Config { - std::size_t num_bytes{4ull * 1024ull * 1024ull * 1024ull}; - std::vector filepaths; +struct PosixConfig : Config { bool overwrite_file{false}; - bool align_buffer{true}; - bool o_direct{true}; - bool drop_file_cache{false}; - bool compat_mode{true}; - unsigned int num_threads{1}; - bool open_file_once{false}; - int repetition{5}; - - static Config parse_args(int argc, char** argv); - static void print_usage(std::string const& program_name); + + virtual void parse_args(int argc, char** argv) override; + virtual void print_usage(std::string const& program_name) override; }; -class BenchmarkManager { - private: - Config const& _config; +class PosixBenchmark : public Benchmark { + friend class Benchmark; + + protected: std::vector> _file_handles; std::vector _bufs; - void initialize(); - void cleanup(); - void run_target(); + void initialize_impl(); + void cleanup_impl(); + void run_target_impl(); public: - BenchmarkManager(Config const& config); - ~BenchmarkManager(); - void run(); + PosixBenchmark(PosixConfig config); + ~PosixBenchmark(); }; } // namespace kvikio::benchmark diff --git a/cpp/benchmarks/utils.cpp b/cpp/benchmarks/utils.cpp index 1a6acb79ed..1fe598da2d 100644 --- a/cpp/benchmarks/utils.cpp +++ b/cpp/benchmarks/utils.cpp @@ -5,5 +5,165 @@ #include "utils.hpp" +#include + +#include +#include +#include + +#include + namespace kvikio::benchmark { + +std::size_t parse_size(std::string const& str) +{ + if (str.empty()) { throw std::invalid_argument("Empty size string"); } + + // Parse the numeric part + std::size_t pos{}; + double value{}; + try { + value = std::stod(str, &pos); + } catch (std::exception const& e) { + throw std::invalid_argument("Invalid size format: " + str); + } + + if (value < 0) { throw std::invalid_argument("Size cannot be negative"); } + + // Extract suffix (everything after the number) + auto suffix = str.substr(pos); + + // No suffix means raw bytes + if (suffix.empty()) { return static_cast(value); } + + // Normalize to uppercase for case-insensitive comparison + std::transform( + suffix.begin(), suffix.end(), suffix.begin(), [](unsigned char c) { return std::tolower(c); }); + + // All multipliers use 1024 (binary), not 1000 + std::size_t multiplier{1}; + + // Support both K/Ki, M/Mi, etc. as synonyms (all 1024-based) + std::size_t constexpr one_Ki{1024ULL}; + std::size_t constexpr one_Mi{1024ULL * one_Ki}; + std::size_t constexpr one_Gi{1024ULL * one_Mi}; + std::size_t constexpr one_Ti{1024ULL * one_Gi}; + if (suffix == "k" || suffix == "ki" || suffix == "kib") { + multiplier = one_Ki; + } else if (suffix == "m" || suffix == "mi" || suffix == "mib") { + multiplier = one_Mi; + } else if (suffix == "g" || suffix == "gi" || suffix == "gib") { + multiplier = one_Gi; + } else if (suffix == "t" || suffix == "ti" || suffix == "tib") { + multiplier = one_Ti; + } else { + throw std::invalid_argument("Invalid size suffix: '" + suffix + + "' (use K/Ki/KiB, M/Mi/MiB, G/Gi/GiB, or T/Ti/TiB)"); + } + + return static_cast(value * multiplier); +} + +void Config::parse_args(int argc, char** argv) +{ + static option long_options[] = { + {"file", required_argument, nullptr, 'f'}, + {"size", required_argument, nullptr, 's'}, + {"threads", required_argument, nullptr, 't'}, + {"repetitions", required_argument, nullptr, 'r'}, + {"no-direct", no_argument, nullptr, 'D'}, + {"no-align", no_argument, nullptr, 'A'}, + {"drop-cache", no_argument, nullptr, 'c'}, + // {"overwrite", no_argument, nullptr, 'w'}, + {"open-once", no_argument, nullptr, 'o'}, + {"help", no_argument, nullptr, 'h'}, + {0, 0, 0, 0} // Sentinel to mark the end of the array. Needed by getopt_long() + }; + + int opt{0}; + int option_index{-1}; + + // - By default getopt_long() returns '?' to indicate errors if an option has missing argument or + // if an unknown option is encountered. The starting ':' in the optstring modifies this behavior. + // Missing argument error now causes the return value to be ':'. Unknow option still leads to '?' + // and its processing is deferred. + // - "f:" means option "-f" takes an argument "c" means option + // - "-c" does not take an argument + while ((opt = getopt_long(argc, argv, ":f:s:t:r:DAcoh", long_options, &option_index)) != -1) { + switch (opt) { + case 'f': { + filepaths.push_back(optarg); + break; + } + case 's': { + num_bytes = parse_size(optarg); // Helper to parse "1G", "500M", etc. + break; + } + case 't': { + num_threads = std::stoul(optarg); + break; + } + case 'r': { + repetition = std::stoi(optarg); + break; + } + case 'D': { + o_direct = false; + break; + } + case 'A': { + align_buffer = false; + break; + } + case 'c': { + drop_file_cache = true; + break; + } + case 'o': { + open_file_once = true; + break; + } + case 'h': { + print_usage(argv[0]); + std::exit(0); + break; + } + case ':': { + // The parsed option has missing argument + std::stringstream ss; + ss << "Missing argument for option " << argv[optind - 1] << " (-" + << static_cast(optopt) << ")"; + throw std::runtime_error(ss.str()); + break; + } + default: { + // Unknown option is deferred to subsequent parsing, if any + break; + } + } + } + + // Validation + if (filepaths.empty()) { throw std::invalid_argument("--file is required"); } + + // Reset getopt state for second pass in the future + optind = 1; +} + +void Config::print_usage(std::string const& program_name) +{ + std::cout << "Usage: " << program_name << " [OPTIONS]\n\n" + << "Options:\n" + << " -f, --file PATH File path to benchmark (required)\n" + << " -s, --size SIZE Number of bytes to read (default: 4G)\n" + << " Supports suffixes: K, M, G, T\n" + << " -t, --threads NUM Number of threads (default: 1)\n" + << " -r, --repetitions NUM Number of repetitions (default: 5)\n" + << " -D, --no-direct Disable O_DIRECT (use buffered I/O)\n" + << " -A, --no-align Disable buffer alignment\n" + << " -c, --drop-cache Drop page cache before each run\n" + << " -o, --open-once Open file once (not per iteration)\n" + << " -h, --help Show this help message\n"; +} + } // namespace kvikio::benchmark diff --git a/cpp/benchmarks/utils.hpp b/cpp/benchmarks/utils.hpp index a4e6e4aea9..852bf4bcb4 100644 --- a/cpp/benchmarks/utils.hpp +++ b/cpp/benchmarks/utils.hpp @@ -3,5 +3,80 @@ * SPDX-License-Identifier: Apache-2.0 */ +#pragma once + +#include +#include +#include +#include +#include + +#include + namespace kvikio::benchmark { +// Helper to parse size strings like "1GiB", "1Gi", "1G". +std::size_t parse_size(std::string const& str); + +struct Config { + std::size_t num_bytes{4ull * 1024ull * 1024ull * 1024ull}; + std::vector filepaths; + bool align_buffer{true}; + bool o_direct{true}; + bool drop_file_cache{false}; + bool compat_mode{true}; + unsigned int num_threads{1}; + bool open_file_once{false}; + int repetition{5}; + + virtual void parse_args(int argc, char** argv); + virtual void print_usage(std::string const& program_name); +}; + +template +class Benchmark { + protected: + ConfigType _config; + + void initialize() { static_cast(this)->initialize_impl(); } + void cleanup() { static_cast(this)->cleanup_impl(); } + void run_target() { static_cast(this)->run_target_impl(); } + + public: + Benchmark(ConfigType config) : _config(std::move(config)) {} + + void run() + { + if (_config.open_file_once) { initialize(); } + + decltype(_config.repetition) count{0}; + double time_elapsed_total_us{0.0}; + for (decltype(_config.repetition) idx = 0; idx < _config.repetition; ++idx) { + if (_config.drop_file_cache) { kvikio::clear_page_cache(); } + + if (!_config.open_file_once) { initialize(); } + + auto start = std::chrono::steady_clock::now(); + run_target(); + auto end = std::chrono::steady_clock::now(); + + std::chrono::duration time_elapsed = end - start; + double time_elapsed_us = time_elapsed.count(); + if (idx > 0) { + ++count; + time_elapsed_total_us += time_elapsed_us; + double bandwidth = _config.num_bytes / time_elapsed_us * 1e6 / 1024.0 / 1024.0; + std::cout << std::string(4, ' ') << std::left << std::setw(4) << idx << std::setw(10) + << bandwidth << " [MiB/s]" << std::endl; + } + + if (!_config.open_file_once) { cleanup(); } + } + double average_bandwidth = + _config.num_bytes * count / time_elapsed_total_us * 1e6 / 1024.0 / 1024.0; + std::cout << std::string(4, ' ') << "Average bandwidth: " << std::setw(10) << average_bandwidth + << " [MiB/s]" << std::endl; + + if (_config.open_file_once) { cleanup(); } + } +}; } // namespace kvikio::benchmark From 77a1c7a42d3da7ef77071d3dd1e8bf565c5b6343 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 20 Nov 2025 14:45:26 -0500 Subject: [PATCH 07/31] Update --- cpp/include/kvikio/file_handle.hpp | 21 +++++++++++---------- cpp/src/file_handle.cpp | 21 +++++++++------------ 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index fd49657c4d..47ad668988 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -21,6 +21,7 @@ #include #include #include +#include "BS_thread_pool.hpp" namespace kvikio { @@ -40,8 +41,6 @@ class FileHandle { CompatModeManager _compat_mode_manager; friend class CompatModeManager; - std::unique_ptr _thread_pool; - public: // 644 is a common setting of Unix file permissions: read and write for owner, read-only for group // and others. @@ -237,10 +236,11 @@ class FileHandle { */ std::future pread(void* buf, std::size_t size, - std::size_t file_offset = 0, - std::size_t task_size = defaults::task_size(), - std::size_t gds_threshold = defaults::gds_threshold(), - bool sync_default_stream = true); + std::size_t file_offset = 0, + std::size_t task_size = defaults::task_size(), + std::size_t gds_threshold = defaults::gds_threshold(), + bool sync_default_stream = true, + BS_thread_pool* thread_pool = &defaults::thread_pool()); /** * @brief Writes specified bytes from device or host memory into the file in parallel. @@ -274,10 +274,11 @@ class FileHandle { */ std::future pwrite(void const* buf, std::size_t size, - std::size_t file_offset = 0, - std::size_t task_size = defaults::task_size(), - std::size_t gds_threshold = defaults::gds_threshold(), - bool sync_default_stream = true); + std::size_t file_offset = 0, + std::size_t task_size = defaults::task_size(), + std::size_t gds_threshold = defaults::gds_threshold(), + bool sync_default_stream = true, + BS_thread_pool* thread_pool = &defaults::thread_pool()); /** * @brief Reads specified bytes from the file into the device memory asynchronously. diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index 06724a7017..d3808ef336 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -31,7 +31,6 @@ FileHandle::FileHandle(std::string const& file_path, : _initialized{true}, _compat_mode_manager{file_path, flags, mode, compat_mode, this} { KVIKIO_NVTX_FUNC_RANGE(); - _thread_pool = std::make_unique(defaults::thread_pool_nthreads()); } FileHandle::FileHandle(FileHandle&& o) noexcept @@ -40,8 +39,7 @@ FileHandle::FileHandle(FileHandle&& o) noexcept _initialized{std::exchange(o._initialized, false)}, _nbytes{std::exchange(o._nbytes, 0)}, _cufile_handle{std::exchange(o._cufile_handle, {})}, - _compat_mode_manager{std::move(o._compat_mode_manager)}, - _thread_pool{std::move(o._thread_pool)} + _compat_mode_manager{std::move(o._compat_mode_manager)} { } @@ -53,7 +51,6 @@ FileHandle& FileHandle::operator=(FileHandle&& o) noexcept _nbytes = std::exchange(o._nbytes, 0); _cufile_handle = std::exchange(o._cufile_handle, {}); _compat_mode_manager = std::move(o._compat_mode_manager); - _thread_pool = std::move(o._thread_pool); return *this; } @@ -153,7 +150,8 @@ std::future FileHandle::pread(void* buf, std::size_t file_offset, std::size_t task_size, std::size_t gds_threshold, - bool sync_default_stream) + bool sync_default_stream, + BS_thread_pool* thread_pool) { auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); @@ -167,8 +165,7 @@ std::future FileHandle::pread(void* buf, _file_direct_off.fd(), buf, size, file_offset, _file_direct_on.fd()); }; - return parallel_io( - op, buf, size, file_offset, task_size, 0, _thread_pool.get(), call_idx, nvtx_color); + return parallel_io(op, buf, size, file_offset, task_size, 0, thread_pool, call_idx, nvtx_color); } CUcontext ctx = get_context_from_pointer(buf); @@ -204,7 +201,7 @@ std::future FileHandle::pread(void* buf, file_offset, task_size, devPtr_offset, - _thread_pool.get(), + thread_pool, call_idx, nvtx_color); } @@ -214,7 +211,8 @@ std::future FileHandle::pwrite(void const* buf, std::size_t file_offset, std::size_t task_size, std::size_t gds_threshold, - bool sync_default_stream) + bool sync_default_stream, + BS_thread_pool* thread_pool) { auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); @@ -228,8 +226,7 @@ std::future FileHandle::pwrite(void const* buf, _file_direct_off.fd(), buf, size, file_offset, _file_direct_on.fd()); }; - return parallel_io( - op, buf, size, file_offset, task_size, 0, _thread_pool.get(), call_idx, nvtx_color); + return parallel_io(op, buf, size, file_offset, task_size, 0, thread_pool, call_idx, nvtx_color); } CUcontext ctx = get_context_from_pointer(buf); @@ -265,7 +262,7 @@ std::future FileHandle::pwrite(void const* buf, file_offset, task_size, devPtr_offset, - _thread_pool.get(), + thread_pool, call_idx, nvtx_color); } From c78273942f55ce6249f85e4a29f9bd01e32abf7a Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 20 Nov 2025 14:52:26 -0500 Subject: [PATCH 08/31] Update --- cpp/include/kvikio/mmap.hpp | 3 ++- cpp/include/kvikio/remote_handle.hpp | 5 +++-- cpp/src/mmap.cpp | 5 +++-- cpp/src/remote_handle.cpp | 6 +++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cpp/include/kvikio/mmap.hpp b/cpp/include/kvikio/mmap.hpp index fe8b71cbf4..dacd77e830 100644 --- a/cpp/include/kvikio/mmap.hpp +++ b/cpp/include/kvikio/mmap.hpp @@ -174,7 +174,8 @@ class MmapHandle { std::future pread(void* buf, std::optional size = std::nullopt, std::size_t offset = 0, - std::size_t task_size = defaults::task_size()); + std::size_t task_size = defaults::task_size(), + BS_thread_pool* thread_pool = &defaults::thread_pool()); }; } // namespace kvikio diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 416e374291..20baf68c25 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -456,8 +456,9 @@ class RemoteHandle { */ std::future pread(void* buf, std::size_t size, - std::size_t file_offset = 0, - std::size_t task_size = defaults::task_size()); + std::size_t file_offset = 0, + std::size_t task_size = defaults::task_size(), + BS_thread_pool* thread_pool = &defaults::thread_pool()); }; } // namespace kvikio diff --git a/cpp/src/mmap.cpp b/cpp/src/mmap.cpp index f1e801f943..6c0942d26d 100644 --- a/cpp/src/mmap.cpp +++ b/cpp/src/mmap.cpp @@ -415,7 +415,8 @@ std::size_t MmapHandle::read(void* buf, std::optional size, std::si std::future MmapHandle::pread(void* buf, std::optional size, std::size_t offset, - std::size_t task_size) + std::size_t task_size, + BS_thread_pool* thread_pool) { KVIKIO_EXPECT(task_size <= defaults::bounce_buffer_size(), "bounce buffer size cannot be less than task size."); @@ -448,7 +449,7 @@ std::future MmapHandle::pread(void* buf, offset, task_size, 0, // dst buffer offset initial value - &defaults::thread_pool(), + thread_pool, call_idx, nvtx_color); } diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 7217294422..8c0ee238b3 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -809,7 +809,8 @@ std::size_t RemoteHandle::read(void* buf, std::size_t size, std::size_t file_off std::future RemoteHandle::pread(void* buf, std::size_t size, std::size_t file_offset, - std::size_t task_size) + std::size_t task_size, + BS_thread_pool* thread_pool) { auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size); @@ -819,8 +820,7 @@ std::future RemoteHandle::pread(void* buf, std::size_t devPtr_offset) -> std::size_t { return read(static_cast(devPtr_base) + devPtr_offset, size, file_offset); }; - return parallel_io( - task, buf, size, file_offset, task_size, 0, &defaults::thread_pool(), call_idx, nvtx_color); + return parallel_io(task, buf, size, file_offset, task_size, 0, thread_pool, call_idx, nvtx_color); } } // namespace kvikio From 4bd39fd665523c033af734b5a2afda112c1d2d0b Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 20 Nov 2025 14:54:50 -0500 Subject: [PATCH 09/31] Update --- cpp/include/kvikio/detail/parallel_operation.hpp | 2 +- cpp/include/kvikio/file_handle.hpp | 2 +- cpp/include/kvikio/mmap.hpp | 1 + cpp/include/kvikio/remote_handle.hpp | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/include/kvikio/detail/parallel_operation.hpp b/cpp/include/kvikio/detail/parallel_operation.hpp index b395e6b20d..76925a2732 100644 --- a/cpp/include/kvikio/detail/parallel_operation.hpp +++ b/cpp/include/kvikio/detail/parallel_operation.hpp @@ -17,8 +17,8 @@ #include #include #include +#include #include -#include "kvikio/threadpool_wrapper.hpp" namespace kvikio { diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index 47ad668988..5559d4bf5e 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -20,8 +20,8 @@ #include #include #include +#include #include -#include "BS_thread_pool.hpp" namespace kvikio { diff --git a/cpp/include/kvikio/mmap.hpp b/cpp/include/kvikio/mmap.hpp index dacd77e830..154819ef47 100644 --- a/cpp/include/kvikio/mmap.hpp +++ b/cpp/include/kvikio/mmap.hpp @@ -9,6 +9,7 @@ #include #include +#include #include namespace kvikio { diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 20baf68c25..9a4aaa2aa2 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -13,6 +13,7 @@ #include #include +#include #include struct curl_slist; From 65e6f3591a858cc2456ea3c49c55034cf5276374 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 21 Nov 2025 00:21:39 -0500 Subject: [PATCH 10/31] Update --- cpp/benchmarks/local/posix_benchmark.cpp | 6 ++---- cpp/benchmarks/utils.cpp | 2 -- cpp/benchmarks/utils.hpp | 9 ++++++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index eeedb1ac33..bcd54eb487 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -85,8 +85,7 @@ PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config) // Initialize file // Create the file if the overwrite flag is on, or if the file does not exist. if (_config.overwrite_file || access(filepath.c_str(), F_OK) != 0) { - kvikio::FileHandle file_handle( - filepath, "w", kvikio::FileHandle::m644, kvikio::CompatMode::ON); + kvikio::FileHandle file_handle(filepath, "w", kvikio::FileHandle::m644); auto fut = file_handle.pwrite(buf, _config.num_bytes); fut.get(); } @@ -105,8 +104,7 @@ void PosixBenchmark::initialize_impl() _file_handles.clear(); for (auto const& filepath : _config.filepaths) { - auto p = std::make_unique( - filepath, "r", kvikio::FileHandle::m644, kvikio::CompatMode::ON); + auto p = std::make_unique(filepath, "r"); if (_config.o_direct) { auto file_status_flags = fcntl(p->fd(), F_GETFL); diff --git a/cpp/benchmarks/utils.cpp b/cpp/benchmarks/utils.cpp index 1fe598da2d..af85ec3556 100644 --- a/cpp/benchmarks/utils.cpp +++ b/cpp/benchmarks/utils.cpp @@ -11,8 +11,6 @@ #include #include -#include - namespace kvikio::benchmark { std::size_t parse_size(std::string const& str) diff --git a/cpp/benchmarks/utils.hpp b/cpp/benchmarks/utils.hpp index 852bf4bcb4..d0b64a931a 100644 --- a/cpp/benchmarks/utils.hpp +++ b/cpp/benchmarks/utils.hpp @@ -11,7 +11,9 @@ #include #include +#include #include +#include "kvikio/compat_mode.hpp" namespace kvikio::benchmark { // Helper to parse size strings like "1GiB", "1Gi", "1G". @@ -42,7 +44,12 @@ class Benchmark { void run_target() { static_cast(this)->run_target_impl(); } public: - Benchmark(ConfigType config) : _config(std::move(config)) {} + Benchmark(ConfigType config) : _config(std::move(config)) + { + defaults::set_thread_pool_nthreads(_config.num_threads); + auto compat_mode = _config.compat_mode ? CompatMode::ON : CompatMode::OFF; + defaults::set_compat_mode(compat_mode); + } void run() { From 8f88010b9a1922b7cedf4c5d5d5937f913935d74 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 21 Nov 2025 00:27:03 -0500 Subject: [PATCH 11/31] Update --- cpp/benchmarks/utils.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/benchmarks/utils.hpp b/cpp/benchmarks/utils.hpp index d0b64a931a..897d228b9d 100644 --- a/cpp/benchmarks/utils.hpp +++ b/cpp/benchmarks/utils.hpp @@ -11,9 +11,9 @@ #include #include +#include #include #include -#include "kvikio/compat_mode.hpp" namespace kvikio::benchmark { // Helper to parse size strings like "1GiB", "1Gi", "1G". @@ -71,10 +71,10 @@ class Benchmark { if (idx > 0) { ++count; time_elapsed_total_us += time_elapsed_us; - double bandwidth = _config.num_bytes / time_elapsed_us * 1e6 / 1024.0 / 1024.0; - std::cout << std::string(4, ' ') << std::left << std::setw(4) << idx << std::setw(10) - << bandwidth << " [MiB/s]" << std::endl; } + double bandwidth = _config.num_bytes / time_elapsed_us * 1e6 / 1024.0 / 1024.0; + std::cout << std::string(4, ' ') << std::left << std::setw(4) << idx << std::setw(10) + << bandwidth << " [MiB/s]" << std::endl; if (!_config.open_file_once) { cleanup(); } } From c7425ce57a7256e7430de24c9794410483b2873a Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 21 Nov 2025 01:02:10 -0500 Subject: [PATCH 12/31] Update --- cpp/benchmarks/local/posix_benchmark.cpp | 28 ++++++++++++++++++++++-- cpp/benchmarks/local/posix_benchmark.hpp | 3 +++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index bcd54eb487..54f3852200 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -14,10 +14,12 @@ #include #include +#include #include #include #include #include +#include namespace kvikio::benchmark { void PosixConfig::parse_args(int argc, char** argv) @@ -33,12 +35,16 @@ void PosixConfig::parse_args(int argc, char** argv) // "f:"" means "-f" takes an argument // "c" means "-c" does not take an argument - while ((opt = getopt_long(argc, argv, ":w", long_options, &option_index)) != -1) { + while ((opt = getopt_long(argc, argv, ":wp", long_options, &option_index)) != -1) { switch (opt) { case 'w': { overwrite_file = true; break; } + case 'p': { + per_file_pool = true; + break; + } case ':': { // The parsed option has missing argument std::stringstream ss; @@ -89,6 +95,12 @@ PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config) auto fut = file_handle.pwrite(buf, _config.num_bytes); fut.get(); } + + // Initialize thread pool + if (_config.per_file_pool) { + auto thread_pool = std::make_unique(_config.num_threads); + _thread_pools.push_back(std::move(thread_pool)); + } } } @@ -130,7 +142,19 @@ void PosixBenchmark::run_target_impl() for (std::size_t i = 0; i < _file_handles.size(); ++i) { auto& file_handle = _file_handles[i]; auto* buf = _bufs[i]; - auto fut = file_handle->pread(buf, _config.num_bytes); + + std::future fut; + if (_config.per_file_pool) { + fut = file_handle->pread(buf, + _config.num_bytes, + 0, + defaults::task_size(), + defaults::gds_threshold(), + true, + _thread_pools[i].get()); + } else { + fut = file_handle->pread(buf, _config.num_bytes); + } futs.push_back(std::move(fut)); } diff --git a/cpp/benchmarks/local/posix_benchmark.hpp b/cpp/benchmarks/local/posix_benchmark.hpp index 0649c92c80..8bc7f1b3ca 100644 --- a/cpp/benchmarks/local/posix_benchmark.hpp +++ b/cpp/benchmarks/local/posix_benchmark.hpp @@ -13,11 +13,13 @@ #include #include +#include namespace kvikio::benchmark { struct PosixConfig : Config { bool overwrite_file{false}; + bool per_file_pool{false}; virtual void parse_args(int argc, char** argv) override; virtual void print_usage(std::string const& program_name) override; @@ -29,6 +31,7 @@ class PosixBenchmark : public Benchmark { protected: std::vector> _file_handles; std::vector _bufs; + std::vector> _thread_pools; void initialize_impl(); void cleanup_impl(); From 4d7cc3f0c10f06d307007063653388f72c9e4300 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 18:05:14 +0000 Subject: [PATCH 13/31] Update --- cpp/benchmarks/local/posix_benchmark.cpp | 2 ++ cpp/benchmarks/local/posix_benchmark.hpp | 1 + cpp/benchmarks/utils.hpp | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index 54f3852200..965503840c 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -162,6 +162,8 @@ void PosixBenchmark::run_target_impl() fut.get(); } } + +std::size_t PosixBenchmark::nbytes_impl() { return _config.num_bytes * _config.filepaths.size(); } } // namespace kvikio::benchmark int main(int argc, char* argv[]) diff --git a/cpp/benchmarks/local/posix_benchmark.hpp b/cpp/benchmarks/local/posix_benchmark.hpp index 8bc7f1b3ca..c059e117bf 100644 --- a/cpp/benchmarks/local/posix_benchmark.hpp +++ b/cpp/benchmarks/local/posix_benchmark.hpp @@ -36,6 +36,7 @@ class PosixBenchmark : public Benchmark { void initialize_impl(); void cleanup_impl(); void run_target_impl(); + std::size_t nbytes_impl(); public: PosixBenchmark(PosixConfig config); diff --git a/cpp/benchmarks/utils.hpp b/cpp/benchmarks/utils.hpp index 897d228b9d..bd6ee47e7e 100644 --- a/cpp/benchmarks/utils.hpp +++ b/cpp/benchmarks/utils.hpp @@ -42,6 +42,7 @@ class Benchmark { void initialize() { static_cast(this)->initialize_impl(); } void cleanup() { static_cast(this)->cleanup_impl(); } void run_target() { static_cast(this)->run_target_impl(); } + std::size_t nbytes() { return static_cast(this)->nbytes_impl(); } public: Benchmark(ConfigType config) : _config(std::move(config)) @@ -72,14 +73,13 @@ class Benchmark { ++count; time_elapsed_total_us += time_elapsed_us; } - double bandwidth = _config.num_bytes / time_elapsed_us * 1e6 / 1024.0 / 1024.0; + double bandwidth = nbytes() / time_elapsed_us * 1e6 / 1024.0 / 1024.0; std::cout << std::string(4, ' ') << std::left << std::setw(4) << idx << std::setw(10) << bandwidth << " [MiB/s]" << std::endl; if (!_config.open_file_once) { cleanup(); } } - double average_bandwidth = - _config.num_bytes * count / time_elapsed_total_us * 1e6 / 1024.0 / 1024.0; + double average_bandwidth = nbytes() * count / time_elapsed_total_us * 1e6 / 1024.0 / 1024.0; std::cout << std::string(4, ' ') << "Average bandwidth: " << std::setw(10) << average_bandwidth << " [MiB/s]" << std::endl; From c5950420d056868c68b8fca00d5400713dcc4f67 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 13:43:13 -0500 Subject: [PATCH 14/31] Cleanup --- cpp/include/kvikio/defaults.hpp | 4 ++-- .../kvikio/detail/parallel_operation.hpp | 18 ++++++++--------- cpp/include/kvikio/file_handle.hpp | 20 +++++++++---------- cpp/include/kvikio/mmap.hpp | 2 +- cpp/include/kvikio/remote_handle.hpp | 6 +++--- cpp/include/kvikio/threadpool_wrapper.hpp | 20 +------------------ cpp/src/defaults.cpp | 2 +- cpp/src/file_handle.cpp | 5 ++--- cpp/src/mmap.cpp | 2 +- cpp/src/remote_handle.cpp | 2 +- 10 files changed, 31 insertions(+), 50 deletions(-) diff --git a/cpp/include/kvikio/defaults.hpp b/cpp/include/kvikio/defaults.hpp index 722986c362..aff8aa5ea8 100644 --- a/cpp/include/kvikio/defaults.hpp +++ b/cpp/include/kvikio/defaults.hpp @@ -111,7 +111,7 @@ std::tuple getenv_or( */ class defaults { private: - BS_thread_pool _thread_pool{get_num_threads_from_env()}; + ThreadPool _thread_pool{get_num_threads_from_env()}; CompatMode _compat_mode; std::size_t _task_size; std::size_t _gds_threshold; @@ -212,7 +212,7 @@ class defaults { * * @return The default thread pool instance. */ - [[nodiscard]] static BS_thread_pool& thread_pool(); + [[nodiscard]] static ThreadPool& thread_pool(); /** * @brief Get the number of threads in the default thread pool. diff --git a/cpp/include/kvikio/detail/parallel_operation.hpp b/cpp/include/kvikio/detail/parallel_operation.hpp index 76925a2732..af18b3506e 100644 --- a/cpp/include/kvikio/detail/parallel_operation.hpp +++ b/cpp/include/kvikio/detail/parallel_operation.hpp @@ -76,9 +76,9 @@ std::future submit_task(F op, std::size_t size, std::size_t file_offset, std::size_t devPtr_offset, - BS_thread_pool* thread_pool = &defaults::thread_pool(), - std::uint64_t nvtx_payload = 0ull, - nvtx_color_type nvtx_color = NvtxManager::default_color()) + ThreadPool* thread_pool = &defaults::thread_pool(), + std::uint64_t nvtx_payload = 0ull, + nvtx_color_type nvtx_color = NvtxManager::default_color()) { static_assert(std::is_invocable_r_v submit_task(F op, template std::future submit_move_only_task( F op_move_only, - BS_thread_pool* thread_pool = &defaults::thread_pool(), - std::uint64_t nvtx_payload = 0ull, - nvtx_color_type nvtx_color = NvtxManager::default_color()) + ThreadPool* thread_pool = &defaults::thread_pool(), + std::uint64_t nvtx_payload = 0ull, + nvtx_color_type nvtx_color = NvtxManager::default_color()) { static_assert(std::is_invocable_r_v); auto op_copyable = make_copyable_lambda(std::move(op_move_only)); @@ -136,9 +136,9 @@ std::future parallel_io(F op, std::size_t file_offset, std::size_t task_size, std::size_t devPtr_offset, - BS_thread_pool* thread_pool = &defaults::thread_pool(), - std::uint64_t call_idx = 0, - nvtx_color_type nvtx_color = NvtxManager::default_color()) + ThreadPool* thread_pool = &defaults::thread_pool(), + std::uint64_t call_idx = 0, + nvtx_color_type nvtx_color = NvtxManager::default_color()) { KVIKIO_EXPECT(task_size > 0, "`task_size` must be positive", std::invalid_argument); static_assert(std::is_invocable_r_v pread(void* buf, std::size_t size, - std::size_t file_offset = 0, - std::size_t task_size = defaults::task_size(), - std::size_t gds_threshold = defaults::gds_threshold(), - bool sync_default_stream = true, - BS_thread_pool* thread_pool = &defaults::thread_pool()); + std::size_t file_offset = 0, + std::size_t task_size = defaults::task_size(), + std::size_t gds_threshold = defaults::gds_threshold(), + bool sync_default_stream = true, + ThreadPool* thread_pool = &defaults::thread_pool()); /** * @brief Writes specified bytes from device or host memory into the file in parallel. @@ -274,11 +274,11 @@ class FileHandle { */ std::future pwrite(void const* buf, std::size_t size, - std::size_t file_offset = 0, - std::size_t task_size = defaults::task_size(), - std::size_t gds_threshold = defaults::gds_threshold(), - bool sync_default_stream = true, - BS_thread_pool* thread_pool = &defaults::thread_pool()); + std::size_t file_offset = 0, + std::size_t task_size = defaults::task_size(), + std::size_t gds_threshold = defaults::gds_threshold(), + bool sync_default_stream = true, + ThreadPool* thread_pool = &defaults::thread_pool()); /** * @brief Reads specified bytes from the file into the device memory asynchronously. diff --git a/cpp/include/kvikio/mmap.hpp b/cpp/include/kvikio/mmap.hpp index 154819ef47..77af652815 100644 --- a/cpp/include/kvikio/mmap.hpp +++ b/cpp/include/kvikio/mmap.hpp @@ -176,7 +176,7 @@ class MmapHandle { std::optional size = std::nullopt, std::size_t offset = 0, std::size_t task_size = defaults::task_size(), - BS_thread_pool* thread_pool = &defaults::thread_pool()); + ThreadPool* thread_pool = &defaults::thread_pool()); }; } // namespace kvikio diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 9a4aaa2aa2..bff9bc2786 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -457,9 +457,9 @@ class RemoteHandle { */ std::future pread(void* buf, std::size_t size, - std::size_t file_offset = 0, - std::size_t task_size = defaults::task_size(), - BS_thread_pool* thread_pool = &defaults::thread_pool()); + std::size_t file_offset = 0, + std::size_t task_size = defaults::task_size(), + ThreadPool* thread_pool = &defaults::thread_pool()); }; } // namespace kvikio diff --git a/cpp/include/kvikio/threadpool_wrapper.hpp b/cpp/include/kvikio/threadpool_wrapper.hpp index 0644b8c9ca..ace7999bd1 100644 --- a/cpp/include/kvikio/threadpool_wrapper.hpp +++ b/cpp/include/kvikio/threadpool_wrapper.hpp @@ -9,24 +9,6 @@ namespace kvikio { -template -class thread_pool_wrapper : public pool_type { - public: - /** - * @brief Construct a new thread pool wrapper. - * - * @param nthreads The number of threads to use. - */ - thread_pool_wrapper(unsigned int nthreads) : pool_type{nthreads} {} - - /** - * @brief Reset the number of threads in the thread pool. - * - * @param nthreads The number of threads to use. - */ - void reset(unsigned int nthreads) { pool_type::reset(nthreads); } -}; - -using BS_thread_pool = thread_pool_wrapper; +using ThreadPool = BS::thread_pool; } // namespace kvikio diff --git a/cpp/src/defaults.cpp b/cpp/src/defaults.cpp index f827ef6cf5..1bbc151b86 100644 --- a/cpp/src/defaults.cpp +++ b/cpp/src/defaults.cpp @@ -172,7 +172,7 @@ bool defaults::is_compat_mode_preferred(CompatMode compat_mode) noexcept bool defaults::is_compat_mode_preferred() { return is_compat_mode_preferred(compat_mode()); } -BS_thread_pool& defaults::thread_pool() { return instance()->_thread_pool; } +ThreadPool& defaults::thread_pool() { return instance()->_thread_pool; } unsigned int defaults::thread_pool_nthreads() { return thread_pool().get_thread_count(); } diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index d3808ef336..a79ea09a02 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -151,7 +150,7 @@ std::future FileHandle::pread(void* buf, std::size_t task_size, std::size_t gds_threshold, bool sync_default_stream, - BS_thread_pool* thread_pool) + ThreadPool* thread_pool) { auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); @@ -212,7 +211,7 @@ std::future FileHandle::pwrite(void const* buf, std::size_t task_size, std::size_t gds_threshold, bool sync_default_stream, - BS_thread_pool* thread_pool) + ThreadPool* thread_pool) { auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); diff --git a/cpp/src/mmap.cpp b/cpp/src/mmap.cpp index 6c0942d26d..0b5c23c0f1 100644 --- a/cpp/src/mmap.cpp +++ b/cpp/src/mmap.cpp @@ -416,7 +416,7 @@ std::future MmapHandle::pread(void* buf, std::optional size, std::size_t offset, std::size_t task_size, - BS_thread_pool* thread_pool) + ThreadPool* thread_pool) { KVIKIO_EXPECT(task_size <= defaults::bounce_buffer_size(), "bounce buffer size cannot be less than task size."); diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index 8c0ee238b3..c5ecb0a293 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -810,7 +810,7 @@ std::future RemoteHandle::pread(void* buf, std::size_t size, std::size_t file_offset, std::size_t task_size, - BS_thread_pool* thread_pool) + ThreadPool* thread_pool) { auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size); From 59e60788206b2379e4310ce768133da891488782 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 13:51:37 -0500 Subject: [PATCH 15/31] Update --- cpp/include/kvikio/file_handle.hpp | 4 ++++ cpp/include/kvikio/mmap.hpp | 2 ++ cpp/include/kvikio/remote_handle.hpp | 2 ++ 3 files changed, 8 insertions(+) diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index 34c40e84c2..b2b61a8485 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -229,6 +229,8 @@ class FileHandle { * in the null stream. When in KvikIO's compatibility mode or when accessing host memory, the * operation is always default stream ordered like the rest of the non-async CUDA API. In this * case, the value of `sync_default_stream` is ignored. + * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default + * thread pool. * @return Future that on completion returns the size of bytes that were successfully read. * * @note The `std::future` object's `wait()` or `get()` should not be called after the lifetime of @@ -267,6 +269,8 @@ class FileHandle { * in the null stream. When in KvikIO's compatibility mode or when accessing host memory, the * operation is always default stream ordered like the rest of the non-async CUDA API. In this * case, the value of `sync_default_stream` is ignored. + * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default + * thread pool. * @return Future that on completion returns the size of bytes that were successfully written. * * @note The `std::future` object's `wait()` or `get()` should not be called after the lifetime of diff --git a/cpp/include/kvikio/mmap.hpp b/cpp/include/kvikio/mmap.hpp index 77af652815..4f56fa880c 100644 --- a/cpp/include/kvikio/mmap.hpp +++ b/cpp/include/kvikio/mmap.hpp @@ -163,6 +163,8 @@ class MmapHandle { * specified, read starts from `offset` to the end of file * @param offset File offset * @param task_size Size of each task in bytes + * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default + * thread pool. * @return Future that on completion returns the size of bytes that were successfully read. * * @exception std::out_of_range if the read region specified by `offset` and `size` is diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index bff9bc2786..2f41a728d1 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -453,6 +453,8 @@ class RemoteHandle { * @param size Number of bytes to read. * @param file_offset File offset in bytes. * @param task_size Size of each task in bytes. + * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default + * thread pool. * @return Future that on completion returns the size of bytes read, which is always `size`. */ std::future pread(void* buf, From afb670d70b970e9b478a873521d41a8a78edc817 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 14:15:31 -0500 Subject: [PATCH 16/31] Add comments --- cpp/include/kvikio/threadpool_wrapper.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/include/kvikio/threadpool_wrapper.hpp b/cpp/include/kvikio/threadpool_wrapper.hpp index ace7999bd1..d0e8d4b286 100644 --- a/cpp/include/kvikio/threadpool_wrapper.hpp +++ b/cpp/include/kvikio/threadpool_wrapper.hpp @@ -9,6 +9,9 @@ namespace kvikio { +/** + * @brief Thread pool type used for parallel I/O operations. + */ using ThreadPool = BS::thread_pool; } // namespace kvikio From c4c22ce186e48903711947db8ae01a71aba3ad27 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 14:27:34 -0500 Subject: [PATCH 17/31] Add nullity check --- cpp/src/file_handle.cpp | 2 ++ cpp/src/mmap.cpp | 1 + cpp/src/remote_handle.cpp | 1 + 3 files changed, 4 insertions(+) diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index a79ea09a02..abec24fa79 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -152,6 +152,7 @@ std::future FileHandle::pread(void* buf, bool sync_default_stream, ThreadPool* thread_pool) { + KVIKIO_EXPECT(thread_pool != nullptr, "The thread pool must not be nullptr"); auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); if (is_host_memory(buf)) { @@ -213,6 +214,7 @@ std::future FileHandle::pwrite(void const* buf, bool sync_default_stream, ThreadPool* thread_pool) { + KVIKIO_EXPECT(thread_pool != nullptr, "The thread pool must not be nullptr"); auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); if (is_host_memory(buf)) { diff --git a/cpp/src/mmap.cpp b/cpp/src/mmap.cpp index 0b5c23c0f1..ff579cfa4e 100644 --- a/cpp/src/mmap.cpp +++ b/cpp/src/mmap.cpp @@ -420,6 +420,7 @@ std::future MmapHandle::pread(void* buf, { KVIKIO_EXPECT(task_size <= defaults::bounce_buffer_size(), "bounce buffer size cannot be less than task size."); + KVIKIO_EXPECT(thread_pool != nullptr, "The thread pool must not be nullptr"); auto actual_size = validate_and_adjust_read_args(size, offset); if (actual_size == 0) { return make_ready_future(actual_size); } diff --git a/cpp/src/remote_handle.cpp b/cpp/src/remote_handle.cpp index c5ecb0a293..7c917c9a0b 100644 --- a/cpp/src/remote_handle.cpp +++ b/cpp/src/remote_handle.cpp @@ -812,6 +812,7 @@ std::future RemoteHandle::pread(void* buf, std::size_t task_size, ThreadPool* thread_pool) { + KVIKIO_EXPECT(thread_pool != nullptr, "The thread pool must not be nullptr"); auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size); auto task = [this](void* devPtr_base, From e73d4d64cf0394358f01008da359135492f08588 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 14:32:08 -0500 Subject: [PATCH 18/31] Add missing comments --- cpp/include/kvikio/detail/parallel_operation.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/include/kvikio/detail/parallel_operation.hpp b/cpp/include/kvikio/detail/parallel_operation.hpp index af18b3506e..d0c96cd8fd 100644 --- a/cpp/include/kvikio/detail/parallel_operation.hpp +++ b/cpp/include/kvikio/detail/parallel_operation.hpp @@ -127,6 +127,10 @@ std::future submit_move_only_task( * @param size Number of bytes to read or write. * @param file_offset Byte offset to the start of the file. * @param task_size Size of each task in bytes. + * @param devPtr_offset Offset relative to the `devPtr_base` pointer to read into. This parameter + * should be used only with registered buffers. + * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default + * thread pool. * @return A future to be used later to check if the operation has finished its execution. */ template From 78489e8e9e4067450bd75659dbac20fb22562e3e Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 14:32:47 -0500 Subject: [PATCH 19/31] Update --- cpp/include/kvikio/detail/parallel_operation.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/kvikio/detail/parallel_operation.hpp b/cpp/include/kvikio/detail/parallel_operation.hpp index d0c96cd8fd..0d82dcd7a9 100644 --- a/cpp/include/kvikio/detail/parallel_operation.hpp +++ b/cpp/include/kvikio/detail/parallel_operation.hpp @@ -127,8 +127,8 @@ std::future submit_move_only_task( * @param size Number of bytes to read or write. * @param file_offset Byte offset to the start of the file. * @param task_size Size of each task in bytes. - * @param devPtr_offset Offset relative to the `devPtr_base` pointer to read into. This parameter - * should be used only with registered buffers. + * @param devPtr_offset Offset relative to the `devPtr_base` pointer. This parameter should be used + * only with registered buffers. * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default * thread pool. * @return A future to be used later to check if the operation has finished its execution. From 19f8af9e1ac01e205c610193119e8e900823909a Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 24 Nov 2025 14:33:29 -0500 Subject: [PATCH 20/31] Update --- cpp/include/kvikio/detail/parallel_operation.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/include/kvikio/detail/parallel_operation.hpp b/cpp/include/kvikio/detail/parallel_operation.hpp index 0d82dcd7a9..1d3c43d287 100644 --- a/cpp/include/kvikio/detail/parallel_operation.hpp +++ b/cpp/include/kvikio/detail/parallel_operation.hpp @@ -145,6 +145,7 @@ std::future parallel_io(F op, nvtx_color_type nvtx_color = NvtxManager::default_color()) { KVIKIO_EXPECT(task_size > 0, "`task_size` must be positive", std::invalid_argument); + KVIKIO_EXPECT(thread_pool != nullptr, "The thread pool must not be nullptr"); static_assert(std::is_invocable_r_v Date: Mon, 24 Nov 2025 22:33:32 +0000 Subject: [PATCH 21/31] Fix build error --- cpp/benchmarks/local/posix_benchmark.cpp | 2 +- cpp/benchmarks/local/posix_benchmark.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index 965503840c..b9ecdeb6f5 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -98,7 +98,7 @@ PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config) // Initialize thread pool if (_config.per_file_pool) { - auto thread_pool = std::make_unique(_config.num_threads); + auto thread_pool = std::make_unique(_config.num_threads); _thread_pools.push_back(std::move(thread_pool)); } } diff --git a/cpp/benchmarks/local/posix_benchmark.hpp b/cpp/benchmarks/local/posix_benchmark.hpp index c059e117bf..6d78ce8dc4 100644 --- a/cpp/benchmarks/local/posix_benchmark.hpp +++ b/cpp/benchmarks/local/posix_benchmark.hpp @@ -31,7 +31,7 @@ class PosixBenchmark : public Benchmark { protected: std::vector> _file_handles; std::vector _bufs; - std::vector> _thread_pools; + std::vector> _thread_pools; void initialize_impl(); void cleanup_impl(); From 45b3cc77dfba0fecdbe0b8500554a23e20777c2d Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 25 Nov 2025 16:03:36 -0500 Subject: [PATCH 22/31] Update --- cpp/tests/test_basic_io.cpp | 53 ++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/cpp/tests/test_basic_io.cpp b/cpp/tests/test_basic_io.cpp index aeda7051d0..1d4b440e49 100644 --- a/cpp/tests/test_basic_io.cpp +++ b/cpp/tests/test_basic_io.cpp @@ -7,12 +7,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include "utils/env.hpp" @@ -27,7 +29,7 @@ class BasicIOTest : public testing::Test { TempDir tmp_dir{false}; _filepath = tmp_dir.path() / "test"; - _dev_a = std::move(DevBuffer::arange(100)); + _dev_a = std::move(DevBuffer::arange(1024ULL * 1024ULL + 124ULL)); _dev_b = std::move(DevBuffer::zero_like(_dev_a)); } @@ -98,6 +100,55 @@ TEST_F(BasicIOTest, write_read_async) CUDA_DRIVER_TRY(kvikio::cudaAPI::instance().StreamDestroy(stream)); } +TEST_F(BasicIOTest, threadpool) +{ + auto thread_pool = std::make_unique(4); + + // Write the buffer to a file using an external thread pool + { + kvikio::FileHandle f(_filepath, "w"); + auto fut = f.pwrite(_dev_a.ptr, + _dev_a.nbytes, // size + 0, // file_offset + kvikio::defaults::task_size(), + kvikio::defaults::gds_threshold(), + true, + thread_pool.get()); + auto nbytes_written = fut.get(); + EXPECT_EQ(nbytes_written, _dev_a.nbytes); + } + + // Read from the buffer + { + std::vector> futs; + std::vector filepaths{_filepath, _filepath}; + std::vector file_handles; + std::vector> dev_buffers; + + for (auto const& filepath : filepaths) { + file_handles.emplace_back(filepath, "r"); + dev_buffers.push_back(DevBuffer::zero_like(_dev_a)); + } + + for (std::size_t i = 0; i < file_handles.size(); ++i) { + auto fut = file_handles[i].pread(dev_buffers[i].ptr, + dev_buffers[i].nbytes, + 0, + kvikio::defaults::task_size(), + kvikio::defaults::gds_threshold(), + true, + thread_pool.get()); + futs.push_back(std::move(fut)); + } + + for (std::size_t i = 0; i < file_handles.size(); ++i) { + auto nbtyes_read = futs[i].get(); + EXPECT_EQ(nbtyes_read, _dev_a.nbytes); + expect_equal(_dev_a, dev_buffers[i]); + } + } +} + class DirectIOTest : public testing::Test { public: using value_type = std::int64_t; From c348cbbb6bdbdc9203666b610a359b0253bc9a80 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 25 Nov 2025 16:19:11 -0500 Subject: [PATCH 23/31] Add unit tests --- cpp/tests/test_basic_io.cpp | 53 ++++++++++++++++++++++++++++++++++++- cpp/tests/test_mmap.cpp | 34 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/cpp/tests/test_basic_io.cpp b/cpp/tests/test_basic_io.cpp index aeda7051d0..113d677e95 100644 --- a/cpp/tests/test_basic_io.cpp +++ b/cpp/tests/test_basic_io.cpp @@ -7,12 +7,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include "utils/env.hpp" @@ -27,7 +29,7 @@ class BasicIOTest : public testing::Test { TempDir tmp_dir{false}; _filepath = tmp_dir.path() / "test"; - _dev_a = std::move(DevBuffer::arange(100)); + _dev_a = std::move(DevBuffer::arange(1024ULL * 1024ULL + 124ULL)); _dev_b = std::move(DevBuffer::zero_like(_dev_a)); } @@ -98,6 +100,55 @@ TEST_F(BasicIOTest, write_read_async) CUDA_DRIVER_TRY(kvikio::cudaAPI::instance().StreamDestroy(stream)); } +TEST_F(BasicIOTest, threadpool) +{ + auto thread_pool = std::make_unique(4); + + // Write to a file using an external thread pool + { + kvikio::FileHandle f(_filepath, "w"); + auto fut = f.pwrite(_dev_a.ptr, + _dev_a.nbytes, // size + 0, // file_offset + kvikio::defaults::task_size(), + kvikio::defaults::gds_threshold(), + true, + thread_pool.get()); + auto nbytes_written = fut.get(); + EXPECT_EQ(nbytes_written, _dev_a.nbytes); + } + + // Read from the file using an external thread pool + { + std::vector> futs; + std::vector filepaths{_filepath, _filepath}; + std::vector file_handles; + std::vector> dev_buffers; + + for (auto const& filepath : filepaths) { + file_handles.emplace_back(filepath, "r"); + dev_buffers.push_back(DevBuffer::zero_like(_dev_a)); + } + + for (std::size_t i = 0; i < file_handles.size(); ++i) { + auto fut = file_handles[i].pread(dev_buffers[i].ptr, + dev_buffers[i].nbytes, + 0, + kvikio::defaults::task_size(), + kvikio::defaults::gds_threshold(), + true, + thread_pool.get()); + futs.push_back(std::move(fut)); + } + + for (std::size_t i = 0; i < file_handles.size(); ++i) { + auto nbtyes_read = futs[i].get(); + EXPECT_EQ(nbtyes_read, _dev_a.nbytes); + expect_equal(_dev_a, dev_buffers[i]); + } + } +} + class DirectIOTest : public testing::Test { public: using value_type = std::int64_t; diff --git a/cpp/tests/test_mmap.cpp b/cpp/tests/test_mmap.cpp index 9e355f4789..1d280d6594 100644 --- a/cpp/tests/test_mmap.cpp +++ b/cpp/tests/test_mmap.cpp @@ -360,3 +360,37 @@ TEST_F(MmapTest, cpp_move) do_test(mmap_handle_2); } } + +TEST_F(MmapTest, threadpool) +{ + auto thread_pool = std::make_unique(4); + + // Read from the file using an external thread pool + { + std::size_t num_elements = _file_size / sizeof(value_type); + std::vector> futs; + std::vector filepaths{_filepath, _filepath}; + std::vector mmap_handles; + std::vector> dev_buffers; + + for (auto const& filepath : filepaths) { + mmap_handles.emplace_back(filepath, "r"); + dev_buffers.push_back(kvikio::test::DevBuffer::zero_like(num_elements)); + } + + for (std::size_t i = 0; i < mmap_handles.size(); ++i) { + auto fut = mmap_handles[i].pread(dev_buffers[i].ptr, + dev_buffers[i].nbytes, + 0, + kvikio::defaults::task_size(), + thread_pool.get()); + futs.push_back(std::move(fut)); + } + + for (std::size_t i = 0; i < mmap_handles.size(); ++i) { + auto nbtyes_read = futs[i].get(); + EXPECT_EQ(nbtyes_read, _file_size); + EXPECT_EQ(_host_buf, dev_buffers[i].to_vector()); + } + } +} From bbf240bac49266180949c64034cf992357061a67 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 25 Nov 2025 16:30:26 -0500 Subject: [PATCH 24/31] Add comments --- cpp/include/kvikio/file_handle.hpp | 16 ++++++++++------ cpp/include/kvikio/mmap.hpp | 8 +++++--- cpp/include/kvikio/remote_handle.hpp | 7 ++++++- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index f39e7c01bf..613a359d1b 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -230,11 +230,13 @@ class FileHandle { * operation is always default stream ordered like the rest of the non-async CUDA API. In this * case, the value of `sync_default_stream` is ignored. * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default - * thread pool. + * thread pool. The caller is responsible for ensuring that the thread pool remains valid until + * the returned future is consumed (i.e., until `get()` or `wait()` is called on it). * @return Future that on completion returns the size of bytes that were successfully read. * - * @note The `std::future` object's `wait()` or `get()` should not be called after the lifetime of - * the FileHandle object ends. Otherwise, the behavior is undefined. + * @note The returned `std::future` object must not outlive either the FileHandle or the thread + * pool. Calling `wait()` or `get()` on the future after the FileHandle or thread pool has been + * destroyed results in undefined behavior. */ std::future pread(void* buf, std::size_t size, @@ -270,11 +272,13 @@ class FileHandle { * operation is always default stream ordered like the rest of the non-async CUDA API. In this * case, the value of `sync_default_stream` is ignored. * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default - * thread pool. + * thread pool. The caller is responsible for ensuring that the thread pool remains valid until + * the returned future is consumed (i.e., until `get()` or `wait()` is called on it). * @return Future that on completion returns the size of bytes that were successfully written. * - * @note The `std::future` object's `wait()` or `get()` should not be called after the lifetime of - * the FileHandle object ends. Otherwise, the behavior is undefined. + * @note The returned `std::future` object must not outlive either the FileHandle or the thread + * pool. Calling `wait()` or `get()` on the future after the FileHandle or thread pool has been + * destroyed results in undefined behavior. */ std::future pwrite(void const* buf, std::size_t size, diff --git a/cpp/include/kvikio/mmap.hpp b/cpp/include/kvikio/mmap.hpp index 4f56fa880c..da6f596a11 100644 --- a/cpp/include/kvikio/mmap.hpp +++ b/cpp/include/kvikio/mmap.hpp @@ -164,15 +164,17 @@ class MmapHandle { * @param offset File offset * @param task_size Size of each task in bytes * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default - * thread pool. + * thread pool. The caller is responsible for ensuring that the thread pool remains valid until + * the returned future is consumed (i.e., until `get()` or `wait()` is called on it). * @return Future that on completion returns the size of bytes that were successfully read. * * @exception std::out_of_range if the read region specified by `offset` and `size` is * outside the initial region specified when the mapping handle was constructed * @exception std::runtime_error if the mapping handle is closed * - * @note The `std::future` object's `wait()` or `get()` should not be called after the lifetime of - * the MmapHandle object ends. Otherwise, the behavior is undefined. + * @note The returned `std::future` object must not outlive either the MmapHandle or the thread + * pool. Calling `wait()` or `get()` on the future after the MmapHandle or thread pool has been + * destroyed results in undefined behavior. */ std::future pread(void* buf, std::optional size = std::nullopt, diff --git a/cpp/include/kvikio/remote_handle.hpp b/cpp/include/kvikio/remote_handle.hpp index 2f41a728d1..0b0808c45e 100644 --- a/cpp/include/kvikio/remote_handle.hpp +++ b/cpp/include/kvikio/remote_handle.hpp @@ -454,8 +454,13 @@ class RemoteHandle { * @param file_offset File offset in bytes. * @param task_size Size of each task in bytes. * @param thread_pool Thread pool to use for parallel execution. Defaults to the global default - * thread pool. + * thread pool. The caller is responsible for ensuring that the thread pool remains valid until + * the returned future is consumed (i.e., until `get()` or `wait()` is called on it). * @return Future that on completion returns the size of bytes read, which is always `size`. + * + * @note The returned `std::future` object must not outlive either the RemoteHandle or the thread + * pool. Calling `wait()` or `get()` on the future after the RemoteHandle or thread pool has been + * destroyed results in undefined behavior. */ std::future pread(void* buf, std::size_t size, From 71837f492195e67ac5fab5da0f37011dfd1e3199 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Wed, 26 Nov 2025 14:53:28 -0500 Subject: [PATCH 25/31] Update --- cpp/include/kvikio/defaults.hpp | 1 + cpp/src/file_handle.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/cpp/include/kvikio/defaults.hpp b/cpp/include/kvikio/defaults.hpp index aff8aa5ea8..c024e3089f 100644 --- a/cpp/include/kvikio/defaults.hpp +++ b/cpp/include/kvikio/defaults.hpp @@ -121,6 +121,7 @@ class defaults { std::vector _http_status_codes; bool _auto_direct_io_read; bool _auto_direct_io_write; + bool _thread_pool_per_block_dev; static unsigned int get_num_threads_from_env(); diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index abec24fa79..b03ebb320a 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -23,6 +23,10 @@ namespace kvikio { +namespace { +ThreadPool* get_thread_pool_per_block_dev() {} +} // namespace + FileHandle::FileHandle(std::string const& file_path, std::string const& flags, mode_t mode, From 5e3102c9f4d0b4a58fb3ca707e5f347a677f29b8 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 27 Nov 2025 00:21:44 -0500 Subject: [PATCH 26/31] Implement per-block-device pool --- cpp/include/kvikio/defaults.hpp | 24 +++- cpp/include/kvikio/file_handle.hpp | 1 + cpp/src/defaults.cpp | 12 ++ cpp/src/file_handle.cpp | 182 ++++++++++++++++++++++++++++- 4 files changed, 212 insertions(+), 7 deletions(-) diff --git a/cpp/include/kvikio/defaults.hpp b/cpp/include/kvikio/defaults.hpp index c024e3089f..190909c2cc 100644 --- a/cpp/include/kvikio/defaults.hpp +++ b/cpp/include/kvikio/defaults.hpp @@ -121,7 +121,7 @@ class defaults { std::vector _http_status_codes; bool _auto_direct_io_read; bool _auto_direct_io_write; - bool _thread_pool_per_block_dev; + bool _thread_pool_per_block_device; static unsigned int get_num_threads_from_env(); @@ -395,6 +395,28 @@ class defaults { * @param flag true to enable opportunistic Direct I/O writes, false to disable */ static void set_auto_direct_io_write(bool flag); + + /** + * @brief Check if per-block-device thread pools are enabled. + * + * The initial value is determined by the environment variable + * `KVIKIO_THREAD_POOL_PER_BLOCK_DEVICE`. If not set, defaults to `false`. + * + * @return Boolean answer + */ + static bool thread_pool_per_block_device(); + + /** + * @brief Enable or disable per-block-device thread pools. + * + * Each pool is initialized with the number of threads specified by `thread_pool_nthreads()`. + * Changes take effect only for files opened after this call. Files already opened retain their + * existing thread pool assignments. + * + * @param flag `true` to enable per-block-device thread pools, `false` to use the single global + * thread pool for all I/O operations. + */ + static void set_thread_pool_per_block_device(bool flag); }; } // namespace kvikio diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp index 613a359d1b..0bf4328b9f 100644 --- a/cpp/include/kvikio/file_handle.hpp +++ b/cpp/include/kvikio/file_handle.hpp @@ -40,6 +40,7 @@ class FileHandle { CUFileHandleWrapper _cufile_handle{}; CompatModeManager _compat_mode_manager; friend class CompatModeManager; + ThreadPool* _thread_pool{}; public: // 644 is a common setting of Unix file permissions: read and write for owner, read-only for group diff --git a/cpp/src/defaults.cpp b/cpp/src/defaults.cpp index 1bbc151b86..841e7314d3 100644 --- a/cpp/src/defaults.cpp +++ b/cpp/src/defaults.cpp @@ -141,6 +141,11 @@ defaults::defaults() _auto_direct_io_read = getenv_or("KVIKIO_AUTO_DIRECT_IO_READ", false); _auto_direct_io_write = getenv_or("KVIKIO_AUTO_DIRECT_IO_WRITE", true); } + + // Determine the default value of `thread_pool_per_block_device` + { + _thread_pool_per_block_device = getenv_or("KVIKIO_THREAD_POOL_PER_BLOCK_DEVICE", false); + } } defaults* defaults::instance() @@ -238,4 +243,11 @@ void defaults::set_auto_direct_io_read(bool flag) { instance()->_auto_direct_io_ bool defaults::auto_direct_io_write() { return instance()->_auto_direct_io_write; } void defaults::set_auto_direct_io_write(bool flag) { instance()->_auto_direct_io_write = flag; } + +bool defaults::thread_pool_per_block_device() { return instance()->_thread_pool_per_block_device; } + +void defaults::set_thread_pool_per_block_device(bool flag) +{ + instance()->_thread_pool_per_block_device = flag; +} } // namespace kvikio diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index b03ebb320a..aaeadea2bd 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -5,10 +5,17 @@ #include #include +#include #include #include + #include #include +#include +#include +#include +#include +#include #include #include @@ -24,7 +31,150 @@ namespace kvikio { namespace { -ThreadPool* get_thread_pool_per_block_dev() {} +/** + * @brief Get the unique device ID for the physical block device hosting a file. + * + * Resolves the underlying block device for a given file path, handling: + * - Partitions: walks up to the parent block device (e.g., sda1 -> sda) + * - NVMe namespaces: maps to the controller (e.g., nvme0n1 -> nvme0) + * - Other block devices (SATA, SAS, dm, md): returns the device's own ID + * + * This enables grouping files by physical hardware for I/O thread pool assignment, ensuring files + * on the same physical device share a thread pool. + * + * For device-mapper devices (LVM, dm-crypt), this returns the dm device ID, not the + * underlying physical device(s). Multiple LVs on the same physical disk will get separate thread + * pools. + * + * @param file_path Path to the file whose block device ID is to be determined. + * @return The block device ID for the physical block device. + */ +dev_t get_block_device_id(std::string const& file_path) +{ + dev_t block_dev_id{}; + unsigned dev_major_id{}; + unsigned dev_minor_id{}; + + struct stat st; + SYSCALL_CHECK(stat(file_path.c_str(), &st)); + + dev_major_id = major(st.st_dev); + dev_minor_id = minor(st.st_dev); + + // Construct sysfs path (which is a symlink) for the block device + // e.g., /sys/dev/block/259:8 + std::string sysfs_path = + "/sys/dev/block/" + std::to_string(dev_major_id) + ":" + std::to_string(dev_minor_id); + + // Resolve the symlink to the actual sysfs device path + // e.g., + // - NVMe: /sys/devices/pci0000:00/.../nvme/nvme1/nvme1n1/nvme1n1p3 + // - SATA: /sys/devices/pci0000:00/.../block/sdb/sdb1 + char resolved_path[PATH_MAX]; + SYSCALL_CHECK(realpath(sysfs_path.c_str(), resolved_path), + "Path failed to resolve", + static_cast(nullptr)); + + // If this is a partition, walk up to the parent block device. + // Partitions have a "partition" file in their sysfs directory. + std::filesystem::path cur_path(resolved_path); + if (std::filesystem::exists(cur_path / "partition")) { cur_path = cur_path.parent_path(); } + + // After walking up, cur_path points to the base block device + // e.g., + // - NVMe: /sys/devices/pci0000:00/.../nvme/nvme1/nvme1n1 + // - SATA: /sys/devices/pci0000:00/.../block/sdb + + // Extract block device name block_dev_name + // e.g., + // - NVMe: nvme1n1 + // - SSD: sdb + std::string const block_dev_name{cur_path.filename()}; + + std::string dev_file; + + // For NVMe, multiple namespaces (nvme0n1, nvme0n2) share the same controller (nvme0) and thus the + // same physical hardware. Map namespace to controller. + static std::regex const nvme_pattern(R"(^(nvme\d+)n\d+$)"); + std::smatch match; + if (std::regex_match(block_dev_name, match, nvme_pattern)) { + // For NVMe, strip namespace, e.g. nvme0n1 -> nvme0 + std::string controller = match[1].str(); // e.g., "nvme0" + + // Controller's dev file: /sys/class/nvme/nvme0/dev + dev_file = "/sys/class/nvme/" + controller + "/dev"; + } else { + // For non-NVMe devices, read dev_t from the block device's sysfs entry + // e.g., /sys/devices/pci0000:00/.../block/sdb/dev + dev_file = cur_path.string() + "/dev"; + } + + // Parse "major:minor" from the dev file + std::ifstream ifs(dev_file); + KVIKIO_EXPECT(!ifs.fail(), "Block device file cannot be opened: " + dev_file); + char colon{}; + ifs >> dev_major_id >> colon >> dev_minor_id; + KVIKIO_EXPECT(colon == ':', + "Parsing failed. Unexpected character encountered: " + std::string(1, colon)); + block_dev_id = makedev(dev_major_id, dev_minor_id); + return block_dev_id; +} + +/** + * @brief Get a thread pool specific to the block device hosting the given file. + * + * Thread pools are created lazily on first access and cached for subsequent lookups. + * Two levels of caching are used: + * - File path --> thread pool (fast path for repeated access to the same file) + * - Block device ID --> thread pool (groups different files on the same device) + * + * @param file_path Path to the file where a thread pool is requested for the underlying block + * device. + * @return Pointer to the appropriate thread pool. The pointer remains valid for the lifetime of the + * program (static storage duration). + * + * @note If device detection fails for any reason (e.g., unsupported filesystem, permission issues), + * the default global thread pool is returned and an error is logged. + */ +ThreadPool* get_thread_pool_per_block_device(std::string const& file_path) +{ + if (!defaults::thread_pool_per_block_device()) { return &defaults::thread_pool(); } + + static std::mutex mtx; + std::lock_guard lock(mtx); + + try { + // Fast path: check if this exact file path has been seen before + static std::unordered_map> + file_path_to_thread_pool_map; + if (auto it = file_path_to_thread_pool_map.find(file_path); + it != file_path_to_thread_pool_map.end()) { + return it->second.get(); + } + + // Resolve file path to its underlying block device + auto block_dev_id = get_block_device_id(file_path); + + // Check if we already have a thread pool for this block device + static std::unordered_map> dev_id_to_thread_pool_map; + if (auto it = dev_id_to_thread_pool_map.find(block_dev_id); + it != dev_id_to_thread_pool_map.end()) { + // Cache the file path mapping for future fast-path lookups + file_path_to_thread_pool_map.emplace(file_path, it->second); + return it->second.get(); + } + + // First file on this block device: create a new dedicated thread pool + auto thread_pool = std::make_shared(defaults::num_threads()); + dev_id_to_thread_pool_map.emplace(block_dev_id, thread_pool); + file_path_to_thread_pool_map.emplace(file_path, thread_pool); + return thread_pool.get(); + } catch (std::exception const& ex) { + std::string const& msg = std::string(ex.what()) + " Falling back to the default thread pool."; + KVIKIO_LOG_ERROR(msg); + return &defaults::thread_pool(); + } +} } // namespace FileHandle::FileHandle(std::string const& file_path, @@ -34,6 +184,7 @@ FileHandle::FileHandle(std::string const& file_path, : _initialized{true}, _compat_mode_manager{file_path, flags, mode, compat_mode, this} { KVIKIO_NVTX_FUNC_RANGE(); + _thread_pool = get_thread_pool_per_block_device(file_path); } FileHandle::FileHandle(FileHandle&& o) noexcept @@ -42,7 +193,8 @@ FileHandle::FileHandle(FileHandle&& o) noexcept _initialized{std::exchange(o._initialized, false)}, _nbytes{std::exchange(o._nbytes, 0)}, _cufile_handle{std::exchange(o._cufile_handle, {})}, - _compat_mode_manager{std::move(o._compat_mode_manager)} + _compat_mode_manager{std::move(o._compat_mode_manager)}, + _thread_pool{std::exchange(o._thread_pool, {})} { } @@ -54,6 +206,7 @@ FileHandle& FileHandle::operator=(FileHandle&& o) noexcept _nbytes = std::exchange(o._nbytes, 0); _cufile_handle = std::exchange(o._cufile_handle, {}); _compat_mode_manager = std::move(o._compat_mode_manager); + _thread_pool = std::exchange(o._thread_pool, {}); return *this; } @@ -75,6 +228,7 @@ void FileHandle::close() noexcept _file_direct_on.close(); _nbytes = 0; _initialized = false; + _thread_pool = nullptr; } catch (...) { } } @@ -157,6 +311,13 @@ std::future FileHandle::pread(void* buf, ThreadPool* thread_pool) { KVIKIO_EXPECT(thread_pool != nullptr, "The thread pool must not be nullptr"); + auto* actual_thread_pool{thread_pool}; + // Use the block-device-specific pool only if it exists and the user didn't explicitly provide a + // custom pool + if (_thread_pool != nullptr && thread_pool == &defaults::thread_pool()) { + actual_thread_pool = _thread_pool; + } + auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); if (is_host_memory(buf)) { @@ -169,7 +330,8 @@ std::future FileHandle::pread(void* buf, _file_direct_off.fd(), buf, size, file_offset, _file_direct_on.fd()); }; - return parallel_io(op, buf, size, file_offset, task_size, 0, thread_pool, call_idx, nvtx_color); + return parallel_io( + op, buf, size, file_offset, task_size, 0, actual_thread_pool, call_idx, nvtx_color); } CUcontext ctx = get_context_from_pointer(buf); @@ -205,7 +367,7 @@ std::future FileHandle::pread(void* buf, file_offset, task_size, devPtr_offset, - thread_pool, + actual_thread_pool, call_idx, nvtx_color); } @@ -219,6 +381,13 @@ std::future FileHandle::pwrite(void const* buf, ThreadPool* thread_pool) { KVIKIO_EXPECT(thread_pool != nullptr, "The thread pool must not be nullptr"); + auto* actual_thread_pool{thread_pool}; + // Use the block-device-specific pool only if it exists and the user didn't explicitly provide a + // custom pool + if (_thread_pool != nullptr && thread_pool == &defaults::thread_pool()) { + actual_thread_pool = _thread_pool; + } + auto& [nvtx_color, call_idx] = detail::get_next_color_and_call_idx(); KVIKIO_NVTX_FUNC_RANGE(size, nvtx_color); if (is_host_memory(buf)) { @@ -231,7 +400,8 @@ std::future FileHandle::pwrite(void const* buf, _file_direct_off.fd(), buf, size, file_offset, _file_direct_on.fd()); }; - return parallel_io(op, buf, size, file_offset, task_size, 0, thread_pool, call_idx, nvtx_color); + return parallel_io( + op, buf, size, file_offset, task_size, 0, actual_thread_pool, call_idx, nvtx_color); } CUcontext ctx = get_context_from_pointer(buf); @@ -267,7 +437,7 @@ std::future FileHandle::pwrite(void const* buf, file_offset, task_size, devPtr_offset, - thread_pool, + actual_thread_pool, call_idx, nvtx_color); } From 52276a1caa919418e775c0f1eb7ec5135e8fee9e Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 28 Nov 2025 03:06:25 +0000 Subject: [PATCH 27/31] Update --- cpp/benchmarks/local/posix_benchmark.cpp | 22 ++++++---------------- cpp/benchmarks/local/posix_benchmark.hpp | 2 +- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/local/posix_benchmark.cpp index b9ecdeb6f5..a7f2bfdf61 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/local/posix_benchmark.cpp @@ -19,7 +19,6 @@ #include #include #include -#include namespace kvikio::benchmark { void PosixConfig::parse_args(int argc, char** argv) @@ -42,7 +41,7 @@ void PosixConfig::parse_args(int argc, char** argv) break; } case 'p': { - per_file_pool = true; + per_drive_pool = true; break; } case ':': { @@ -88,6 +87,9 @@ PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config) _bufs.push_back(buf); + // Initialize KvikIO setting + if (_config.per_drive_pool) { kvikio::defaults::set_thread_pool_per_block_device(true); } + // Initialize file // Create the file if the overwrite flag is on, or if the file does not exist. if (_config.overwrite_file || access(filepath.c_str(), F_OK) != 0) { @@ -95,12 +97,6 @@ PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config) auto fut = file_handle.pwrite(buf, _config.num_bytes); fut.get(); } - - // Initialize thread pool - if (_config.per_file_pool) { - auto thread_pool = std::make_unique(_config.num_threads); - _thread_pools.push_back(std::move(thread_pool)); - } } } @@ -144,14 +140,8 @@ void PosixBenchmark::run_target_impl() auto* buf = _bufs[i]; std::future fut; - if (_config.per_file_pool) { - fut = file_handle->pread(buf, - _config.num_bytes, - 0, - defaults::task_size(), - defaults::gds_threshold(), - true, - _thread_pools[i].get()); + if (_config.per_drive_pool) { + fut = file_handle->pread(buf, _config.num_bytes); } else { fut = file_handle->pread(buf, _config.num_bytes); } diff --git a/cpp/benchmarks/local/posix_benchmark.hpp b/cpp/benchmarks/local/posix_benchmark.hpp index 6d78ce8dc4..11e6a81c2c 100644 --- a/cpp/benchmarks/local/posix_benchmark.hpp +++ b/cpp/benchmarks/local/posix_benchmark.hpp @@ -19,7 +19,7 @@ namespace kvikio::benchmark { struct PosixConfig : Config { bool overwrite_file{false}; - bool per_file_pool{false}; + bool per_drive_pool{false}; virtual void parse_args(int argc, char** argv) override; virtual void print_usage(std::string const& program_name) override; From df7501a11350d4d92ff1807775110787cd9f6f5f Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Wed, 10 Dec 2025 09:34:20 -0500 Subject: [PATCH 28/31] Update --- cpp/src/file_handle.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/file_handle.cpp b/cpp/src/file_handle.cpp index 2cf483b3ed..b978b49dc3 100644 --- a/cpp/src/file_handle.cpp +++ b/cpp/src/file_handle.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include From a31feebc81f77454cd10cab644e6633abf4f1bd4 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 19 Dec 2025 15:57:29 -0500 Subject: [PATCH 29/31] Big update --- cpp/benchmarks/CMakeLists.txt | 8 +- cpp/benchmarks/io/common.cpp | 243 ++++++++++++++++++ cpp/benchmarks/{utils.hpp => io/common.hpp} | 39 ++- .../sequential_benchmark.cpp} | 85 +++--- .../sequential_benchmark.hpp} | 15 +- cpp/benchmarks/utils.cpp | 167 ------------ 6 files changed, 332 insertions(+), 225 deletions(-) create mode 100644 cpp/benchmarks/io/common.cpp rename cpp/benchmarks/{utils.hpp => io/common.hpp} (74%) rename cpp/benchmarks/{local/posix_benchmark.cpp => io/sequential_benchmark.cpp} (61%) rename cpp/benchmarks/{local/posix_benchmark.hpp => io/sequential_benchmark.hpp} (72%) delete mode 100644 cpp/benchmarks/utils.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 36a4593aa2..5212732e9a 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -44,7 +44,9 @@ function(kvikio_add_benchmark) CUDA_STANDARD_REQUIRED ON ) - target_link_libraries(${_KVIKIO_NAME} PUBLIC benchmark::benchmark kvikio::kvikio) + target_link_libraries( + ${_KVIKIO_NAME} PUBLIC benchmark::benchmark kvikio::kvikio CUDA::cudart_static + ) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(KVIKIO_CXX_FLAGS "-Wall;-Werror;-Wno-unknown-pragmas") @@ -61,4 +63,6 @@ endfunction() kvikio_add_benchmark(NAME THREADPOOL_BENCHMARK SOURCES "threadpool/threadpool_benchmark.cpp") -kvikio_add_benchmark(NAME POSIX_BENCHMARK SOURCES "local/posix_benchmark.cpp" "utils.cpp") +kvikio_add_benchmark( + NAME SEQUENTIAL_BENCHMARK SOURCES "io/sequential_benchmark.cpp" "io/common.cpp" +) diff --git a/cpp/benchmarks/io/common.cpp b/cpp/benchmarks/io/common.cpp new file mode 100644 index 0000000000..cb8d33e7b5 --- /dev/null +++ b/cpp/benchmarks/io/common.cpp @@ -0,0 +1,243 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "common.hpp" + +#include + +#include +#include +#include +#include "kvikio/detail/utils.hpp" +#include "kvikio/utils.hpp" + +namespace kvikio::benchmark { + +std::size_t parse_size(std::string const& str) +{ + if (str.empty()) { throw std::invalid_argument("Empty size string"); } + + // Parse the numeric part + std::size_t pos{}; + double value{}; + try { + value = std::stod(str, &pos); + } catch (std::exception const& e) { + throw std::invalid_argument("Invalid size format: " + str); + } + + if (value < 0) { throw std::invalid_argument("Size cannot be negative"); } + + // Extract suffix (everything after the number) + auto suffix = str.substr(pos); + + // No suffix means raw bytes + if (suffix.empty()) { return static_cast(value); } + + // Normalize to lowercase for case-insensitive comparison + std::transform( + suffix.begin(), suffix.end(), suffix.begin(), [](unsigned char c) { return std::tolower(c); }); + + // All multipliers use 1024 (binary), not 1000 + std::size_t multiplier{1}; + + // Support both K/Ki, M/Mi, etc. as synonyms (all 1024-based) + std::size_t constexpr one_Ki{1024ULL}; + std::size_t constexpr one_Mi{1024ULL * one_Ki}; + std::size_t constexpr one_Gi{1024ULL * one_Mi}; + std::size_t constexpr one_Ti{1024ULL * one_Gi}; + std::size_t constexpr one_Pi{1024ULL * one_Ti}; + if (suffix == "k" || suffix == "ki" || suffix == "kb" || suffix == "kib") { + multiplier = one_Ki; + } else if (suffix == "m" || suffix == "mi" || suffix == "mb" || suffix == "mib") { + multiplier = one_Mi; + } else if (suffix == "g" || suffix == "gi" || suffix == "gb" || suffix == "gib") { + multiplier = one_Gi; + } else if (suffix == "t" || suffix == "ti" || suffix == "tb" || suffix == "tib") { + multiplier = one_Ti; + } else if (suffix == "p" || suffix == "pi" || suffix == "pb" || suffix == "pib") { + multiplier = one_Pi; + } else { + throw std::invalid_argument( + "Invalid size suffix: '" + suffix + + "' (use K/Ki/KB/KiB, M/Mi/MB/MiB, G/Gi/GB/GiB, T/Ti/TB/TiB, or P/Pi/PB/PiB)"); + } + + return static_cast(value * multiplier); +} + +bool parse_flag(std::string const& str) +{ + if (str.empty()) { throw std::invalid_argument("Empty flag"); } + + // Normalize to lowercase for case-insensitive comparison + auto result{str}; + std::transform( + result.begin(), result.end(), result.begin(), [](unsigned char c) { return std::tolower(c); }); + + if (result == "true" || result == "on" || result == "yes" || result == "1") { + return true; + } else if (result == "false" || result == "off" || result == "no" || result == "0") { + return false; + } else { + throw std::invalid_argument("Invalid flag: '" + str + + "' (use true/false, on/off, yes/no, or 1/0)"); + } +} + +void Config::parse_args(int argc, char** argv) +{ + enum LongOnlyOpts { + O_DIRECT = 1000, + ALIGN_BUFFER, + DROP_CACHE, + OPEN_ONCE, + COMPAT_MODE, + PER_BLOCK_DEVICE_POOL, + }; + + static option long_options[] = { + {"file", required_argument, nullptr, 'f'}, + {"size", required_argument, nullptr, 's'}, + {"threads", required_argument, nullptr, 't'}, + {"use-gpu-buffer", required_argument, nullptr, 'g'}, + {"gpu-index", required_argument, nullptr, 'd'}, + {"repetitions", required_argument, nullptr, 'r'}, + {"o-direct", required_argument, nullptr, LongOnlyOpts::O_DIRECT}, + {"align-buffer", required_argument, nullptr, LongOnlyOpts::ALIGN_BUFFER}, + {"drop-cache", required_argument, nullptr, LongOnlyOpts::DROP_CACHE}, + {"overwrite", required_argument, nullptr, 'w'}, + {"open-once", required_argument, nullptr, LongOnlyOpts::OPEN_ONCE}, + {"compat-mode", required_argument, nullptr, LongOnlyOpts::COMPAT_MODE}, + {"per-block-device-pool", required_argument, nullptr, LongOnlyOpts::PER_BLOCK_DEVICE_POOL}, + {"help", no_argument, nullptr, 'h'}, + {0, 0, 0, 0} // Sentinel to mark the end of the array. Needed by getopt_long() + }; + + int opt{0}; + int option_index{-1}; + + // - By default getopt_long() returns '?' to indicate errors if an option has missing argument or + // if an unknown option is encountered. The starting ':' in the optstring modifies this behavior. + // Missing argument error now causes the return value to be ':'. Unknow option still leads to '?' + // and its processing is deferred. + // - "f:" means option "-f" takes an argument + // - "c" means option "-c" does not take an argument + while ((opt = getopt_long(argc, argv, ":f:s:t:g:d:r:w:h", long_options, &option_index)) != -1) { + switch (opt) { + case 'f': { + filepaths.push_back(optarg); + break; + } + case 's': { + num_bytes = parse_size(optarg); // Helper to parse "1G", "500M", etc. + break; + } + case 't': { + num_threads = std::stoul(optarg); + break; + } + case 'g': { + use_gpu_buffer = parse_flag(optarg); + break; + } + case 'd': { + gpu_index = std::stoi(optarg); + break; + } + case 'r': { + repetition = std::stoi(optarg); + break; + } + case 'w': { + overwrite_file = parse_flag(optarg); + break; + } + case LongOnlyOpts::O_DIRECT: { + o_direct = parse_flag(optarg); + break; + } + case LongOnlyOpts::ALIGN_BUFFER: { + align_buffer = parse_flag(optarg); + break; + } + case LongOnlyOpts::DROP_CACHE: { + drop_file_cache = parse_flag(optarg); + break; + } + case LongOnlyOpts::OPEN_ONCE: { + open_file_once = parse_flag(optarg); + break; + } + case LongOnlyOpts::COMPAT_MODE: { + compat_mode = parse_flag(optarg); + break; + } + case LongOnlyOpts::PER_BLOCK_DEVICE_POOL: { + per_block_device_pool = parse_flag(optarg); + break; + } + case 'h': { + print_usage(argv[0]); + std::exit(0); + break; + } + case ':': { + // The parsed option has missing argument + std::stringstream ss; + ss << "Missing argument for option " << argv[optind - 1] << " (-" + << static_cast(optopt) << ")"; + throw std::runtime_error(ss.str()); + break; + } + default: { + // Unknown option is deferred to subsequent parsing, if any + break; + } + } + } + + // Validation + if (filepaths.empty()) { throw std::invalid_argument("--file is required"); } + + // Reset getopt state for second pass in the future + optind = 1; +} + +void Config::print_usage(std::string const& program_name) +{ + std::cout + << "Usage: " << program_name << " [OPTIONS]\n\n" + << "Options:\n" + << " -f, --file PATH File path to benchmark (required, repeatable)\n" + << " -s, --size SIZE Number of bytes to read (default: 4G)\n" + << " Supports suffixes: K, M, G, T, P\n" + << " -t, --threads NUM Number of threads (default: 1)\n" + << " -r, --repetitions NUM Number of repetitions (default: 5)\n" + << " -g, --use-gpu-buffer BOOL Use GPU device memory (default: false)\n" + << " -d, --gpu-index INDEX GPU device index (default: 0)\n" + << " -w, --overwrite BOOL Overwrite existing file (default: false)\n" + << " --o-direct BOOL Use O_DIRECT (default: true)\n" + << " --align-buffer BOOL Use aligned buffer (default: true)\n" + << " --drop-cache BOOL Drop page cache before each run (default: false)\n" + << " --open-once BOOL Open file once, not per repetition (default: false)\n" + << " --compat-mode BOOL Use compatibility mode (default: true)\n" + << " --per-block-device-pool BOOL Use per-block-device thread pools (default: true)\n" + << " -h, --help Show this help message\n"; +} + +void* CudaPageAlignedDeviceAllocator::allocate(std::size_t size) +{ + void* buffer{}; + auto const page_size = get_page_size(); + auto const up_size = size + page_size; + KVIKIO_CHECK_CUDA(cudaMalloc(&buffer, up_size)); + auto* aligned_buffer = kvikio::detail::align_up(buffer, page_size); + return aligned_buffer; +} + +void CudaPageAlignedDeviceAllocator::deallocate(void* buffer, std::size_t /*size*/) {} + +} // namespace kvikio::benchmark diff --git a/cpp/benchmarks/utils.hpp b/cpp/benchmarks/io/common.hpp similarity index 74% rename from cpp/benchmarks/utils.hpp rename to cpp/benchmarks/io/common.hpp index bd6ee47e7e..f32275786f 100644 --- a/cpp/benchmarks/utils.hpp +++ b/cpp/benchmarks/io/common.hpp @@ -5,6 +5,7 @@ #pragma once +#include #include #include #include @@ -15,20 +16,40 @@ #include #include +#define KVIKIO_CHECK_CUDA(err_code) kvikio::benchmark::check_cuda(err_code, __FILE__, __LINE__) + namespace kvikio::benchmark { +inline void check_cuda(cudaError_t err_code, const char* file, int line) +{ + if (err_code == cudaError_t::cudaSuccess) { return; } + std::stringstream ss; + int current_device{}; + cudaGetDevice(¤t_device); + ss << "CUDA runtime error on device " << current_device << ": " << cudaGetErrorName(err_code) + << " (" << err_code << "): " << cudaGetErrorString(err_code) << " at " << file << ":" << line + << "\n"; + throw std::runtime_error(ss.str()); +} + // Helper to parse size strings like "1GiB", "1Gi", "1G". std::size_t parse_size(std::string const& str); +bool parse_flag(std::string const& str); + struct Config { - std::size_t num_bytes{4ull * 1024ull * 1024ull * 1024ull}; std::vector filepaths; - bool align_buffer{true}; + std::size_t num_bytes{4ull * 1024ull * 1024ull * 1024ull}; + unsigned int num_threads{1}; + bool use_gpu_buffer{false}; + int gpu_index{0}; + int repetition{5}; + bool overwrite_file{false}; bool o_direct{true}; + bool align_buffer{true}; bool drop_file_cache{false}; - bool compat_mode{true}; - unsigned int num_threads{1}; bool open_file_once{false}; - int repetition{5}; + bool compat_mode{true}; + bool per_block_device_pool{true}; virtual void parse_args(int argc, char** argv); virtual void print_usage(std::string const& program_name); @@ -50,6 +71,7 @@ class Benchmark { defaults::set_thread_pool_nthreads(_config.num_threads); auto compat_mode = _config.compat_mode ? CompatMode::ON : CompatMode::OFF; defaults::set_compat_mode(compat_mode); + defaults::set_thread_pool_per_block_device(_config.per_block_device_pool); } void run() @@ -86,4 +108,11 @@ class Benchmark { if (_config.open_file_once) { cleanup(); } } }; + +class CudaPageAlignedDeviceAllocator { + public: + void* allocate(std::size_t size); + + void deallocate(void* buffer, std::size_t size); +}; } // namespace kvikio::benchmark diff --git a/cpp/benchmarks/local/posix_benchmark.cpp b/cpp/benchmarks/io/sequential_benchmark.cpp similarity index 61% rename from cpp/benchmarks/local/posix_benchmark.cpp rename to cpp/benchmarks/io/sequential_benchmark.cpp index a7f2bfdf61..72780838c2 100644 --- a/cpp/benchmarks/local/posix_benchmark.cpp +++ b/cpp/benchmarks/io/sequential_benchmark.cpp @@ -3,16 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "posix_benchmark.hpp" -#include "../utils.hpp" +#include "sequential_benchmark.hpp" +#include "common.hpp" #include +#include #include #include #include #include +#include #include #include #include @@ -21,29 +23,19 @@ #include namespace kvikio::benchmark { -void PosixConfig::parse_args(int argc, char** argv) +void SequentialConfig::parse_args(int argc, char** argv) { Config::parse_args(argc, argv); static option long_options[] = { - {"overwrite", no_argument, nullptr, 'w'}, {0, 0, 0, 0} - // Sentinel to mark the end of the array. Needed by getopt_long() + {0, 0, 0, 0} // Sentinel to mark the end of the array. Needed by getopt_long() + }; int opt{0}; int option_index{-1}; - // "f:"" means "-f" takes an argument - // "c" means "-c" does not take an argument - while ((opt = getopt_long(argc, argv, ":wp", long_options, &option_index)) != -1) { + while ((opt = getopt_long(argc, argv, ":", long_options, &option_index)) != -1) { switch (opt) { - case 'w': { - overwrite_file = true; - break; - } - case 'p': { - per_drive_pool = true; - break; - } case ':': { // The parsed option has missing argument std::stringstream ss; @@ -63,33 +55,36 @@ void PosixConfig::parse_args(int argc, char** argv) optind = 1; } -void PosixConfig::print_usage(std::string const& program_name) +void SequentialConfig::print_usage(std::string const& program_name) { Config::print_usage(program_name); - std::cout << " -w, --overwrite Overwrite existing file\n"; } -PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config)) +SequentialBenchmark::SequentialBenchmark(SequentialConfig config) : Benchmark(std::move(config)) { for (auto const& filepath : _config.filepaths) { // Initialize buffer void* buf{}; - if (_config.align_buffer) { - auto const page_size = get_page_size(); - auto const aligned_size = kvikio::detail::align_up(_config.num_bytes, page_size); - buf = std::aligned_alloc(page_size, aligned_size); + if (_config.use_gpu_buffer) { + KVIKIO_CHECK_CUDA(cudaSetDevice(_config.gpu_index)); + if (_config.align_buffer) { + CudaPageAlignedDeviceAllocator alloc; + buf = alloc.allocate(_config.num_bytes); + } else { + KVIKIO_CHECK_CUDA(cudaMalloc(&buf, _config.num_bytes)); + } } else { - buf = std::malloc(_config.num_bytes); + if (_config.align_buffer) { + kvikio::PageAlignedAllocator alloc; + buf = alloc.allocate(_config.num_bytes); + } else { + buf = std::malloc(_config.num_bytes); + } } - std::memset(buf, 0, _config.num_bytes); - _bufs.push_back(buf); - // Initialize KvikIO setting - if (_config.per_drive_pool) { kvikio::defaults::set_thread_pool_per_block_device(true); } - // Initialize file // Create the file if the overwrite flag is on, or if the file does not exist. if (_config.overwrite_file || access(filepath.c_str(), F_OK) != 0) { @@ -100,14 +95,21 @@ PosixBenchmark::PosixBenchmark(PosixConfig config) : Benchmark(std::move(config) } } -PosixBenchmark::~PosixBenchmark() +SequentialBenchmark::~SequentialBenchmark() { for (auto&& buf : _bufs) { - std::free(buf); + if (_config.use_gpu_buffer) { + if (_config.align_buffer) { + } else { + KVIKIO_CHECK_CUDA(cudaFree(buf)); + } + } else { + std::free(buf); + } } } -void PosixBenchmark::initialize_impl() +void SequentialBenchmark::initialize_impl() { _file_handles.clear(); @@ -124,14 +126,14 @@ void PosixBenchmark::initialize_impl() } } -void PosixBenchmark::cleanup_impl() +void SequentialBenchmark::cleanup_impl() { for (auto&& file_handle : _file_handles) { file_handle->close(); } } -void PosixBenchmark::run_target_impl() +void SequentialBenchmark::run_target_impl() { std::vector> futs; @@ -140,11 +142,7 @@ void PosixBenchmark::run_target_impl() auto* buf = _bufs[i]; std::future fut; - if (_config.per_drive_pool) { - fut = file_handle->pread(buf, _config.num_bytes); - } else { - fut = file_handle->pread(buf, _config.num_bytes); - } + fut = file_handle->pread(buf, _config.num_bytes); futs.push_back(std::move(fut)); } @@ -153,15 +151,18 @@ void PosixBenchmark::run_target_impl() } } -std::size_t PosixBenchmark::nbytes_impl() { return _config.num_bytes * _config.filepaths.size(); } +std::size_t SequentialBenchmark::nbytes_impl() +{ + return _config.num_bytes * _config.filepaths.size(); +} } // namespace kvikio::benchmark int main(int argc, char* argv[]) { try { - kvikio::benchmark::PosixConfig config; + kvikio::benchmark::SequentialConfig config; config.parse_args(argc, argv); - kvikio::benchmark::PosixBenchmark bench(std::move(config)); + kvikio::benchmark::SequentialBenchmark bench(std::move(config)); bench.run(); } catch (std::exception const& e) { std::cerr << "Error: " << e.what() << std::endl; diff --git a/cpp/benchmarks/local/posix_benchmark.hpp b/cpp/benchmarks/io/sequential_benchmark.hpp similarity index 72% rename from cpp/benchmarks/local/posix_benchmark.hpp rename to cpp/benchmarks/io/sequential_benchmark.hpp index 11e6a81c2c..5f426bde16 100644 --- a/cpp/benchmarks/local/posix_benchmark.hpp +++ b/cpp/benchmarks/io/sequential_benchmark.hpp @@ -5,7 +5,7 @@ #pragma once -#include "../utils.hpp" +#include "common.hpp" #include #include @@ -17,16 +17,13 @@ namespace kvikio::benchmark { -struct PosixConfig : Config { - bool overwrite_file{false}; - bool per_drive_pool{false}; - +struct SequentialConfig : Config { virtual void parse_args(int argc, char** argv) override; virtual void print_usage(std::string const& program_name) override; }; -class PosixBenchmark : public Benchmark { - friend class Benchmark; +class SequentialBenchmark : public Benchmark { + friend class Benchmark; protected: std::vector> _file_handles; @@ -39,7 +36,7 @@ class PosixBenchmark : public Benchmark { std::size_t nbytes_impl(); public: - PosixBenchmark(PosixConfig config); - ~PosixBenchmark(); + SequentialBenchmark(SequentialConfig config); + ~SequentialBenchmark(); }; } // namespace kvikio::benchmark diff --git a/cpp/benchmarks/utils.cpp b/cpp/benchmarks/utils.cpp deleted file mode 100644 index af85ec3556..0000000000 --- a/cpp/benchmarks/utils.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "utils.hpp" - -#include - -#include -#include -#include - -namespace kvikio::benchmark { - -std::size_t parse_size(std::string const& str) -{ - if (str.empty()) { throw std::invalid_argument("Empty size string"); } - - // Parse the numeric part - std::size_t pos{}; - double value{}; - try { - value = std::stod(str, &pos); - } catch (std::exception const& e) { - throw std::invalid_argument("Invalid size format: " + str); - } - - if (value < 0) { throw std::invalid_argument("Size cannot be negative"); } - - // Extract suffix (everything after the number) - auto suffix = str.substr(pos); - - // No suffix means raw bytes - if (suffix.empty()) { return static_cast(value); } - - // Normalize to uppercase for case-insensitive comparison - std::transform( - suffix.begin(), suffix.end(), suffix.begin(), [](unsigned char c) { return std::tolower(c); }); - - // All multipliers use 1024 (binary), not 1000 - std::size_t multiplier{1}; - - // Support both K/Ki, M/Mi, etc. as synonyms (all 1024-based) - std::size_t constexpr one_Ki{1024ULL}; - std::size_t constexpr one_Mi{1024ULL * one_Ki}; - std::size_t constexpr one_Gi{1024ULL * one_Mi}; - std::size_t constexpr one_Ti{1024ULL * one_Gi}; - if (suffix == "k" || suffix == "ki" || suffix == "kib") { - multiplier = one_Ki; - } else if (suffix == "m" || suffix == "mi" || suffix == "mib") { - multiplier = one_Mi; - } else if (suffix == "g" || suffix == "gi" || suffix == "gib") { - multiplier = one_Gi; - } else if (suffix == "t" || suffix == "ti" || suffix == "tib") { - multiplier = one_Ti; - } else { - throw std::invalid_argument("Invalid size suffix: '" + suffix + - "' (use K/Ki/KiB, M/Mi/MiB, G/Gi/GiB, or T/Ti/TiB)"); - } - - return static_cast(value * multiplier); -} - -void Config::parse_args(int argc, char** argv) -{ - static option long_options[] = { - {"file", required_argument, nullptr, 'f'}, - {"size", required_argument, nullptr, 's'}, - {"threads", required_argument, nullptr, 't'}, - {"repetitions", required_argument, nullptr, 'r'}, - {"no-direct", no_argument, nullptr, 'D'}, - {"no-align", no_argument, nullptr, 'A'}, - {"drop-cache", no_argument, nullptr, 'c'}, - // {"overwrite", no_argument, nullptr, 'w'}, - {"open-once", no_argument, nullptr, 'o'}, - {"help", no_argument, nullptr, 'h'}, - {0, 0, 0, 0} // Sentinel to mark the end of the array. Needed by getopt_long() - }; - - int opt{0}; - int option_index{-1}; - - // - By default getopt_long() returns '?' to indicate errors if an option has missing argument or - // if an unknown option is encountered. The starting ':' in the optstring modifies this behavior. - // Missing argument error now causes the return value to be ':'. Unknow option still leads to '?' - // and its processing is deferred. - // - "f:" means option "-f" takes an argument "c" means option - // - "-c" does not take an argument - while ((opt = getopt_long(argc, argv, ":f:s:t:r:DAcoh", long_options, &option_index)) != -1) { - switch (opt) { - case 'f': { - filepaths.push_back(optarg); - break; - } - case 's': { - num_bytes = parse_size(optarg); // Helper to parse "1G", "500M", etc. - break; - } - case 't': { - num_threads = std::stoul(optarg); - break; - } - case 'r': { - repetition = std::stoi(optarg); - break; - } - case 'D': { - o_direct = false; - break; - } - case 'A': { - align_buffer = false; - break; - } - case 'c': { - drop_file_cache = true; - break; - } - case 'o': { - open_file_once = true; - break; - } - case 'h': { - print_usage(argv[0]); - std::exit(0); - break; - } - case ':': { - // The parsed option has missing argument - std::stringstream ss; - ss << "Missing argument for option " << argv[optind - 1] << " (-" - << static_cast(optopt) << ")"; - throw std::runtime_error(ss.str()); - break; - } - default: { - // Unknown option is deferred to subsequent parsing, if any - break; - } - } - } - - // Validation - if (filepaths.empty()) { throw std::invalid_argument("--file is required"); } - - // Reset getopt state for second pass in the future - optind = 1; -} - -void Config::print_usage(std::string const& program_name) -{ - std::cout << "Usage: " << program_name << " [OPTIONS]\n\n" - << "Options:\n" - << " -f, --file PATH File path to benchmark (required)\n" - << " -s, --size SIZE Number of bytes to read (default: 4G)\n" - << " Supports suffixes: K, M, G, T\n" - << " -t, --threads NUM Number of threads (default: 1)\n" - << " -r, --repetitions NUM Number of repetitions (default: 5)\n" - << " -D, --no-direct Disable O_DIRECT (use buffered I/O)\n" - << " -A, --no-align Disable buffer alignment\n" - << " -c, --drop-cache Drop page cache before each run\n" - << " -o, --open-once Open file once (not per iteration)\n" - << " -h, --help Show this help message\n"; -} - -} // namespace kvikio::benchmark From 439c1633cc9b23315ffcb048c95e6936b05effc5 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 19 Dec 2025 16:39:42 -0500 Subject: [PATCH 30/31] Update --- cpp/benchmarks/io/common.cpp | 14 -------------- cpp/benchmarks/io/common.hpp | 6 ------ 2 files changed, 20 deletions(-) diff --git a/cpp/benchmarks/io/common.cpp b/cpp/benchmarks/io/common.cpp index cb8d33e7b5..2b27b0dc56 100644 --- a/cpp/benchmarks/io/common.cpp +++ b/cpp/benchmarks/io/common.cpp @@ -94,8 +94,6 @@ void Config::parse_args(int argc, char** argv) ALIGN_BUFFER, DROP_CACHE, OPEN_ONCE, - COMPAT_MODE, - PER_BLOCK_DEVICE_POOL, }; static option long_options[] = { @@ -110,8 +108,6 @@ void Config::parse_args(int argc, char** argv) {"drop-cache", required_argument, nullptr, LongOnlyOpts::DROP_CACHE}, {"overwrite", required_argument, nullptr, 'w'}, {"open-once", required_argument, nullptr, LongOnlyOpts::OPEN_ONCE}, - {"compat-mode", required_argument, nullptr, LongOnlyOpts::COMPAT_MODE}, - {"per-block-device-pool", required_argument, nullptr, LongOnlyOpts::PER_BLOCK_DEVICE_POOL}, {"help", no_argument, nullptr, 'h'}, {0, 0, 0, 0} // Sentinel to mark the end of the array. Needed by getopt_long() }; @@ -171,14 +167,6 @@ void Config::parse_args(int argc, char** argv) open_file_once = parse_flag(optarg); break; } - case LongOnlyOpts::COMPAT_MODE: { - compat_mode = parse_flag(optarg); - break; - } - case LongOnlyOpts::PER_BLOCK_DEVICE_POOL: { - per_block_device_pool = parse_flag(optarg); - break; - } case 'h': { print_usage(argv[0]); std::exit(0); @@ -223,8 +211,6 @@ void Config::print_usage(std::string const& program_name) << " --align-buffer BOOL Use aligned buffer (default: true)\n" << " --drop-cache BOOL Drop page cache before each run (default: false)\n" << " --open-once BOOL Open file once, not per repetition (default: false)\n" - << " --compat-mode BOOL Use compatibility mode (default: true)\n" - << " --per-block-device-pool BOOL Use per-block-device thread pools (default: true)\n" << " -h, --help Show this help message\n"; } diff --git a/cpp/benchmarks/io/common.hpp b/cpp/benchmarks/io/common.hpp index f32275786f..9c27f352ae 100644 --- a/cpp/benchmarks/io/common.hpp +++ b/cpp/benchmarks/io/common.hpp @@ -12,7 +12,6 @@ #include #include -#include #include #include @@ -48,8 +47,6 @@ struct Config { bool align_buffer{true}; bool drop_file_cache{false}; bool open_file_once{false}; - bool compat_mode{true}; - bool per_block_device_pool{true}; virtual void parse_args(int argc, char** argv); virtual void print_usage(std::string const& program_name); @@ -69,9 +66,6 @@ class Benchmark { Benchmark(ConfigType config) : _config(std::move(config)) { defaults::set_thread_pool_nthreads(_config.num_threads); - auto compat_mode = _config.compat_mode ? CompatMode::ON : CompatMode::OFF; - defaults::set_compat_mode(compat_mode); - defaults::set_thread_pool_per_block_device(_config.per_block_device_pool); } void run() From e6d2e07c6ba59fad0cfe185a62f71469276d0d8e Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 23 Dec 2025 23:20:56 -0500 Subject: [PATCH 31/31] Update --- cpp/benchmarks/io/common.cpp | 83 +++++++++- cpp/benchmarks/io/common.hpp | 123 ++++++++++++++- cpp/benchmarks/io/sequential_benchmark.cpp | 173 +++++++++++++-------- cpp/benchmarks/io/sequential_benchmark.hpp | 42 +++-- 4 files changed, 343 insertions(+), 78 deletions(-) diff --git a/cpp/benchmarks/io/common.cpp b/cpp/benchmarks/io/common.cpp index 2b27b0dc56..de25dae56b 100644 --- a/cpp/benchmarks/io/common.cpp +++ b/cpp/benchmarks/io/common.cpp @@ -10,11 +10,29 @@ #include #include #include -#include "kvikio/detail/utils.hpp" -#include "kvikio/utils.hpp" + +#include namespace kvikio::benchmark { +Backend parse_backend(std::string const& str) +{ + if (str.empty()) { throw std::invalid_argument("Empty size string"); } + + // Normalize to lowercase for case-insensitive comparison + auto backend{str}; + std::transform(backend.begin(), backend.end(), backend.begin(), [](unsigned char c) { + return std::tolower(c); + }); + if (backend == "filehandle") { + return Backend::FILEHANDLE; + } else if (backend == "cufile") { + return Backend::CUFILE; + } else { + throw std::invalid_argument("Invalid flag: '" + str + "' (use \"FileHandle\", \"cuFile\") "); + } +} + std::size_t parse_size(std::string const& str) { if (str.empty()) { throw std::invalid_argument("Empty size string"); } @@ -87,6 +105,44 @@ bool parse_flag(std::string const& str) } } +Backend parse_backend(int argc, char** argv) +{ + constexpr int BACKEND = 1000; + Backend result{FILEHANDLE}; + static option long_options[] = { + {"backend", required_argument, nullptr, BACKEND}, {0, 0, 0, 0} + // Sentinel to mark the end of the array. Needed by getopt_long() + }; + + int opt{0}; + int option_index{-1}; + while ((opt = getopt_long(argc, argv, "-:", long_options, &option_index)) != -1) { + switch (opt) { + case BACKEND: { + result = parse_backend(optarg); + break; + } + case ':': { + // The parsed option has missing argument + std::stringstream ss; + ss << "Missing argument for option " << argv[optind - 1] << " (-" + << static_cast(optopt) << ")"; + throw std::runtime_error(ss.str()); + break; + } + default: { + // Unknown option is deferred to subsequent parsing, if any + break; + } + } + } + + // Reset getopt state for second pass in the future + optind = 0; + + return result; +} + void Config::parse_args(int argc, char** argv) { enum LongOnlyOpts { @@ -121,7 +177,7 @@ void Config::parse_args(int argc, char** argv) // and its processing is deferred. // - "f:" means option "-f" takes an argument // - "c" means option "-c" does not take an argument - while ((opt = getopt_long(argc, argv, ":f:s:t:g:d:r:w:h", long_options, &option_index)) != -1) { + while ((opt = getopt_long(argc, argv, "-:f:s:t:g:d:r:w:h", long_options, &option_index)) != -1) { switch (opt) { case 'f': { filepaths.push_back(optarg); @@ -191,7 +247,7 @@ void Config::parse_args(int argc, char** argv) if (filepaths.empty()) { throw std::invalid_argument("--file is required"); } // Reset getopt state for second pass in the future - optind = 1; + optind = 0; } void Config::print_usage(std::string const& program_name) @@ -220,10 +276,27 @@ void* CudaPageAlignedDeviceAllocator::allocate(std::size_t size) auto const page_size = get_page_size(); auto const up_size = size + page_size; KVIKIO_CHECK_CUDA(cudaMalloc(&buffer, up_size)); - auto* aligned_buffer = kvikio::detail::align_up(buffer, page_size); + auto* aligned_buffer = detail::align_up(buffer, page_size); return aligned_buffer; } void CudaPageAlignedDeviceAllocator::deallocate(void* buffer, std::size_t /*size*/) {} +CuFileHandle::CuFileHandle(std::string const& file_path, + std::string const& flags, + bool o_direct, + mode_t mode) + : _file_wrapper(file_path, flags, o_direct, mode) +{ + _cufile_handle_wrapper.register_handle(_file_wrapper.fd()); +} + +void CuFileHandle::close() +{ + _cufile_handle_wrapper.unregister_handle(); + _file_wrapper.close(); +} + +CUfileHandle_t CuFileHandle::handle() const noexcept { return _cufile_handle_wrapper.handle(); } + } // namespace kvikio::benchmark diff --git a/cpp/benchmarks/io/common.hpp b/cpp/benchmarks/io/common.hpp index 9c27f352ae..c150c3f2cf 100644 --- a/cpp/benchmarks/io/common.hpp +++ b/cpp/benchmarks/io/common.hpp @@ -10,9 +10,12 @@ #include #include #include +#include #include +#include #include +#include #include #define KVIKIO_CHECK_CUDA(err_code) kvikio::benchmark::check_cuda(err_code, __FILE__, __LINE__) @@ -30,6 +33,14 @@ inline void check_cuda(cudaError_t err_code, const char* file, int line) throw std::runtime_error(ss.str()); } +enum Backend { + FILEHANDLE, + CUFILE, +}; + +Backend parse_backend(std::string const& str); +Backend parse_backend(int argc, char** argv); + // Helper to parse size strings like "1GiB", "1Gi", "1G". std::size_t parse_size(std::string const& str); @@ -55,7 +66,7 @@ struct Config { template class Benchmark { protected: - ConfigType _config; + ConfigType const& _config; void initialize() { static_cast(this)->initialize_impl(); } void cleanup() { static_cast(this)->cleanup_impl(); } @@ -63,7 +74,7 @@ class Benchmark { std::size_t nbytes() { return static_cast(this)->nbytes_impl(); } public: - Benchmark(ConfigType config) : _config(std::move(config)) + Benchmark(ConfigType const& config) : _config(config) { defaults::set_thread_pool_nthreads(_config.num_threads); } @@ -109,4 +120,112 @@ class CudaPageAlignedDeviceAllocator { void deallocate(void* buffer, std::size_t size); }; + +template +class Buffer { + public: + Buffer(ConfigType config) : _config(config) { allocate(); } + ~Buffer() { deallocate(); } + + Buffer(Buffer const&) = delete; + Buffer& operator=(Buffer const&) = delete; + + Buffer(Buffer&& o) noexcept + : _config(std::exchange(o._config, {})), + _data(std::exchange(o._data, {})), + _original_data(std::exchange(o._original_data, {})) + { + } + + Buffer& operator=(Buffer&& o) noexcept + { + if (this == &o) { return *this; } + deallocate(); + _config = std::exchange(o._config, {}); + _data = std::exchange(o._data, {}); + _original_data = std::exchange(o._original_data, {}); + } + + void* data() const { return _data; } + void* size() const { return _size; } + + private: + void allocate() + { + if (_config.use_gpu_buffer) { + KVIKIO_CHECK_CUDA(cudaSetDevice(_config.gpu_index)); + if (_config.align_buffer) { + CudaPageAlignedDeviceAllocator alloc; + _original_data = alloc.allocate(_config.num_bytes); + } else { + KVIKIO_CHECK_CUDA(cudaMalloc(&_original_data, _config.num_bytes)); + } + } else { + if (_config.align_buffer) { + PageAlignedAllocator alloc; + _original_data = alloc.allocate(_config.num_bytes); + } else { + _original_data = std::malloc(_config.num_bytes); + } + } + + _data = _original_data; + } + + void deallocate() + { + if (_config.use_gpu_buffer) { + if (_config.align_buffer) { + } else { + KVIKIO_CHECK_CUDA(cudaFree(_original_data)); + } + } else { + std::free(_original_data); + } + + _data = nullptr; + _original_data = nullptr; + _size = 0; + } + + ConfigType _config; + void* _data{}; + void* _original_data{}; + std::size_t _size{}; +}; + +template +void create_file(ConfigType const& config, + std::string const& filepath, + Buffer const& buf) +{ + // Create the file if the overwrite flag is on, or if the file does not exist, or if the file size + // is wrong. + if (config.overwrite_file || access(filepath.c_str(), F_OK) != 0 || + get_file_size(filepath) != config.num_bytes) { + FileHandle file_handle(filepath, "w", FileHandle::m644); + auto fut = file_handle.pwrite(buf.data(), config.num_bytes); + fut.get(); + } +} + +class CuFileHandle { + public: + CuFileHandle(std::string const& file_path, std::string const& flags, bool o_direct, mode_t mode); + ~CuFileHandle() = default; + + CuFileHandle(CuFileHandle const&) = delete; + CuFileHandle& operator=(CuFileHandle const&) = delete; + + CuFileHandle(CuFileHandle&&) noexcept = default; + CuFileHandle& operator=(CuFileHandle&&) = default; + + void close(); + + CUfileHandle_t handle() const noexcept; + + private: + FileWrapper _file_wrapper; + CUFileHandleWrapper _cufile_handle_wrapper; +}; } // namespace kvikio::benchmark diff --git a/cpp/benchmarks/io/sequential_benchmark.cpp b/cpp/benchmarks/io/sequential_benchmark.cpp index 72780838c2..82f12ca4dc 100644 --- a/cpp/benchmarks/io/sequential_benchmark.cpp +++ b/cpp/benchmarks/io/sequential_benchmark.cpp @@ -5,6 +5,7 @@ #include "sequential_benchmark.hpp" #include "common.hpp" +#include "kvikio/shim/cufile.hpp" #include @@ -14,7 +15,6 @@ #include #include -#include #include #include #include @@ -23,7 +23,8 @@ #include namespace kvikio::benchmark { -void SequentialConfig::parse_args(int argc, char** argv) + +void KvikIOSequentialConfig::parse_args(int argc, char** argv) { Config::parse_args(argc, argv); static option long_options[] = { @@ -34,7 +35,7 @@ void SequentialConfig::parse_args(int argc, char** argv) int opt{0}; int option_index{-1}; - while ((opt = getopt_long(argc, argv, ":", long_options, &option_index)) != -1) { + while ((opt = getopt_long(argc, argv, "-:", long_options, &option_index)) != -1) { switch (opt) { case ':': { // The parsed option has missing argument @@ -52,69 +53,31 @@ void SequentialConfig::parse_args(int argc, char** argv) } // Reset getopt state for second pass in the future - optind = 1; + optind = 0; } -void SequentialConfig::print_usage(std::string const& program_name) +void KvikIOSequentialConfig::print_usage(std::string const& program_name) { Config::print_usage(program_name); } -SequentialBenchmark::SequentialBenchmark(SequentialConfig config) : Benchmark(std::move(config)) +KvikIOSequentialBenchmark::KvikIOSequentialBenchmark(KvikIOSequentialConfig const& config) + : Benchmark(config) { for (auto const& filepath : _config.filepaths) { - // Initialize buffer - void* buf{}; - - if (_config.use_gpu_buffer) { - KVIKIO_CHECK_CUDA(cudaSetDevice(_config.gpu_index)); - if (_config.align_buffer) { - CudaPageAlignedDeviceAllocator alloc; - buf = alloc.allocate(_config.num_bytes); - } else { - KVIKIO_CHECK_CUDA(cudaMalloc(&buf, _config.num_bytes)); - } - } else { - if (_config.align_buffer) { - kvikio::PageAlignedAllocator alloc; - buf = alloc.allocate(_config.num_bytes); - } else { - buf = std::malloc(_config.num_bytes); - } - } - - _bufs.push_back(buf); - - // Initialize file - // Create the file if the overwrite flag is on, or if the file does not exist. - if (_config.overwrite_file || access(filepath.c_str(), F_OK) != 0) { - kvikio::FileHandle file_handle(filepath, "w", kvikio::FileHandle::m644); - auto fut = file_handle.pwrite(buf, _config.num_bytes); - fut.get(); - } + _bufs.emplace_back(std::make_unique>(_config)); + create_file(_config, filepath, *_bufs.back()); } } -SequentialBenchmark::~SequentialBenchmark() -{ - for (auto&& buf : _bufs) { - if (_config.use_gpu_buffer) { - if (_config.align_buffer) { - } else { - KVIKIO_CHECK_CUDA(cudaFree(buf)); - } - } else { - std::free(buf); - } - } -} +KvikIOSequentialBenchmark::~KvikIOSequentialBenchmark() {} -void SequentialBenchmark::initialize_impl() +void KvikIOSequentialBenchmark::initialize_impl() { _file_handles.clear(); for (auto const& filepath : _config.filepaths) { - auto p = std::make_unique(filepath, "r"); + auto p = std::make_unique(filepath, "r"); if (_config.o_direct) { auto file_status_flags = fcntl(p->fd(), F_GETFL); @@ -126,23 +89,19 @@ void SequentialBenchmark::initialize_impl() } } -void SequentialBenchmark::cleanup_impl() +void KvikIOSequentialBenchmark::cleanup_impl() { for (auto&& file_handle : _file_handles) { file_handle->close(); } } -void SequentialBenchmark::run_target_impl() +void KvikIOSequentialBenchmark::run_target_impl() { std::vector> futs; for (std::size_t i = 0; i < _file_handles.size(); ++i) { - auto& file_handle = _file_handles[i]; - auto* buf = _bufs[i]; - - std::future fut; - fut = file_handle->pread(buf, _config.num_bytes); + auto fut = _file_handles[i]->pread(_bufs[i]->data(), _config.num_bytes); futs.push_back(std::move(fut)); } @@ -151,7 +110,88 @@ void SequentialBenchmark::run_target_impl() } } -std::size_t SequentialBenchmark::nbytes_impl() +std::size_t KvikIOSequentialBenchmark::nbytes_impl() +{ + return _config.num_bytes * _config.filepaths.size(); +} + +void CuFileSequentialConfig::parse_args(int argc, char** argv) +{ + Config::parse_args(argc, argv); + static option long_options[] = { + {0, 0, 0, 0} // Sentinel to mark the end of the array. Needed by getopt_long() + + }; + + int opt{0}; + int option_index{-1}; + + while ((opt = getopt_long(argc, argv, "-:", long_options, &option_index)) != -1) { + switch (opt) { + case ':': { + // The parsed option has missing argument + std::stringstream ss; + ss << "Missing argument for option " << argv[optind - 1] << " (-" + << static_cast(optopt) << ")"; + throw std::runtime_error(ss.str()); + break; + } + default: { + // Unknown option is deferred to subsequent parsing, if any + break; + } + } + } + + // Reset getopt state for second pass in the future + optind = 0; +} + +void CuFileSequentialConfig::print_usage(std::string const& program_name) +{ + Config::print_usage(program_name); +} + +CuFileSequentialBenchmark::CuFileSequentialBenchmark(CuFileSequentialConfig const& config) + : Benchmark(config) +{ + for (auto const& filepath : _config.filepaths) { + _bufs.emplace_back(std::make_unique>(_config)); + create_file(_config, filepath, *_bufs.back()); + } +} + +CuFileSequentialBenchmark::~CuFileSequentialBenchmark() {} + +void CuFileSequentialBenchmark::initialize_impl() +{ + _file_handles.clear(); + + for (auto const& filepath : _config.filepaths) { + auto o_direct = _config.o_direct; + auto p = std::make_unique(filepath, "r", o_direct, FileHandle::m644); + _file_handles.push_back(std::move(p)); + } +} + +void CuFileSequentialBenchmark::cleanup_impl() +{ + for (auto&& file_handle : _file_handles) { + file_handle->close(); + } +} + +void CuFileSequentialBenchmark::run_target_impl() +{ + for (std::size_t i = 0; i < _file_handles.size(); ++i) { + off_t file_offset{0}; + off_t dev_ptr_offset{0}; + cuFileAPI::instance().Read( + _file_handles[i]->handle(), _bufs[i]->data(), _config.num_bytes, file_offset, dev_ptr_offset); + } +} + +std::size_t CuFileSequentialBenchmark::nbytes_impl() { return _config.num_bytes * _config.filepaths.size(); } @@ -160,10 +200,19 @@ std::size_t SequentialBenchmark::nbytes_impl() int main(int argc, char* argv[]) { try { - kvikio::benchmark::SequentialConfig config; - config.parse_args(argc, argv); - kvikio::benchmark::SequentialBenchmark bench(std::move(config)); - bench.run(); + auto backend = kvikio::benchmark::parse_backend(argc, argv); + + if (backend == kvikio::benchmark::Backend::FILEHANDLE) { + kvikio::benchmark::KvikIOSequentialConfig config; + config.parse_args(argc, argv); + kvikio::benchmark::KvikIOSequentialBenchmark bench(config); + bench.run(); + } else { + kvikio::benchmark::CuFileSequentialConfig config; + config.parse_args(argc, argv); + kvikio::benchmark::CuFileSequentialBenchmark bench(config); + bench.run(); + } } catch (std::exception const& e) { std::cerr << "Error: " << e.what() << std::endl; return 1; diff --git a/cpp/benchmarks/io/sequential_benchmark.hpp b/cpp/benchmarks/io/sequential_benchmark.hpp index 5f426bde16..40bb46a8ba 100644 --- a/cpp/benchmarks/io/sequential_benchmark.hpp +++ b/cpp/benchmarks/io/sequential_benchmark.hpp @@ -13,22 +13,22 @@ #include #include -#include +#include namespace kvikio::benchmark { -struct SequentialConfig : Config { +struct KvikIOSequentialConfig : Config { virtual void parse_args(int argc, char** argv) override; virtual void print_usage(std::string const& program_name) override; }; -class SequentialBenchmark : public Benchmark { - friend class Benchmark; +class KvikIOSequentialBenchmark + : public Benchmark { + friend class Benchmark; protected: - std::vector> _file_handles; - std::vector _bufs; - std::vector> _thread_pools; + std::vector> _file_handles; + std::vector>> _bufs; void initialize_impl(); void cleanup_impl(); @@ -36,7 +36,31 @@ class SequentialBenchmark : public Benchmark { + friend class Benchmark; + + protected: + std::vector> _file_handles; + std::vector>> _bufs; + + void initialize_impl(); + void cleanup_impl(); + void run_target_impl(); + std::size_t nbytes_impl(); + + public: + CuFileSequentialBenchmark(CuFileSequentialConfig const& config); + ~CuFileSequentialBenchmark(); +}; + } // namespace kvikio::benchmark