diff --git a/conda/recipes/libkvikio/recipe.yaml b/conda/recipes/libkvikio/recipe.yaml index bd49a3091b..f6d2444ab0 100644 --- a/conda/recipes/libkvikio/recipe.yaml +++ b/conda/recipes/libkvikio/recipe.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 schema_version: 1 @@ -112,6 +112,8 @@ outputs: tests: - script: - test -f $PREFIX/include/kvikio/file_handle.hpp + - test -x $PREFIX/share/kvikio/nsys-plugins/kvikio_nic/kvikio_nic_nsys_plugin + - test -f $PREFIX/share/kvikio/nsys-plugins/kvikio_nic/nsys-plugin.yaml about: homepage: ${{ load_from_file("python/libkvikio/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libkvikio/pyproject.toml").project.license }} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dfc4860592..32a5aed3c6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -52,6 +52,7 @@ option(KvikIO_BUILD_BENCHMARKS "Configure CMake to build benchmarks" OFF) option(KvikIO_BUILD_EXAMPLES "Configure CMake to build examples" ON) option(KvikIO_BUILD_TESTS "Configure CMake to build tests" ON) option(KvikIO_REMOTE_SUPPORT "Configure CMake to build with remote IO support" ON) +option(KvikIO_BUILD_NSYS_PLUGIN "Configure CMake to build the Nsight Systems NIC plugin" ON) # ################################################################################################## # * conda environment ------------------------------------------------------------------------------ @@ -264,6 +265,13 @@ if(KvikIO_BUILD_EXAMPLES) add_subdirectory(examples) endif() +# ################################################################################################## +# * add Nsight Systems plugin ---------------------------------------------------------------------- + +if(KvikIO_BUILD_NSYS_PLUGIN) + add_subdirectory(nsys_plugins/nic) +endif() + if(KvikIO_BUILD_TESTS AND CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) include(cmake/thirdparty/get_gtest.cmake) diff --git a/cpp/nsys_plugins/nic/CMakeLists.txt b/cpp/nsys_plugins/nic/CMakeLists.txt new file mode 100644 index 0000000000..d787955183 --- /dev/null +++ b/cpp/nsys_plugins/nic/CMakeLists.txt @@ -0,0 +1,40 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +# The plugin is a standalone Nsight Systems collector. It reads NVTX headers only (header-only) and +# libdl, so it deliberately does not link libkvikio and is fully relocatable. +set(NSYS_PLUGIN_INSTALL_DIR share/kvikio/nsys-plugins/kvikio_nic) + +# The NIC-reading core (nic_monitor.cpp) is compiled directly into the plugin so it links no shared +# library beyond NVTX (header-only) and libdl. +add_executable(kvikio_nic_nsys_plugin kvikio_nic_nsys_plugin.cpp nic_monitor.cpp) + +set_target_properties( + kvikio_nic_nsys_plugin + PROPERTIES CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + INSTALL_RPATH "" +) + +target_link_libraries(kvikio_nic_nsys_plugin PRIVATE nvtx3::nvtx3-cpp ${CMAKE_DL_LIBS}) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options( + kvikio_nic_nsys_plugin PRIVATE "$<$:-Wall;-Werror;-Wno-unknown-pragmas>" + ) +endif() + +install( + TARGETS kvikio_nic_nsys_plugin + COMPONENT nsys_plugin + DESTINATION ${NSYS_PLUGIN_INSTALL_DIR} +) +install( + FILES nsys-plugin.yaml + COMPONENT nsys_plugin + DESTINATION ${NSYS_PLUGIN_INSTALL_DIR} +) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp new file mode 100644 index 0000000000..8b767df48a --- /dev/null +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -0,0 +1,381 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// Nsight Systems plugin that samples per-interface network bandwidth and emits it as NVTX counter +// groups, so the bandwidth curve lands on the same timeline with the application activity. +// +// nsys spawns this executable as a collector for the duration of a profiling session, enabled with +// `--enable=kvikio_nic[,args]` and discovered via `NSYS_PLUGIN_SEARCH_DIRS`. It depends only on +// NVTX headers and libdl, so it is standalone and does not depend on libkvikio. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "nic_monitor.hpp" + +using kvikio::nsys_plugin::compute_rates; +using kvikio::nsys_plugin::default_interfaces; +using kvikio::nsys_plugin::NicCounterReader; +using kvikio::nsys_plugin::NicRates; + +namespace { + +/** + * @brief Tag type naming this plugin's NVTX domain. + * + * The domain is separate from libkvikio's own domain because the plugin runs its own process. + */ +struct kvikio_nic_domain { + static constexpr char const* name{"KvikIO NIC"}; +}; + +// Set from a signal handler for a clean shutdown. +volatile std::sig_atomic_t g_stop = 0; + +namespace constants { +// Exit codes +constexpr int exit_success = 0; +constexpr int exit_no_interfaces = 1; +constexpr int exit_usage_error = 2; + +constexpr nvtxSemanticsCounter_t rate_semantics{ + .header = {.structSize = sizeof(nvtxSemanticsCounter_t), + .semanticId = NVTX_SEMANTIC_ID_COUNTERS_V1, + .version = NVTX_COUNTER_SEMANTIC_VERSION, + .next = nullptr}, + .flags = NVTX_COUNTER_FLAGS_NONE, + .unit = "MiB/s", + .unitScaleNumerator = 1, + .unitScaleDenominator = 1, + .limitType = NVTX_COUNTER_LIMIT_UNDEFINED, +}; +} // namespace constants + +/** + * @brief Parsed command line configuration. + */ +struct Config { + std::chrono::microseconds interval{20000}; ///< Sampling interval. + std::optional device_filter; ///< If set, monitor interfaces matching this regex. +}; + +/** + * @brief Print usage to stderr and terminate the process. + * + * @param prog Program name (argv[0]). + * @param code Process exit code. + */ +[[noreturn]] void print_help_and_exit(char const* prog, int code) +{ + std::fprintf(stderr, + "Usage: %s [options]\n" + " -i | --interval Sampling interval in microseconds (default 20000)\n" + " -d | --device Interface name regex (default: up or unknown, non-loopback)\n" + " -h | --help Print this help message\n", + prog); + std::exit(code); +} + +/** + * @brief Parse the command line into a Config, exiting on `--help` or a malformed argument. + * + * Accepts arguments of these forms to be more consistent with nsys built-in sample: + * - `-i N` + * - `--interval N` + * - `--interval=N` + * - `-iN` + * + * Note that on the nsys command line the plugin arguments follow the plugin name as a comma + * separated list (commas only, no spaces). These are therefore equivalent: + * - `--enable=kvikio_nic,-d,eth0,-i,20000` + * - `--enable=kvikio_nic,--device=eth0,--interval=20000` + * + * @param argc Program argument count. + * @param argv Program argument vector. + * @return The parsed configuration. + */ +Config parse_args(int argc, char** argv) +{ + Config config; + for (int i = 1; i < argc; ++i) { + std::string_view const arg{argv[i]}; + std::string_view name; + std::optional inline_value; + if (arg.starts_with("--")) { + auto const eq = arg.find('='); + if (eq == std::string_view::npos) { + // Example: --interval 20000 + name = arg; // --interval + } else { + // Example: --interval=20000 + name = arg.substr(0, eq); // --interval + inline_value = arg.substr(eq + 1); // 20000 + } + } else if (arg.size() >= 2 && arg.front() == '-') { + // Example: -i 20000 or -i20000 + name = arg.substr(0, 2); // -i + if (arg.size() > 2) { + // Example: -i20000 + inline_value = arg.substr(2); // 20000 + } + } else { + std::fprintf(stderr, + "kvikio_nic: unexpected argument '%.*s'\n", + static_cast(arg.size()), + arg.data()); + print_help_and_exit(argv[0], constants::exit_usage_error); + } + + // Take this option's value from the inline form (`--interval=N`, `-iN`) or the next argument. + auto take_value = [&]() -> std::string { + if (inline_value.has_value()) { return std::string{inline_value.value()}; } + if (i + 1 < argc) { return std::string{argv[++i]}; } + std::fprintf(stderr, + "kvikio_nic: option '%.*s' requires a value\n", + static_cast(name.size()), + name.data()); + print_help_and_exit(argv[0], constants::exit_usage_error); + }; + + if (name == "-h" || name == "--help") { + print_help_and_exit(argv[0], constants::exit_success); + } else if (name == "-i" || name == "--interval") { + auto const value = take_value(); + long long parsed = 0; + auto const [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + auto const fully_parsed = (ec == std::errc{} && ptr == value.data() + value.size()); + if (!fully_parsed || parsed <= 0) { + std::fprintf(stderr, + "kvikio_nic: invalid interval '%s' (expected a positive integer)\n", + value.c_str()); + print_help_and_exit(argv[0], constants::exit_usage_error); + } + config.interval = std::chrono::microseconds{parsed}; + } else if (name == "-d" || name == "--device") { + auto const value = take_value(); + try { + config.device_filter = std::regex{value}; + } catch (std::regex_error const& e) { + std::fprintf( + stderr, "kvikio_nic: invalid device regex '%s' (%s)\n", value.c_str(), e.what()); + print_help_and_exit(argv[0], constants::exit_usage_error); + } + } else { + std::fprintf( + stderr, "kvikio_nic: unknown option '%.*s'\n", static_cast(name.size()), name.data()); + print_help_and_exit(argv[0], constants::exit_usage_error); + } + } + return config; +} + +/** + * @brief Choose the interfaces to monitor. + * + * @param config Parsed configuration. + * @return With no `--device` filter, all up or unknown, non-loopback interfaces. With a filter, all + * interfaces whose name matches the regex, bypassing the up check so an explicit request is + * honored. Sorted for a stable order. + */ +std::vector select_interfaces(Config const& config) +{ + if (!config.device_filter.has_value()) { return default_interfaces(); } + std::vector result; + std::error_code ec; + std::filesystem::directory_iterator it{ + std::filesystem::path{kvikio::nsys_plugin::constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto name = entry.path().filename().string(); + if (std::regex_match(name, config.device_filter.value())) { result.push_back(std::move(name)); } + } + std::sort(result.begin(), result.end()); + return result; +} + +/** + * @brief Register the {rx, tx} payload schema. + * + * @param domain NVTX domain handle. + * @return The schema id to pass as nvtxCounterAttr_t::schemaId. + */ +std::uint64_t register_rate_schema(nvtxDomainHandle_t domain) +{ + static_assert(std::is_standard_layout_v); + std::array const entries = { + nvtxPayloadSchemaEntry_t{.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, + .name = "rx", + .description = "Receive rate", + .offset = offsetof(NicRates, rx)}, + nvtxPayloadSchemaEntry_t{.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, + .name = "tx", + .description = "Transmit rate", + .offset = offsetof(NicRates, tx)}, + }; + nvtxPayloadSchemaAttr_t attr{}; + attr.fieldMask = NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NAME | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_TYPE | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_FLAGS | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_ENTRIES | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NUM_ENTRIES | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_STATIC_SIZE; + attr.name = "NIC bandwidth"; + attr.type = NVTX_PAYLOAD_SCHEMA_TYPE_STATIC; + attr.flags = NVTX_PAYLOAD_SCHEMA_FLAG_COUNTER_GROUP; + attr.entries = entries.data(); + attr.numEntries = entries.size(); + attr.payloadStaticSize = sizeof(NicRates); + return nvtxPayloadSchemaRegister(domain, &attr); +} + +/** + * @brief Register an NVTX counter group in the given domain. + * + * @param domain NVTX domain handle. + * @param name Counter group name (for example "eth0"). + * @param schema_id Payload schema id from register_rate_schema. + * @return The counter id to pass to nvtxCounterSample(). + */ +std::uint64_t register_counter(nvtxDomainHandle_t domain, + std::string const& name, + std::uint64_t schema_id) +{ + nvtxCounterAttr_t attr{}; + attr.structSize = sizeof(nvtxCounterAttr_t); + attr.schemaId = schema_id; + attr.name = name.c_str(); + attr.scopeId = NVTX_SCOPE_CURRENT_VM; + attr.semantics = &constants::rate_semantics.header; + attr.counterId = NVTX_COUNTER_ID_NONE; + return nvtxCounterRegister(domain, &attr); +} + +/** + * @brief Samples NIC byte counters at a fixed interval and emits them as NVTX counter groups. + * + * Construction registers one `` counter group per interface plus `total`. `run()` then + * differences the counters each tick until the stop flag is raised by the signal handler. + */ +class BandwidthCollector { + public: + /** + * @brief Register the NVTX schema and counter groups for the given interfaces. + * + * @param interfaces Interfaces to monitor. + * @param interval Sampling interval. + */ + BandwidthCollector(std::vector interfaces, std::chrono::microseconds interval) + : _interval{interval}, + _reader{std::move(interfaces)}, + _domain{nvtx3::domain::get()} + { + auto const schema_id = register_rate_schema(_domain); + _counter_ids.reserve(_reader.interfaces().size()); + for (auto const& name : _reader.interfaces()) { + _counter_ids.push_back(register_counter(_domain, name, schema_id)); + } + _total_counter_id = register_counter(_domain, "total", schema_id); + } + + /** + * @brief Run the sampling loop until @p stop becomes nonzero. + * + * @param stop Stop flag, set asynchronously by the signal handler. + */ + void run(volatile std::sig_atomic_t const& stop) const + { + auto const& interfaces = _reader.interfaces(); + std::fprintf(stderr, + "kvikio_nic: sampling %zu interface(s) every %lld us\n", + interfaces.size(), + static_cast(_interval.count())); + + using clock = std::chrono::steady_clock; + auto prev_counters = _reader.read(); + auto prev_time = clock::now(); + auto next_deadline = prev_time; + + while (stop == 0) { + // Use sleep_until (instead of sleep_for) to minimize sampling frequency drift. + // A signal does not interrupt the sleep. + next_deadline += _interval; + std::this_thread::sleep_until(next_deadline); + + auto const now = clock::now(); + auto cur_counters = _reader.read(); + auto const dt = std::chrono::duration{now - prev_time}.count(); + + NicRates total{0.0, 0.0}; + for (std::size_t i = 0; i < interfaces.size(); ++i) { + NicRates rates{0.0, 0.0}; + if (prev_counters[i].has_value() && cur_counters[i].has_value()) { + rates = compute_rates(prev_counters[i].value(), cur_counters[i].value(), dt); + } + total.rx += rates.rx; + total.tx += rates.tx; + nvtxCounterSample(_domain, _counter_ids[i], &rates, sizeof(rates)); + } + nvtxCounterSample(_domain, _total_counter_id, &total, sizeof(total)); + + prev_counters = std::move(cur_counters); + prev_time = now; + } + } + + private: + std::chrono::microseconds _interval; + NicCounterReader _reader; + nvtxDomainHandle_t _domain; + std::vector _counter_ids; + std::uint64_t _total_counter_id{0}; +}; + +} // namespace + +// C language linkage (extern) to be standard conformant. +// Also internal linkage (static) as a good practice. +// nsys stops the collector by sending SIGTERM (then SIGKILL after a grace period). Catch it so the +// loop breaks and the process exits cleanly with code 0 instead of an abnormal termination. Also +// catch SIGINT to have the same clean exit for Ctrl-C. +extern "C" { +static void kvikio_nic_handle_signal(int /*signum*/) { g_stop = 1; } +} + +int main(int argc, char** argv) +{ + auto const config = parse_args(argc, argv); + auto interfaces = select_interfaces(config); + if (interfaces.empty()) { + std::fprintf(stderr, "kvikio_nic: no matching network interfaces to monitor.\n"); + return constants::exit_no_interfaces; + } + + std::signal(SIGTERM, kvikio_nic_handle_signal); + std::signal(SIGINT, kvikio_nic_handle_signal); + + BandwidthCollector const collector{std::move(interfaces), config.interval}; + collector.run(g_stop); + return constants::exit_success; +} diff --git a/cpp/nsys_plugins/nic/nic_monitor.cpp b/cpp/nsys_plugins/nic/nic_monitor.cpp new file mode 100644 index 0000000000..9a0507d727 --- /dev/null +++ b/cpp/nsys_plugins/nic/nic_monitor.cpp @@ -0,0 +1,157 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "nic_monitor.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace kvikio::nsys_plugin { + +namespace { + +/** + * @brief Read a sysfs pseudo-file in a single read(). + * + * @param path Path of the pseudo-file to read. + * @param buf Scratch buffer that backs the returned view. At most `buf.size()` bytes are read, so + * it never overflows. + * @return A view of the bytes read (into @p buf), or std::nullopt if the file cannot be read. The + * string view is not necessarily null terminated. + */ +[[nodiscard]] std::optional read_pseudo_file(char const* path, + std::span buf) noexcept +{ + auto const fd = ::open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { return std::nullopt; } + auto const n = ::read(fd, buf.data(), buf.size()); + ::close(fd); + if (n <= 0) { return std::nullopt; } + return std::string_view{buf.data(), static_cast(n)}; +} + +/** + * @brief Parse a single unsigned integer from a sysfs pseudo-file. + * + * @param path Path of the pseudo-file to read. + * @return The parsed value, or std::nullopt if the file is unparseable. + */ +[[nodiscard]] std::optional read_u64_file(std::string const& path) noexcept +{ + // A uint64 counter is at most 20 digits, so 32 bytes is ample (20 digits + newline) + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return std::nullopt; } + std::uint64_t value{}; + // from_chars is locale-independent, non-allocating, and non-throwing, and stops at the trailing + // newline. + [[maybe_unused]] auto const [_, ec] = + std::from_chars(content->data(), content->data() + content->size(), value); + // Note: ec type is std::errc, which is an enum class and has no bool operator. In C++26, the + // return value of type `std::from_chars_result` will have bool operator. + // TODO: In C++26, remove structured binding and use the result's bool operator to check success + // directly. + if (ec != std::errc{}) { return std::nullopt; } + return value; +} + +} // namespace + +bool iface_is_up(std::string const& name) +{ + auto const path = std::string{constants::sysfs_net} + "/" + name + "/operstate"; + // The longest operstate value is "lowerlayerdown" (14 chars + newline), so 16 bytes is ample. + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return false; } + std::string_view state = content.value(); + // Remove any trailing newline or space character + while (!state.empty() && (state.back() == '\n' || state.back() == ' ')) { + state.remove_suffix(1); + } + // Some working NICs report "unknown" (virtual NICs), so this state should be accepted. The other + // states (down, notpresent, lowerlayerdown, testing, dormant) should be excluded. + return state == "up" || state == "unknown"; +} + +std::vector default_interfaces() +{ + std::vector result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto name = entry.path().filename().string(); + // Ignore loopback interface + if (name == "lo") { continue; } + if (!iface_is_up(name)) { continue; } + result.push_back(std::move(name)); + } + std::sort(result.begin(), result.end()); + return result; +} + +NicCounterReader::NicCounterReader(std::vector interfaces) + : _interfaces{std::move(interfaces)} +{ + _rx_paths.reserve(_interfaces.size()); + _tx_paths.reserve(_interfaces.size()); + for (auto const& name : _interfaces) { + auto const stats = std::string{constants::sysfs_net} + "/" + name + "/statistics/"; + _rx_paths.push_back(stats + "rx_bytes"); + _tx_paths.push_back(stats + "tx_bytes"); + } +} + +std::vector> NicCounterReader::read() const +{ + std::vector> result; + result.reserve(_interfaces.size()); + for (std::size_t i = 0; i < _interfaces.size(); ++i) { + auto const rx = read_u64_file(_rx_paths[i]); + auto const tx = read_u64_file(_tx_paths[i]); + if (rx.has_value() && tx.has_value()) { + result.emplace_back(NicCounters{rx.value(), tx.value()}); + } else { + // Report an unreadable interface (for example one that was removed or never existed) as + // missing. + result.emplace_back(std::nullopt); + } + } + return result; +} + +std::vector const& NicCounterReader::interfaces() const noexcept +{ + return _interfaces; +} + +NicRates compute_rates(NicCounters const& prev, NicCounters const& cur, double dt_seconds) noexcept +{ + NicRates rates{0.0, 0.0}; + if (dt_seconds <= 0.0) { return rates; } + if (cur.rx_bytes >= prev.rx_bytes) { + auto const delta = cur.rx_bytes - prev.rx_bytes; + rates.rx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; + } + if (cur.tx_bytes >= prev.tx_bytes) { + auto const delta = cur.tx_bytes - prev.tx_bytes; + rates.tx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; + } + return rates; +} + +} // namespace kvikio::nsys_plugin diff --git a/cpp/nsys_plugins/nic/nic_monitor.hpp b/cpp/nsys_plugins/nic/nic_monitor.hpp new file mode 100644 index 0000000000..e23ce0cd9d --- /dev/null +++ b/cpp/nsys_plugins/nic/nic_monitor.hpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include +#include + +// Internal support code for the KvikIO Nsight Systems NIC plugin. Reads network byte counters from +// sysfs and converts them to rates. Compiled directly into the plugin and its test. Not part of +// libkvikio binary or public header. + +namespace kvikio::nsys_plugin { + +namespace constants { +inline constexpr double bytes_per_mib = 1024.0 * 1024.0; +inline constexpr char const* sysfs_net = "/sys/class/net"; +} // namespace constants + +/** + * @brief Cumulative byte counters for a single network interface. + * + * The values are the kernel's running totals since boot, as exposed by + * `/sys/class/net//statistics/{rx,tx}_bytes`. + */ +struct NicCounters { + std::uint64_t rx_bytes; ///< Total bytes received since boot. + std::uint64_t tx_bytes; ///< Total bytes transmitted since boot. +}; + +/** + * @brief One bandwidth sample for a single interface: the receive and transmit rates. + * + * The field names are deliberately unit-neutral. `compute_rates` produces the values in MiB/s. + */ +struct NicRates { + double rx; ///< Receive rate. + double tx; ///< Transmit rate. +}; + +/** + * @brief Check whether a network interface is up. + * + * @param name Interface name. + * @return True only when operstate is "up" or "unknown". False for any other state or if it cannot + * be read. + */ +[[nodiscard]] bool iface_is_up(std::string const& name); + +/** + * @brief Enumerate the default set of interfaces to monitor. + * + * @return All "up" non-loopback interfaces under `/sys/class/net`, sorted for a stable order. + */ +[[nodiscard]] std::vector default_interfaces(); + +/** + * @brief Reads the byte counters of a fixed set of network interfaces. + * + * The two sysfs paths (rx and tx) per interface are precomputed once at construction. + */ +class NicCounterReader { + public: + /** + * @brief Construct a reader over the given interfaces. + * + * The names are taken as-is, without filtering or validation, so loopback ("lo") is allowed. + * + * @param interfaces Interface names to read, in the order `read()` reports them. + */ + explicit NicCounterReader(std::vector interfaces); + + /** + * @brief Read the current cumulative byte counters of every configured interface. + * + * @return One entry per interface, in constructor order. An entry is std::nullopt when either of + * its counters is unreadable (for example the interface was removed or never existed). + */ + [[nodiscard]] std::vector> read() const; + + /** + * @brief The interfaces this reader was constructed with. + */ + [[nodiscard]] std::vector const& interfaces() const noexcept; + + private: + std::vector _interfaces; + // Precomputed `/sys/class/net//statistics/{rx,tx}_bytes` paths, parallel to `_interfaces`. + std::vector _rx_paths; + std::vector _tx_paths; +}; + +/** + * @brief Convert a pair of cumulative counter samples into receive and transmit rates in MiB/s. + * + * @param prev The earlier counter sample. + * @param cur The later counter sample. + * @param dt_seconds Elapsed wall-clock seconds between the two samples. + * @return Rates in MiB/s. A direction whose counter did not advance yields zero for that direction + * (this guards the first sample as well as a counter wrap or reset). Both rates are zero when @p + * dt_seconds is not positive. + */ +[[nodiscard]] NicRates compute_rates(NicCounters const& prev, + NicCounters const& cur, + double dt_seconds) noexcept; + +} // namespace kvikio::nsys_plugin diff --git a/cpp/nsys_plugins/nic/nsys-plugin.yaml b/cpp/nsys_plugins/nic/nsys-plugin.yaml new file mode 100644 index 0000000000..ebfcc38718 --- /dev/null +++ b/cpp/nsys_plugins/nic/nsys-plugin.yaml @@ -0,0 +1,24 @@ +PluginName: kvikio_nic +FeatureName: Collect KvikIO NIC bandwidth +ExecutablePath: kvikio_nic_nsys_plugin +Description: Per-interface RX/TX network bandwidth from /sys/class/net, as NVTX counter groups. +ExtendedDescription: | + Command line options: + -i | --interval Sampling interval in microseconds (default 20000) + -d | --device Interface name regex (default: up or unknown, non-loopback) + -h | --help Print this help message + + Plugin arguments follow the plugin name as a comma separated list (commas only, no + spaces). Each comma separated token is passed verbatim as one argument. These are + equivalent: + --enable=kvikio_nic,-d,eth0,-i,20000 + --enable=kvikio_nic,-deth0,-i20000 + --enable=kvikio_nic,--device=eth0,--interval=20000 + + With no --device filter, all up (or unknown) non-loopback interfaces are monitored. + A --device regex selects interfaces by name and bypasses the up check, so an explicit + request is always honored. +ExtendedDescriptionForGui: | + Options: + -i | --interval Sampling interval in microseconds (default 20000) + -d | --device Interface name regex (default: up or unknown, non-loopback) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 4e4c7e6efc..d1c7685008 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -62,6 +62,73 @@ function(kvikio_add_test) ) endfunction() +#[=======================================================================[.rst: +kvikio_add_nsys_plugin_test +--------------------------- + +Create a test for the support code of a KvikIO Nsight Systems plugin. + +Plugin support code (under ``cpp/nsys_plugins/``) deliberately does not depend on +libkvikio or CUDA, so this test links only GoogleTest and compiles the plugin sources directly, +rather than going through ``kvikio_add_test``. + +.. code-block:: cmake + + kvikio_add_nsys_plugin_test(NAME SOURCES PLUGIN_SOURCES ) + + ``NAME`` + Test name. Single-value argument. + + ``SOURCES`` + List of test source files, relative to ``cpp/tests``. Multi-value argument. + + ``PLUGIN_SOURCES`` + List of plugin source files, relative to ``cpp/nsys_plugins`` (for example + ``nic/nic_monitor.cpp``). The parent directory of each file is added as an include + directory. Multi-value argument. +#]=======================================================================] +function(kvikio_add_nsys_plugin_test) + cmake_parse_arguments( + _KVIKIO # prefix + "" # optional + "NAME" # single value + "SOURCES;PLUGIN_SOURCES" # multi-value + ${ARGN} + ) + + if(DEFINED _KVIKIO_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unknown argument: ${_KVIKIO_UNPARSED_ARGUMENTS}") + endif() + + set(_plugin_root "${KvikIO_SOURCE_DIR}/nsys_plugins") + set(_plugin_sources "") + set(_plugin_include_dirs "") + foreach(_src IN LISTS _KVIKIO_PLUGIN_SOURCES) + list(APPEND _plugin_sources "${_plugin_root}/${_src}") + cmake_path(GET _src PARENT_PATH _src_dir) + list(APPEND _plugin_include_dirs "$") + endforeach() + list(REMOVE_DUPLICATES _plugin_include_dirs) + + add_executable(${_KVIKIO_NAME} ${_KVIKIO_SOURCES} ${_plugin_sources}) + set_target_properties( + ${_KVIKIO_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + ) + target_include_directories(${_KVIKIO_NAME} PRIVATE ${_plugin_include_dirs}) + target_link_libraries( + ${_KVIKIO_NAME} PRIVATE GTest::gmock GTest::gmock_main GTest::gtest GTest::gtest_main + ) + + rapids_test_add( + NAME ${_KVIKIO_NAME} + COMMAND ${_KVIKIO_NAME} + INSTALL_COMPONENT_SET testing + ) +endfunction() + kvikio_add_test(NAME BASIC_IO_TEST SOURCES test_basic_io.cpp utils/env.cpp) kvikio_add_test(NAME BOUNCE_BUFFER_TEST SOURCES test_bounce_buffer.cpp) @@ -82,6 +149,10 @@ kvikio_add_test(NAME MMAP_TEST SOURCES test_mmap.cpp) kvikio_add_test(NAME CONCURRENT_REQUEST_LIMITER_TEST SOURCES test_concurrent_request_limiter.cpp) +kvikio_add_nsys_plugin_test( + NAME NIC_MONITOR_TEST SOURCES test_nic_monitor.cpp PLUGIN_SOURCES nic/nic_monitor.cpp +) + if(KvikIO_REMOTE_SUPPORT) kvikio_add_test(NAME REMOTE_HANDLE_TEST SOURCES test_remote_handle.cpp utils/env.cpp) kvikio_add_test(NAME HDFS_TEST SOURCES test_hdfs.cpp utils/hdfs_helper.cpp) diff --git a/cpp/tests/test_nic_monitor.cpp b/cpp/tests/test_nic_monitor.cpp new file mode 100644 index 0000000000..d298a88260 --- /dev/null +++ b/cpp/tests/test_nic_monitor.cpp @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include + +#include "nic_monitor.hpp" + +using kvikio::nsys_plugin::compute_rates; +using kvikio::nsys_plugin::default_interfaces; +using kvikio::nsys_plugin::iface_is_up; +using kvikio::nsys_plugin::NicCounterReader; +using kvikio::nsys_plugin::NicCounters; +using kvikio::nsys_plugin::constants::bytes_per_mib; + +TEST(NicMonitor, ReaderReadsLoopback) +{ + NicCounterReader const reader{{"lo"}}; + auto const counters = reader.read(); + ASSERT_EQ(counters.size(), 1U); + // The loopback interface is present on essentially every Linux host. + EXPECT_TRUE(counters[0].has_value()); +} + +TEST(NicMonitor, ReaderReportsMissingInterfaceWithoutValue) +{ + NicCounterReader const reader{{"nonexistent_iface"}}; + auto const counters = reader.read(); + ASSERT_EQ(counters.size(), 1U); + EXPECT_FALSE(counters[0].has_value()); +} + +TEST(NicMonitor, IfaceIsUpAcceptsLoopbackAndRejectsMissing) +{ + // Loopback reports operstate "unknown", which the up/unknown policy accepts. + EXPECT_TRUE(iface_is_up("lo")); + // A name that cannot be read must be treated as not up rather than accepted. + EXPECT_FALSE(iface_is_up("nonexistent_iface")); +} + +TEST(NicMonitor, DefaultInterfacesExcludeLoopback) +{ + for (auto const& name : default_interfaces()) { + EXPECT_NE(name, "lo"); + } +} + +TEST(NicMonitor, ComputeRatesConvertsDeltasToMiBps) +{ + NicCounters const prev{0, 0}; + NicCounters const cur{static_cast(bytes_per_mib) * 2, + static_cast(bytes_per_mib) * 6}; + auto const rates = compute_rates(prev, cur, 2.0); + // 2 MiB received over 2 s -> 1 MiB/s + // 6 MiB transmitted over 2 s -> 3 MiB/s. + EXPECT_DOUBLE_EQ(rates.rx, 1.0); + EXPECT_DOUBLE_EQ(rates.tx, 3.0); +} + +TEST(NicMonitor, ComputeRatesGuardsWrapAndNonPositiveInterval) +{ + NicCounters const high{1000, 1000}; + NicCounters const low{10, 10}; + // A counter that appears to go backwards (wrap or reset) yields zero. + auto const wrapped = compute_rates(high, low, 1.0); + EXPECT_DOUBLE_EQ(wrapped.rx, 0.0); + EXPECT_DOUBLE_EQ(wrapped.tx, 0.0); + // A non-positive elapsed time yields zero. + auto const no_time = compute_rates(low, high, 0.0); + EXPECT_DOUBLE_EQ(no_time.rx, 0.0); + EXPECT_DOUBLE_EQ(no_time.tx, 0.0); +} diff --git a/docs/source/api.rst b/docs/source/api.rst index 71897a0769..433e35dbf5 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -79,3 +79,9 @@ Defaults .. autofunction:: set .. autofunction:: get + +Nsight Systems plugin +--------------------- +.. currentmodule:: kvikio.nsys + +.. autofunction:: nsys_plugin_search_dir diff --git a/docs/source/index.rst b/docs/source/index.rst index 43cc3c9913..acf09a9804 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,5 +25,6 @@ Contents zarr remote_file runtime_settings + profiling api genindex diff --git a/docs/source/profiling.rst b/docs/source/profiling.rst new file mode 100644 index 0000000000..318cd55b3b --- /dev/null +++ b/docs/source/profiling.rst @@ -0,0 +1,43 @@ +Profiling +========= + +NIC bandwidth on the Nsight Systems timeline +-------------------------------------------- +KvikIO ships a standalone `Nsight Systems `_ (nsys) plugin named ``kvikio_nic`` that samples per-interface network bandwidth and places it on the nsys timeline. + +The plugin reads the Linux kernel byte counters (``/sys/class/net//statistics/{rx,tx}_bytes``) at a fixed interval and emits one NVTX counter group per monitored interface plus a ``total`` group, all under the ``KvikIO NIC`` NVTX domain. Each group carries the receive (``rx``) and transmit (``tx``) rates. + +Installation +------------ +The plugin ships by default on both packaging channels, inside the ``libkvikio`` conda package and inside the ``libkvikio`` wheel. It consists of an executable (``kvikio_nic_nsys_plugin``) and a manifest (``nsys-plugin.yaml``) placed together in a ``kvikio_nic`` directory. Use :py:func:`kvikio.nsys_plugin_search_dir` to locate that directory on either channel. + +Command line options +-------------------- + * ``-i | --interval``: Sampling interval in microseconds (default 20000, i.e. 50 Hz). + * ``-d | --device``: Interface name regex. If not given, all up (or unknown) non-loopback interfaces are monitored. An explicit regex selects interfaces by name and bypasses the up check, so an explicit request is always honored. + +On the nsys command line, plugin arguments follow the plugin name as a comma separated list. These are equivalent: + +.. code-block:: bash + + nsys profile --enable=kvikio_nic,-d,eth0,-i,20000 ... + nsys profile --enable=kvikio_nic,-deth0,-i20000 ... + nsys profile --enable=kvikio_nic,--device=eth0,--interval=20000 ... + +Usage with nsys 2026.2.1 or newer +--------------------------------- +Nsight Systems 2026.2.1 introduced ``NSYS_PLUGIN_SEARCH_DIRS`` for discovering third party plugins: + +.. code-block:: bash + + export NSYS_PLUGIN_SEARCH_DIRS="$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')" + nsys profile --enable=help # should list kvikio_nic + nsys profile --enable=kvikio_nic,--device=eth0 --trace=cuda,nvtx,osrt -o nsys_report my_application + +Usage with older nsys +--------------------- +Older nsys versions only discover plugins inside the nsys installation itself. Either copy or symlink the ``kvikio_nic`` directory next to the bundled plugins (write access to the installation is needed, typically root): + +.. code-block:: bash + + sudo ln -s "$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')/kvikio_nic" /opt/nvidia/nsight-systems//target-linux-x64/plugins/ diff --git a/python/kvikio/kvikio/__init__.py b/python/kvikio/kvikio/__init__.py index ee9d43d197..6c663d4576 100644 --- a/python/kvikio/kvikio/__init__.py +++ b/python/kvikio/kvikio/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # If libkvikio was installed as a wheel, we must request it to load the library symbols. @@ -23,6 +23,7 @@ get_page_cache_info, ) from kvikio.mmap import Mmap +from kvikio.nsys import nsys_plugin_search_dir from kvikio.remote_file import ( RemoteEndpointType, RemoteFile, @@ -44,6 +45,7 @@ "infer_remote_endpoint_type", "is_remote_file_available", "kvikio_deprecation_notice", + "nsys_plugin_search_dir", "RemoteEndpointType", "RemoteFile", "stream_register", diff --git a/python/kvikio/kvikio/nsys.py b/python/kvikio/kvikio/nsys.py new file mode 100644 index 0000000000..ec14add748 --- /dev/null +++ b/python/kvikio/kvikio/nsys.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + + +def nsys_plugin_search_dir() -> str: + """Return the directory that holds KvikIO's bundled Nsight Systems plugins. + + Works for both installation channels, which lay out the plugin differently. + + - pip (inside the ``libkvikio`` wheel, a Python package): + + .. code-block:: text + + site-packages/libkvikio/ + |-- __init__.py, load.py, nsys.py + |-- lib/libkvikio.so + |-- share/kvikio/nsys-plugins/ --> returned + |-- kvikio_nic/{kvikio_nic_nsys_plugin, nsys-plugin.yaml} + + - conda (the ``libkvikio`` conda package, C++ files only, no Python module): + + .. code-block:: text + + $CONDA_PREFIX/ + |-- lib/libkvikio.so + |-- include/kvikio/ + |-- share/kvikio/nsys-plugins/ --> returned + |-- kvikio_nic/{kvikio_nic_nsys_plugin, nsys-plugin.yaml} + + See :doc:`profiling ` for how to enable the plugin in Nsight Systems. + + Returns + ------- + str + The plugin search directory. The path is returned even if it does not exist, + for example when the plugin was not built. + """ + try: + import libkvikio + + # pip: the plugin ships inside the libkvikio wheel. + return libkvikio.nsys_plugin_search_dir() + except ModuleNotFoundError: + # conda: the libkvikio conda package installs the plugin into the + # environment prefix, which is sys.prefix in a conda environment. + return os.path.join(sys.prefix, "share", "kvikio", "nsys-plugins") diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 7a7de6e790..cb66c48212 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -32,5 +32,6 @@ unset(kvikio_FOUND) set(KvikIO_BUILD_BENCHMARKS OFF) set(KvikIO_BUILD_EXAMPLES OFF) set(KvikIO_BUILD_TESTS OFF) +set(KvikIO_BUILD_NSYS_PLUGIN ON) add_subdirectory(../../cpp kvikio-cpp) diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py index 8051956848..74db43a7c9 100644 --- a/python/libkvikio/libkvikio/__init__.py +++ b/python/libkvikio/libkvikio/__init__.py @@ -1,7 +1,13 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libkvikio._version import __git_commit__, __version__ from libkvikio.load import load_library +from libkvikio.nsys import nsys_plugin_search_dir -__all__ = ["__git_commit__", "__version__", "load_library"] +__all__ = [ + "__git_commit__", + "__version__", + "load_library", + "nsys_plugin_search_dir", +] diff --git a/python/libkvikio/libkvikio/nsys.py b/python/libkvikio/libkvikio/nsys.py new file mode 100644 index 0000000000..3cdf6d1bbb --- /dev/null +++ b/python/libkvikio/libkvikio/nsys.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os + + +def nsys_plugin_search_dir() -> str: + """Return the directory that holds KvikIO's bundled Nsight Systems plugins. Valid + only for pip installation. + + Returns + ------- + str + The ``share/kvikio/nsys-plugins`` directory inside the installed ``libkvikio`` + package. + """ + return os.path.join(os.path.dirname(__file__), "share", "kvikio", "nsys-plugins")