From e718cb0403a6d25e025b9d3a3c73fa504da8c579 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 2 Jul 2026 09:26:06 -0400 Subject: [PATCH 01/16] Develop a network monitoring tool --- cpp/CMakeLists.txt | 1 + .../kvikio/experimental/nic_monitor.hpp | 107 ++++ cpp/src/experimental/nic_monitor.cpp | 279 ++++++++++ cpp/tests/CMakeLists.txt | 2 + cpp/tests/test_nic_monitor.cpp | 55 ++ python/kvikio/kvikio/_lib/CMakeLists.txt | 2 +- python/kvikio/kvikio/_lib/nic_monitor.pyx | 83 +++ python/kvikio/kvikio/tools/__init__.py | 9 + .../kvikio/kvikio/tools/remote_io_monitor.py | 517 ++++++++++++++++++ python/kvikio/tests/test_remote_io_monitor.py | 138 +++++ 10 files changed, 1192 insertions(+), 1 deletion(-) create mode 100644 cpp/include/kvikio/experimental/nic_monitor.hpp create mode 100644 cpp/src/experimental/nic_monitor.cpp create mode 100644 cpp/tests/test_nic_monitor.cpp create mode 100644 python/kvikio/kvikio/_lib/nic_monitor.pyx create mode 100644 python/kvikio/kvikio/tools/__init__.py create mode 100644 python/kvikio/kvikio/tools/remote_io_monitor.py create mode 100644 python/kvikio/tests/test_remote_io_monitor.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dfc4860592..08384e1245 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -168,6 +168,7 @@ set(SOURCES "src/file_utils.cpp" "src/logger.cpp" "src/mmap.cpp" + "src/experimental/nic_monitor.cpp" "src/detail/bounce_buffer_cache.cpp" "src/detail/concurrent_request_limiter.cpp" "src/detail/env.cpp" diff --git a/cpp/include/kvikio/experimental/nic_monitor.hpp b/cpp/include/kvikio/experimental/nic_monitor.hpp new file mode 100644 index 0000000000..9d7cce2cbe --- /dev/null +++ b/cpp/include/kvikio/experimental/nic_monitor.hpp @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace kvikio::experimental { + +/** + * @brief Cumulative byte counters for a single network interface. + * + * The values are the kernel's running totals since boot, as exposed by + * `/sys/class/net//statistics/{rx,tx}_bytes`. + */ +struct NicCounters { + std::uint64_t rx_bytes; ///< Total bytes received since boot. + std::uint64_t tx_bytes; ///< Total bytes transmitted since boot. +}; + +/** + * @brief Read the current cumulative byte counters for every network interface. + * + * Reads `/sys/class/net//statistics/{rx,tx}_bytes`. Interfaces whose counters cannot be read + * are skipped rather than reported as an error. + * + * @return A map from interface name to its cumulative counters. Empty if `/sys/class/net` is + * unavailable. + */ +std::map read_nic_counters(); + +/** + * @brief Samples NIC bandwidth and emits it as NVTX counter groups. + * + * On `start()`, a background `std::jthread` differences the kernel byte counters at a fixed + * frequency and emits one NVTX counter group per monitored interface (named `nic_MiBps.`) + * plus a summed `nic_MiBps.total`, in the `libkvikio` NVTX domain. Each group carries the receive + * and transmit rates (`rx_MiBps`, `tx_MiBps`) sampled together, so they render as a single grouped + * counter track on the Nsight Systems timeline. + */ +class NicBandwidthMonitor { + public: + /** + * @brief Construct a monitor. + * + * @param freq_hz Sampling frequency in hertz. Must be positive. + * @param interfaces Interfaces to monitor. If empty, all UP non-loopback interfaces are selected + * once at `start()`. + * + * @exception std::invalid_argument if @p freq_hz is not positive. + */ + explicit NicBandwidthMonitor(double freq_hz = 20.0, std::vector interfaces = {}); + + /** + * @brief Stop the sampling thread if it is running. + */ + ~NicBandwidthMonitor(); + + // Non-copyable + NicBandwidthMonitor(NicBandwidthMonitor const&) = delete; + NicBandwidthMonitor& operator=(NicBandwidthMonitor const&) = delete; + + /** + * @brief Select interfaces (if not given), register NVTX counters, and launch the sampling + * thread. Has no effect if already running. + */ + void start(); + + /** + * @brief Signal the sampling thread to stop and join it. Has no effect if not running. + */ + void stop(); + + /** + * @brief Whether the sampling thread is currently running. + */ + [[nodiscard]] bool running() const noexcept; + + /** + * @brief The interfaces being monitored (populated at `start()`). + */ + [[nodiscard]] std::vector const& interfaces() const noexcept; + + private: + void run(std::stop_token stop_token); + + double _freq_hz; + std::vector _interfaces; + + // NVTX counter ids, parallel to `_interfaces`, plus the total counter id. + std::vector _counter_ids; + std::uint64_t _total_counter_id{0}; + + // Declared after the members used by `run()`, so the jthread's auto-join on destruction happens + // before those members are destroyed. + std::jthread _thread; + + bool _running{false}; +}; + +} // namespace kvikio::experimental diff --git a/cpp/src/experimental/nic_monitor.cpp b/cpp/src/experimental/nic_monitor.cpp new file mode 100644 index 0000000000..a12c330d4e --- /dev/null +++ b/cpp/src/experimental/nic_monitor.cpp @@ -0,0 +1,279 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include + +namespace kvikio::experimental { + +namespace { + +namespace constants { +constexpr double bytes_per_mib = 1024.0 * 1024.0; +constexpr char const* sysfs_net = "/sys/class/net"; +} // namespace constants + +/** + * @brief Read a sysfs pseudo-file in a single read(). + * + * @param path Path of the pseudo-file to read. + * @param buf Scratch buffer that backs the returned view. At most `buf.size()` bytes are read, so + * it never overflows. + * @return A view of the bytes read (into @p buf), or std::nullopt if the file cannot be read. The + * string view is not necessarily null terminated. + */ +std::optional read_pseudo_file(char const* path, std::span buf) noexcept +{ + auto const fd = ::open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { return std::nullopt; } + auto const n = ::read(fd, buf.data(), buf.size()); + ::close(fd); + if (n <= 0) { return std::nullopt; } + return std::string_view{buf.data(), static_cast(n)}; +} + +/** + * @brief Parse a single unsigned integer from a sysfs pseudo-file. + * + * @param path Path of the pseudo-file to read. + * @return The parsed value, or std::nullopt if the file is unparseable. + */ +std::optional read_u64_file(std::string const& path) noexcept +{ + // A uint64 counter is at most 20 digits, so 32 bytes is ample (20 digits + newline) + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return std::nullopt; } + std::uint64_t value{}; + // from_chars is locale-independent, non-allocating, and non-throwing, and stops at the trailing + // newline. + [[maybe_unused]] auto const [_, ec] = + std::from_chars(content->data(), content->data() + content->size(), value); + if (ec != std::errc{}) { return std::nullopt; } + return value; +} + +/** + * @brief Check whether a network interface is up. + * + * @param name Interface name. + * @return True only when operstate is "up" or "unknown". False for any other state or if it cannot + * be read. + */ +bool iface_is_up(std::string const& name) +{ + auto const path = std::string{constants::sysfs_net} + "/" + name + "/operstate"; + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return false; } + std::string_view state = content.value(); + while (!state.empty() && (state.back() == '\n' || state.back() == ' ')) { + state.remove_suffix(1); + } + // Some working NICs report "unknown" (loopback or virtual NICs), so this state should be + // accepted. The other states (down, notpresent, lowerlayerdown, testing, dormant) should be + // excluded. + return state == "up" || state == "unknown"; +} + +/** + * @brief Enumerate the default set of interfaces to monitor. + * + * @return All "up" non-loopback interfaces under `/sys/class/net`, sorted for a stable order. + */ +std::vector default_interfaces() +{ + std::vector result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto name = entry.path().filename().string(); + if (name == "lo") { continue; } + if (!iface_is_up(name)) { continue; } + result.push_back(std::move(name)); + } + std::sort(result.begin(), result.end()); + return result; +} + +// One sample of a counter group: receive and transmit rates for one interface. +struct NicRates { + double rx_mibps; + double tx_mibps; +}; + +/** + * @brief Register the {rx_MiBps, tx_MiBps} payload schema for the counter groups. + * + * @param domain NVTX domain handle. + * @return The schema id to pass as nvtxCounterAttr_t::schemaId. + */ +std::uint64_t register_rate_schema(nvtxDomainHandle_t domain) +{ + nvtxPayloadSchemaEntry_t const entries[] = { + {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "rx_MiBps"}, + {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "tx_MiBps"}, + }; + nvtxPayloadSchemaAttr_t attr{}; + attr.fieldMask = NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_TYPE | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_ENTRIES | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NUM_ENTRIES | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_STATIC_SIZE; + attr.type = NVTX_PAYLOAD_SCHEMA_TYPE_STATIC; + attr.entries = entries; + attr.numEntries = 2; + attr.payloadStaticSize = sizeof(NicRates); + return nvtxPayloadSchemaRegister(domain, &attr); +} + +/** + * @brief Register an NVTX counter group in the given domain using a payload schema. + * + * @param domain NVTX domain handle. + * @param name Counter name (for example "nic_MiBps.eth0"). + * @param schema_id Payload schema id from register_rate_schema. + * @return The counter id to pass to nvtxCounterSample(). + */ +std::uint64_t register_counter(nvtxDomainHandle_t domain, + std::string const& name, + std::uint64_t schema_id) +{ + nvtxCounterAttr_t attr{}; + attr.structSize = sizeof(nvtxCounterAttr_t); + attr.schemaId = schema_id; + attr.name = name.c_str(); + attr.scopeId = NVTX_SCOPE_NONE; + attr.counterId = NVTX_COUNTER_ID_NONE; + return nvtxCounterRegister(domain, &attr); +} + +} // namespace + +std::map read_nic_counters() +{ + std::map result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto const name = entry.path().filename().string(); + auto const stats = entry.path() / "statistics"; + auto const rx = read_u64_file((stats / "rx_bytes").string()); + auto const tx = read_u64_file((stats / "tx_bytes").string()); + // If an interface exposes no parseable byte counters, skip it. + if (!rx.has_value() || !tx.has_value()) { continue; } + result.emplace(name, NicCounters{rx.value(), tx.value()}); + } + return result; +} + +NicBandwidthMonitor::NicBandwidthMonitor(double freq_hz, std::vector interfaces) + : _freq_hz{freq_hz}, _interfaces{std::move(interfaces)} +{ + KVIKIO_EXPECT(freq_hz > 0.0, "freq_hz must be positive", std::invalid_argument); +} + +NicBandwidthMonitor::~NicBandwidthMonitor() { stop(); } + +void NicBandwidthMonitor::start() +{ + if (_running) { return; } + if (_interfaces.empty()) { _interfaces = default_interfaces(); } + + nvtxDomainHandle_t domain = nvtx3::domain::get(); + auto const schema_id = register_rate_schema(domain); + _counter_ids.clear(); + _counter_ids.reserve(_interfaces.size()); + for (auto const& name : _interfaces) { + _counter_ids.push_back(register_counter(domain, "nic_MiBps." + name, schema_id)); + } + _total_counter_id = register_counter(domain, "nic_MiBps.total", schema_id); + + _running = true; + _thread = std::jthread{[this](std::stop_token stop_token) { run(stop_token); }}; +} + +void NicBandwidthMonitor::stop() +{ + if (!_running) { return; } + _thread.request_stop(); + if (_thread.joinable()) { _thread.join(); } + _running = false; +} + +bool NicBandwidthMonitor::running() const noexcept { return _running; } + +std::vector const& NicBandwidthMonitor::interfaces() const noexcept +{ + return _interfaces; +} + +void NicBandwidthMonitor::run(std::stop_token stop_token) +{ + nvtxDomainHandle_t domain = nvtx3::domain::get(); + using clock = std::chrono::steady_clock; + auto const interval = std::chrono::duration{1.0 / _freq_hz}; + + auto prev = read_nic_counters(); + auto prev_t = clock::now(); + + while (!stop_token.stop_requested()) { + std::this_thread::sleep_for(std::chrono::duration_cast(interval)); + + auto const now = clock::now(); + auto const cur = read_nic_counters(); + double const dt = std::chrono::duration{now - prev_t}.count(); + + NicRates total{0.0, 0.0}; + for (std::size_t i = 0; i < _interfaces.size(); ++i) { + NicRates rates{0.0, 0.0}; + auto const itc = cur.find(_interfaces[i]); + auto const itp = prev.find(_interfaces[i]); + // Guard the first tick and a missing interface; guard each direction's counter wrap/reset + // independently. + if (dt > 0.0 && itc != cur.end() && itp != prev.end()) { + if (itc->second.rx_bytes >= itp->second.rx_bytes) { + auto const delta = itc->second.rx_bytes - itp->second.rx_bytes; + rates.rx_mibps = static_cast(delta) / dt / constants::bytes_per_mib; + } + if (itc->second.tx_bytes >= itp->second.tx_bytes) { + auto const delta = itc->second.tx_bytes - itp->second.tx_bytes; + rates.tx_mibps = static_cast(delta) / dt / constants::bytes_per_mib; + } + } + total.rx_mibps += rates.rx_mibps; + total.tx_mibps += rates.tx_mibps; + nvtxCounterSample(domain, _counter_ids[i], &rates, sizeof(rates)); + } + nvtxCounterSample(domain, _total_counter_id, &total, sizeof(total)); + + prev = cur; + prev_t = now; + } +} + +} // namespace kvikio::experimental diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 4e4c7e6efc..280899e2c2 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -82,6 +82,8 @@ kvikio_add_test(NAME MMAP_TEST SOURCES test_mmap.cpp) kvikio_add_test(NAME CONCURRENT_REQUEST_LIMITER_TEST SOURCES test_concurrent_request_limiter.cpp) +kvikio_add_test(NAME NIC_MONITOR_TEST SOURCES test_nic_monitor.cpp) + if(KvikIO_REMOTE_SUPPORT) kvikio_add_test(NAME REMOTE_HANDLE_TEST SOURCES test_remote_handle.cpp utils/env.cpp) kvikio_add_test(NAME HDFS_TEST SOURCES test_hdfs.cpp utils/hdfs_helper.cpp) diff --git a/cpp/tests/test_nic_monitor.cpp b/cpp/tests/test_nic_monitor.cpp new file mode 100644 index 0000000000..faaeb6cf4d --- /dev/null +++ b/cpp/tests/test_nic_monitor.cpp @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +#include + +#include + +using kvikio::experimental::NicBandwidthMonitor; +using kvikio::experimental::read_nic_counters; + +TEST(NicMonitor, ReadCountersIncludesLoopback) +{ + auto counters = read_nic_counters(); + // The loopback interface is present on essentially every Linux host. + ASSERT_TRUE(counters.find("lo") != counters.end()); +} + +TEST(NicMonitor, RejectsNonPositiveFrequency) +{ + EXPECT_THROW(NicBandwidthMonitor(0.0), std::invalid_argument); + EXPECT_THROW(NicBandwidthMonitor(-1.0), std::invalid_argument); +} + +TEST(NicMonitor, StartStopExplicitInterface) +{ + NicBandwidthMonitor monitor{200.0, std::vector{"lo"}}; + EXPECT_FALSE(monitor.running()); + monitor.start(); + EXPECT_TRUE(monitor.running()); + std::this_thread::sleep_for(std::chrono::milliseconds{100}); + monitor.stop(); + EXPECT_FALSE(monitor.running()); + EXPECT_EQ(monitor.interfaces().size(), 1U); +} + +TEST(NicMonitor, DefaultSelectsInterfacesAndStopIsIdempotent) +{ + NicBandwidthMonitor monitor{100.0}; + monitor.start(); + std::this_thread::sleep_for(std::chrono::milliseconds{50}); + // `lo` is excluded from the default set; the monitor still runs cleanly. + for (auto const& name : monitor.interfaces()) { + EXPECT_NE(name, "lo"); + } + monitor.stop(); + monitor.stop(); // Safe to call more than once. + EXPECT_FALSE(monitor.running()); +} diff --git a/python/kvikio/kvikio/_lib/CMakeLists.txt b/python/kvikio/kvikio/_lib/CMakeLists.txt index d98cf5b047..73d06968eb 100644 --- a/python/kvikio/kvikio/_lib/CMakeLists.txt +++ b/python/kvikio/kvikio/_lib/CMakeLists.txt @@ -7,7 +7,7 @@ # Set the list of Cython files to build, one .so per file set(cython_modules arr.pyx buffer.pyx defaults.pyx cufile_driver.pyx file_handle.pyx future.pyx - mmap.pyx stream.pyx + mmap.pyx nic_monitor.pyx stream.pyx ) if(KvikIO_REMOTE_SUPPORT) diff --git a/python/kvikio/kvikio/_lib/nic_monitor.pyx b/python/kvikio/kvikio/_lib/nic_monitor.pyx new file mode 100644 index 0000000000..0cdaa9e730 --- /dev/null +++ b/python/kvikio/kvikio/_lib/nic_monitor.pyx @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# distutils: language = c++ +# cython: language_level=3 + +from cython.operator cimport dereference as deref + +from libcpp cimport bool +from libcpp.memory cimport make_unique, unique_ptr +from libcpp.string cimport string +from libcpp.utility cimport move +from libcpp.vector cimport vector + + +cdef extern from "" \ + namespace "kvikio::experimental" nogil: + cdef cppclass cpp_NicBandwidthMonitor "kvikio::experimental::NicBandwidthMonitor": + cpp_NicBandwidthMonitor(double freq_hz, vector[string] interfaces) except + + void start() except + + void stop() except + + bool running() + const vector[string]& interfaces() + + +cdef class NicBandwidthMonitor: + """Samples NIC receive bandwidth and emits it as NVTX counters. + + A background C++ thread differences the kernel byte counters at a fixed + frequency and emits one NVTX float64 counter per interface (named + ``nic_rx_MiBps.``) plus a summed ``nic_rx_MiBps.total``. The sampling + runs entirely in C++ and never holds the Python GIL. + + Parameters + ---------- + freq_hz + Sampling frequency in hertz. Must be positive. + interfaces + Interface names to monitor. If ``None`` or empty, all UP non-loopback + interfaces are selected when :meth:`start` is called. + """ + cdef unique_ptr[cpp_NicBandwidthMonitor] _handle + + def __init__(self, double freq_hz=20.0, interfaces=None): + cdef vector[string] cpp_ifaces + if interfaces is not None: + for iface in interfaces: + cpp_ifaces.push_back(str(iface).encode()) + with nogil: + self._handle = make_unique[cpp_NicBandwidthMonitor]( + freq_hz, move(cpp_ifaces) + ) + + def start(self) -> None: + """Register the NVTX counters and launch the sampling thread.""" + with nogil: + deref(self._handle).start() + + def stop(self) -> None: + """Stop and join the sampling thread. Safe to call more than once.""" + with nogil: + deref(self._handle).stop() + + def running(self) -> bool: + """Whether the sampling thread is currently running.""" + cdef bool result + with nogil: + result = deref(self._handle).running() + return result + + def interfaces(self) -> list: + """The interfaces being monitored (populated after :meth:`start`).""" + cdef vector[string] result + with nogil: + result = deref(self._handle).interfaces() + return [iface.decode() for iface in result] + + def __enter__(self) -> "NicBandwidthMonitor": + self.start() + return self + + def __exit__(self, *exc) -> None: + self.stop() diff --git a/python/kvikio/kvikio/tools/__init__.py b/python/kvikio/kvikio/tools/__init__.py new file mode 100644 index 0000000000..250e79de4c --- /dev/null +++ b/python/kvikio/kvikio/tools/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""KvikIO tools: profiling and diagnostic utilities. + +This subpackage holds standalone tools that help measure and profile KvikIO +I/O. The first tool is :mod:`kvikio.tools.remote_io_monitor`, which samples NIC +bandwidth and publishes it as an NVTX counter for the Nsight Systems timeline. +""" diff --git a/python/kvikio/kvikio/tools/remote_io_monitor.py b/python/kvikio/kvikio/tools/remote_io_monitor.py new file mode 100644 index 0000000000..38d69a48a2 --- /dev/null +++ b/python/kvikio/kvikio/tools/remote_io_monitor.py @@ -0,0 +1,517 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""NIC bandwidth monitor for KvikIO remote I/O. + +Samples per-interface network byte counters while an application runs and +reports the receive rate in MiB/s. There are two independent outputs: + +* **NVTX counter track** (default): emitted by the native C++ engine + (:class:`kvikio._lib.nic_monitor.NicBandwidthMonitor`) so the bandwidth curve + renders on the Nsight Systems timeline next to CUDA and KvikIO activity. The + C++ sampling thread never holds the Python GIL. +* **CSV time series** (``--csv``): a dependency-free, pure-Python sampler that + needs neither nsys nor the native engine, useful for a quick log and for + testing. + +Run it as a launcher that wraps any command (the application goes after ``--``):: + + nsys profile --trace=cuda,nvtx,osrt --trace-fork-before-exec=true \\ + --force-overwrite=true --output=netio \\ + python -m kvikio.tools.remote_io_monitor --freq-hz 50 -- + +For embedded use inside your own Python process, the native engine is a context +manager: ``with kvikio._lib.nic_monitor.NicBandwidthMonitor(20.0): run_query()`` +(GIL-immune, and the NVTX track lands in the app's own rows). +""" + +from __future__ import annotations + +import abc +import argparse +import enum +import os +import signal +import subprocess +import sys +import time +from collections.abc import Callable, Iterable, Sequence +from typing import Final, NamedTuple, Optional + +_SYSFS_NET: Final[str] = "/sys/class/net" +_MIB: Final[float] = float(1 << 20) + + +def _read_text(path: str) -> str: + """Read and return the full contents of a small text file.""" + with open(path) as f: + return f.read() + + +# --------------------------------------------------------------------------- +# Counter sources (used by the pure-Python CSV path; the native NVTX engine +# reads /sys directly in C++). +# --------------------------------------------------------------------------- + + +class NicCounters(NamedTuple): + """Cumulative byte counters for a single network interface. + + Attributes + ---------- + rx_bytes : int + Total bytes received since boot. + tx_bytes : int + Total bytes transmitted since boot. + """ + + rx_bytes: int + tx_bytes: int + + +class NicCounterSourceKind(enum.Enum): + """Selects the implementation that reads NIC byte counters. + + Attributes + ---------- + AUTO : int + Use ``SYSFS`` when ``/sys/class/net`` is available, otherwise ``PSUTIL``. + SYSFS : int + Read ``/sys/class/net//statistics/{rx,tx}_bytes`` directly. + PSUTIL : int + Read counters via ``psutil.net_io_counters(pernic=True)``. + """ + + AUTO = 0 + SYSFS = 1 + PSUTIL = 2 + + @staticmethod + def parse(name: str) -> NicCounterSourceKind: + """Return the kind matching a case-insensitive name.""" + return NicCounterSourceKind[name.strip().upper()] + + +class NicCounterSource(abc.ABC): + """Reads cumulative NIC byte counters, one entry per interface.""" + + @abc.abstractmethod + def read_counters(self) -> dict[str, NicCounters]: + """Return current cumulative counters keyed by interface name.""" + + @staticmethod + def create( + kind: NicCounterSourceKind = NicCounterSourceKind.AUTO, + ) -> NicCounterSource: + """Create a counter source. + + Parameters + ---------- + kind + Which implementation to use. ``AUTO`` prefers the sysfs reader when + ``/sys/class/net`` exists and falls back to psutil. + + Returns + ------- + NicCounterSource + A ready-to-use source instance. + + Raises + ------ + ValueError + If ``kind`` is not a known source kind. + """ + if kind is NicCounterSourceKind.AUTO: + kind = ( + NicCounterSourceKind.SYSFS + if os.path.isdir(_SYSFS_NET) + else NicCounterSourceKind.PSUTIL + ) + if kind is NicCounterSourceKind.SYSFS: + return SysfsCounterSource() + if kind is NicCounterSourceKind.PSUTIL: + return PsutilCounterSource() + raise ValueError(f"unknown counter source kind: {kind!r}") + + +class SysfsCounterSource(NicCounterSource): + """Counter source backed by ``/sys/class/net//statistics``.""" + + def read_counters(self) -> dict[str, NicCounters]: + """Read RX/TX byte counters for every interface under sysfs.""" + result: dict[str, NicCounters] = {} + try: + names = os.listdir(_SYSFS_NET) + except OSError: + return result + for name in names: + stats = os.path.join(_SYSFS_NET, name, "statistics") + try: + rx = int(_read_text(os.path.join(stats, "rx_bytes"))) + tx = int(_read_text(os.path.join(stats, "tx_bytes"))) + except (OSError, ValueError): + # Interface vanished or exposes no byte counters; skip it. + continue + result[name] = NicCounters(rx, tx) + return result + + +class PsutilCounterSource(NicCounterSource): + """Counter source backed by ``psutil.net_io_counters(pernic=True)``. + + ``psutil`` is imported lazily, so it is only required when this source is + used (the default ``sysfs``/``auto`` path has no such dependency). + """ + + def __init__(self) -> None: + try: + import psutil + except ImportError as exc: + raise RuntimeError( + "PsutilCounterSource requires psutil; install it with " + "'pip install psutil', or use the default sysfs source." + ) from exc + self._psutil = psutil + + def read_counters(self) -> dict[str, NicCounters]: + """Read RX/TX byte counters for every interface psutil reports.""" + per_nic = self._psutil.net_io_counters(pernic=True) + return { + name: NicCounters(stat.bytes_recv, stat.bytes_sent) + for name, stat in per_nic.items() + } + + +# --------------------------------------------------------------------------- +# CSV path: a pure-Python sampler (no native engine, no nsys). +# --------------------------------------------------------------------------- + + +class BandwidthSample(NamedTuple): + """One bandwidth reading across the monitored interfaces. + + Attributes + ---------- + monotonic_s : float + ``time.monotonic()`` timestamp when the reading was taken. + rates_mibps : dict[str, float] + Receive rate in MiB/s for each monitored interface. + total_mibps : float + Sum of ``rates_mibps`` across interfaces, kept separate so that + ``"total"`` is never mistaken for a real interface name. + """ + + monotonic_s: float + rates_mibps: dict[str, float] + total_mibps: float + + +class Sink(abc.ABC): + """Destination for :class:`BandwidthSample` records.""" + + @abc.abstractmethod + def open(self, interfaces: Sequence[str]) -> None: + """Prepare the sink for the given fixed set of interfaces.""" + + @abc.abstractmethod + def emit(self, sample: BandwidthSample) -> None: + """Write one sample to the destination.""" + + @abc.abstractmethod + def close(self) -> None: + """Release any resources held by the sink.""" + + +class CsvSink(Sink): + """Appends samples to a CSV file, one column per interface plus total.""" + + def __init__(self, path: str) -> None: + self._path = path + self._file = None + self._interfaces: list[str] = [] + + def open(self, interfaces: Sequence[str]) -> None: + """Open the file and write the header row.""" + self._interfaces = list(interfaces) + self._file = open(self._path, "w", buffering=1) + header = ["monotonic_s", "total_mibps", *self._interfaces] + self._file.write(",".join(header) + "\n") + + def emit(self, sample: BandwidthSample) -> None: + """Append one row for ``sample``.""" + if self._file is None: + return + row = [f"{sample.monotonic_s:.6f}", f"{sample.total_mibps:.3f}"] + row += [f"{sample.rates_mibps.get(n, 0.0):.3f}" for n in self._interfaces] + self._file.write(",".join(row) + "\n") + + def close(self) -> None: + """Close the file.""" + if self._file is not None: + self._file.close() + self._file = None + + +def _iface_is_up(name: str) -> bool: + """Return whether an interface is administratively up. + + Reads ``/sys/class/net//operstate``. When the state cannot be read the + interface is kept, since an unknown state is not a reason to drop it. + """ + try: + state = _read_text(os.path.join(_SYSFS_NET, name, "operstate")).strip() + except OSError: + return True + return state != "down" + + +def _select_interfaces( + available: Iterable[str], requested: Optional[Sequence[str]] +) -> list[str]: + """Choose which interfaces to monitor. + + When ``requested`` is given it is used verbatim. Otherwise all UP + non-loopback interfaces are selected; traffic is never used as a filter + because an idle-at-start interface can become active later. + """ + if requested: + return list(requested) + return sorted(n for n in available if n != "lo" and _iface_is_up(n)) + + +class CsvBandwidthLogger: + """Pure-Python sampler that writes per-interface MiB/s to a CSV file. + + This path needs neither the native engine nor nsys, so it is testable in + pure Python and works as a dependency-free fallback. + + Parameters + ---------- + source + Counter source to read. + path + CSV file to write. + freq_hz + Sampling frequency in hertz. + interfaces + Interfaces to log. ``None`` selects all UP non-loopback interfaces at + :meth:`open` time. + """ + + def __init__( + self, + source: NicCounterSource, + path: str, + *, + freq_hz: float = 20.0, + interfaces: Optional[Sequence[str]] = None, + ) -> None: + if freq_hz <= 0.0: + raise ValueError("freq_hz must be positive") + self._source = source + self._sink = CsvSink(path) + self._freq_hz = float(freq_hz) + self._requested = list(interfaces) if interfaces is not None else None + self._interfaces: list[str] = [] + self._prev: dict[str, NicCounters] = {} + self._prev_t = 0.0 + self._opened = False + + def open(self) -> None: + """Select interfaces, take a baseline reading, and open the CSV file.""" + if self._opened: + return + counters = self._source.read_counters() + self._interfaces = _select_interfaces(counters.keys(), self._requested) + self._prev = counters + self._prev_t = time.monotonic() + self._sink.open(self._interfaces) + self._opened = True + + def close(self) -> None: + """Close the CSV file. Safe to call more than once.""" + if not self._opened: + return + self._sink.close() + self._opened = False + + def sample(self) -> BandwidthSample: + """Read the source and return the rate since the previous sample.""" + now = time.monotonic() + counters = self._source.read_counters() + dt = now - self._prev_t + rates: dict[str, float] = {} + for name in self._interfaces: + cur = counters.get(name) + prev = self._prev.get(name) + if cur is None or prev is None or dt <= 0.0: + rates[name] = 0.0 + else: + delta = max(0, cur.rx_bytes - prev.rx_bytes) + rates[name] = delta / dt / _MIB + self._prev = counters + self._prev_t = now + return BandwidthSample(now, rates, sum(rates.values())) + + def run_until(self, stop: Callable[[], bool]) -> None: + """Sample and write at ``freq_hz`` until ``stop()`` is true. + + Writes one final sample after the loop to capture the tail/drain. + """ + if not self._opened: + raise RuntimeError("CsvBandwidthLogger.run_until requires open()") + interval = 1.0 / self._freq_hz + next_t = time.monotonic() + while not stop(): + self._sink.emit(self.sample()) + next_t += interval + time.sleep(max(0.0, next_t - time.monotonic())) + self._sink.emit(self.sample()) + + +# --------------------------------------------------------------------------- +# NVTX path: the native C++ engine. +# --------------------------------------------------------------------------- + + +def _make_native_monitor(freq_hz: float, interfaces: Optional[Sequence[str]]): + """Construct the native NVTX engine, with a clear error if unavailable.""" + try: + from kvikio._lib.nic_monitor import NicBandwidthMonitor + except ImportError as exc: + raise RuntimeError( + "The native NIC monitor is unavailable; build KvikIO " + "(build-all) to enable the NVTX counter track, or run with " + "--no-nvtx --csv for a pure-Python CSV log." + ) from exc + return NicBandwidthMonitor(freq_hz, list(interfaces) if interfaces else None) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _build_arg_parser() -> argparse.ArgumentParser: + """Build the CLI argument parser (monitor flags only; app goes after --).""" + parser = argparse.ArgumentParser( + prog="python -m kvikio.tools.remote_io_monitor", + description=( + "Sample NIC bandwidth while running an application. By default it " + "emits an NVTX counter track (native engine); add --csv for a " + "pure-Python CSV log. Put the application command after '--'." + ), + ) + parser.add_argument( + "--iface", + action="append", + metavar="NAME", + help="Interface to monitor; repeatable. Default: all UP non-loopback.", + ) + parser.add_argument( + "--freq-hz", + type=float, + default=20.0, + help="Sampling frequency in hertz (default: %(default)s).", + ) + parser.add_argument( + "--no-nvtx", + action="store_true", + help="Do not emit the NVTX counter track (use with --csv).", + ) + parser.add_argument( + "--csv", + metavar="PATH", + help="Also write samples to this CSV file (pure-Python, no nsys needed).", + ) + parser.add_argument( + "--counter-source", + default="auto", + choices=["auto", "sysfs", "psutil"], + help="Counter source for the CSV path (default: %(default)s).", + ) + return parser + + +def _split_argv(argv: Sequence[str]) -> tuple[list[str], list[str]]: + """Split ``argv`` at the first ``--`` into (monitor args, app argv).""" + argv = list(argv) + if "--" not in argv: + return argv, [] + idx = argv.index("--") + return argv[:idx], argv[idx + 1 :] + + +def main(argv: Optional[Sequence[str]] = None) -> int: + """Run an application under the NIC monitor (CLI entry point). + + Parameters + ---------- + argv + Argument list excluding the program name. Defaults to ``sys.argv[1:]``. + + Returns + ------- + int + The application's exit code, or 1 on a usage error. + """ + monitor_argv, app_argv = _split_argv( + sys.argv[1:] if argv is None else argv + ) + args = _build_arg_parser().parse_args(monitor_argv) + if not app_argv: + print("error: no application command given after '--'", file=sys.stderr) + return 1 + + nvtx_enabled = not args.no_nvtx + csv_enabled = args.csv is not None + if not nvtx_enabled and not csv_enabled: + print( + "error: --no-nvtx given without --csv leaves nothing to do", + file=sys.stderr, + ) + return 1 + + # Build outputs before launching the app so a failure (e.g. native engine + # not built) does not leave an orphaned child process. + monitor = _make_native_monitor(args.freq_hz, args.iface) if nvtx_enabled else None + csv_logger = None + if csv_enabled: + source = NicCounterSource.create( + NicCounterSourceKind.parse(args.counter_source) + ) + csv_logger = CsvBandwidthLogger( + source, args.csv, freq_hz=args.freq_hz, interfaces=args.iface + ) + csv_logger.open() + + if monitor is not None: + monitor.start() + proc = subprocess.Popen(app_argv) + + def _forward(signum, frame) -> None: + # Pass the signal to the child so it can shut down; the loop ends when + # the child exits. + proc.send_signal(signum) + + old_int = signal.signal(signal.SIGINT, _forward) + old_term = signal.signal(signal.SIGTERM, _forward) + try: + if csv_logger is not None: + csv_logger.run_until(lambda: proc.poll() is not None) + else: + interval = min(0.05, 1.0 / args.freq_hz) + while proc.poll() is None: + time.sleep(interval) + finally: + if monitor is not None: + monitor.stop() + if csv_logger is not None: + csv_logger.close() + signal.signal(signal.SIGINT, old_int) + signal.signal(signal.SIGTERM, old_term) + return proc.returncode if proc.returncode is not None else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/kvikio/tests/test_remote_io_monitor.py b/python/kvikio/tests/test_remote_io_monitor.py new file mode 100644 index 0000000000..0786db5c67 --- /dev/null +++ b/python/kvikio/tests/test_remote_io_monitor.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys + +import pytest + +from kvikio.tools import remote_io_monitor as rim + + +class FakeSource(rim.NicCounterSource): + """Counter source whose RX/TX grow by a fixed step on every read.""" + + def __init__(self, names=("eth0", "eth1"), step=1 << 20): + self._names = list(names) + self._step = step + self._calls = 0 + + def read_counters(self): + self._calls += 1 + base = self._calls * self._step + return {n: rim.NicCounters(base, base) for n in self._names} + + +def test_source_kind_parse(): + assert rim.NicCounterSourceKind.parse("auto") is rim.NicCounterSourceKind.AUTO + assert rim.NicCounterSourceKind.parse("SYSFS") is rim.NicCounterSourceKind.SYSFS + assert rim.NicCounterSourceKind.parse("PsUtil") is rim.NicCounterSourceKind.PSUTIL + + +def test_create_auto_returns_a_source(): + source = rim.NicCounterSource.create() + assert isinstance(source, rim.NicCounterSource) + counters = source.read_counters() + assert isinstance(counters, dict) + for name, c in counters.items(): + assert isinstance(name, str) + assert isinstance(c, rim.NicCounters) + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="sysfs is Linux only" +) +def test_sysfs_source_reports_loopback(): + counters = rim.SysfsCounterSource().read_counters() + assert "lo" in counters + + +def test_select_interfaces_respects_request(): + chosen = rim._select_interfaces(["lo", "eth0", "eth1"], ["eth1"]) + assert chosen == ["eth1"] + + +def test_csv_logger_sample_math(tmp_path): + logger = rim.CsvBandwidthLogger( + FakeSource(), + str(tmp_path / "bw.csv"), + freq_hz=50.0, + interfaces=["eth0", "eth1"], + ) + logger.open() + sample = logger.sample() + assert set(sample.rates_mibps) == {"eth0", "eth1"} + assert all(rate > 0.0 for rate in sample.rates_mibps.values()) + assert sample.total_mibps == pytest.approx(sum(sample.rates_mibps.values())) + logger.close() + + +def test_csv_logger_writes_csv(tmp_path): + csv_path = tmp_path / "bw.csv" + logger = rim.CsvBandwidthLogger( + FakeSource(), str(csv_path), freq_hz=200.0, interfaces=["eth0", "eth1"] + ) + logger.open() + counter = {"n": 0} + + def stop(): + counter["n"] += 1 + return counter["n"] > 3 + + logger.run_until(stop) + logger.close() + + lines = csv_path.read_text().strip().splitlines() + assert lines[0] == "monotonic_s,total_mibps,eth0,eth1" + assert len(lines) >= 2 # header + at least one row + + +def test_csv_logger_rejects_bad_freq(tmp_path): + with pytest.raises(ValueError): + rim.CsvBandwidthLogger(FakeSource(), str(tmp_path / "x.csv"), freq_hz=0.0) + + +def test_cli_runs_app_and_writes_csv(tmp_path): + csv_path = tmp_path / "cli.csv" + rc = rim.main( + [ + "--no-nvtx", + "--csv", + str(csv_path), + "--freq-hz", + "100", + "--counter-source", + "auto", + "--", + sys.executable, + "-c", + "import time; time.sleep(0.15)", + ] + ) + assert rc == 0 + assert csv_path.exists() + lines = csv_path.read_text().strip().splitlines() + assert lines[0].startswith("monotonic_s,total_mibps,") + + +def test_cli_requires_app_command(): + assert rim.main(["--no-nvtx", "--csv", "x.csv"]) == 1 + + +def test_cli_no_output_is_error(): + assert rim.main(["--no-nvtx", "--", sys.executable, "-c", "pass"]) == 1 + + +def test_native_engine_if_built(): + nm = pytest.importorskip("kvikio._lib.nic_monitor") + import time + + monitor = nm.NicBandwidthMonitor(100.0, ["lo"]) + assert not monitor.running() + monitor.start() + assert monitor.running() + time.sleep(0.1) + monitor.stop() + assert not monitor.running() + assert monitor.interfaces() == ["lo"] From e88620d5e132e46b62491631875731afabc3b65b Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 2 Jul 2026 16:36:06 -0400 Subject: [PATCH 02/16] Switch to the plugin method --- conda/recipes/libkvikio/recipe.yaml | 38 +- cpp/CMakeLists.txt | 17 +- .../kvikio/experimental/nic_monitor.hpp | 223 +++++--- cpp/nsys_plugins/nic/CMakeLists.txt | 56 ++ .../nic/kvikio_nic_nsys_plugin.cpp | 326 +++++++++++ cpp/nsys_plugins/nic/nic_monitor.cpp | 132 +++++ cpp/nsys_plugins/nic/nic_monitor.hpp | 87 +++ cpp/nsys_plugins/nic/nsys-plugin.yaml | 17 + cpp/src/experimental/nic_monitor.cpp | 279 ---------- cpp/tests/CMakeLists.txt | 27 +- cpp/tests/test_nic_monitor.cpp | 71 +-- python/kvikio/kvikio/_lib/CMakeLists.txt | 2 +- python/kvikio/kvikio/_lib/nic_monitor.pyx | 83 --- python/kvikio/kvikio/tools/__init__.py | 9 - .../kvikio/kvikio/tools/remote_io_monitor.py | 517 ------------------ python/kvikio/tests/test_remote_io_monitor.py | 138 ----- python/libkvikio/CMakeLists.txt | 7 +- python/libkvikio/libkvikio/__init__.py | 10 +- python/libkvikio/libkvikio/nsys.py | 25 + 19 files changed, 929 insertions(+), 1135 deletions(-) create mode 100644 cpp/nsys_plugins/nic/CMakeLists.txt create mode 100644 cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp create mode 100644 cpp/nsys_plugins/nic/nic_monitor.cpp create mode 100644 cpp/nsys_plugins/nic/nic_monitor.hpp create mode 100644 cpp/nsys_plugins/nic/nsys-plugin.yaml delete mode 100644 cpp/src/experimental/nic_monitor.cpp delete mode 100644 python/kvikio/kvikio/_lib/nic_monitor.pyx delete mode 100644 python/kvikio/kvikio/tools/__init__.py delete mode 100644 python/kvikio/kvikio/tools/remote_io_monitor.py delete mode 100644 python/kvikio/tests/test_remote_io_monitor.py create mode 100644 python/libkvikio/libkvikio/nsys.py diff --git a/conda/recipes/libkvikio/recipe.yaml b/conda/recipes/libkvikio/recipe.yaml index bd49a3091b..f731c46f3d 100644 --- a/conda/recipes/libkvikio/recipe.yaml +++ b/conda/recipes/libkvikio/recipe.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 schema_version: 1 @@ -151,3 +151,39 @@ outputs: homepage: ${{ load_from_file("python/libkvikio/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libkvikio/pyproject.toml").project.license }} summary: libkvikio tests + + # The plugin is a standalone executable (NVTX header-only plus libdl); it links neither libkvikio + # nor CUDA. The requirements below mirror the libkvikio-tests output so the same validated build + # environment and libstdc++ runtime baseline apply; they can be slimmed once separately verified. + - package: + name: libkvikio-nsys-plugin + version: ${{ version }} + build: + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" + script: + content: | + cmake --install cpp/build --component nsys_plugin + requirements: + build: + - cmake ${{ cmake_version }} + - ${{ compiler("c") }} + host: + - ${{ pin_subpackage("libkvikio", exact=True) }} + - cuda-version =${{ cuda_version }} + run: + - if: cuda_version >= "13.0" + then: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + else: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="12.2.0a0") }} + ignore_run_exports: + by_name: + - cuda-version + - libnuma + - libcufile + about: + homepage: ${{ load_from_file("python/libkvikio/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libkvikio/pyproject.toml").project.license }} + summary: KvikIO Nsight Systems NIC bandwidth plugin diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 08384e1245..4be0fc2e3a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -53,6 +53,13 @@ option(KvikIO_BUILD_EXAMPLES "Configure CMake to build examples" ON) option(KvikIO_BUILD_TESTS "Configure CMake to build tests" ON) option(KvikIO_REMOTE_SUPPORT "Configure CMake to build with remote IO support" ON) +# The Nsight Systems NIC plugin reads /sys/class/net, so it is only meaningful on Linux. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + option(KvikIO_BUILD_NSYS_PLUGIN "Configure CMake to build the Nsight Systems NIC plugin" ON) +else() + option(KvikIO_BUILD_NSYS_PLUGIN "Configure CMake to build the Nsight Systems NIC plugin" OFF) +endif() + # ################################################################################################## # * conda environment ------------------------------------------------------------------------------ rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) @@ -168,7 +175,6 @@ set(SOURCES "src/file_utils.cpp" "src/logger.cpp" "src/mmap.cpp" - "src/experimental/nic_monitor.cpp" "src/detail/bounce_buffer_cache.cpp" "src/detail/concurrent_request_limiter.cpp" "src/detail/env.cpp" @@ -265,6 +271,13 @@ if(KvikIO_BUILD_EXAMPLES) add_subdirectory(examples) endif() +# ################################################################################################## +# * add Nsight Systems plugin ---------------------------------------------------------------------- + +if(KvikIO_BUILD_NSYS_PLUGIN) + add_subdirectory(nsys_plugins/nic) +endif() + if(KvikIO_BUILD_TESTS AND CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) include(cmake/thirdparty/get_gtest.cmake) diff --git a/cpp/include/kvikio/experimental/nic_monitor.hpp b/cpp/include/kvikio/experimental/nic_monitor.hpp index 9d7cce2cbe..ded02e1428 100644 --- a/cpp/include/kvikio/experimental/nic_monitor.hpp +++ b/cpp/include/kvikio/experimental/nic_monitor.hpp @@ -1,18 +1,32 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include +#include +#include #include +#include #include -#include +#include +#include #include -#include +#include +#include #include +#include +#include + namespace kvikio::experimental { +namespace constants { +inline constexpr double bytes_per_mib = 1024.0 * 1024.0; +inline constexpr char const* sysfs_net = "/sys/class/net"; +} // namespace constants + /** * @brief Cumulative byte counters for a single network interface. * @@ -24,6 +38,103 @@ struct NicCounters { std::uint64_t tx_bytes; ///< Total bytes transmitted since boot. }; +/** + * @brief One bandwidth sample for a single interface: the receive and transmit rates. + * + * The field names are deliberately unit-neutral. `compute_rates` produces the values in MiB/s, but + * the unit is not baked into the type; a consumer attaches it separately (for example as NVTX + * counter semantics), so switching units later does not ripple into these names. + */ +struct NicRates { + double rx; ///< Receive rate. + double tx; ///< Transmit rate. +}; + +/** + * @brief Read a sysfs pseudo-file in a single read(). + * + * @param path Path of the pseudo-file to read. + * @param buf Scratch buffer that backs the returned view. At most `buf.size()` bytes are read, so + * it never overflows. + * @return A view of the bytes read (into @p buf), or std::nullopt if the file cannot be read. The + * string view is not necessarily null terminated. + */ +[[nodiscard]] inline std::optional read_pseudo_file(char const* path, + std::span buf) noexcept +{ + auto const fd = ::open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { return std::nullopt; } + auto const n = ::read(fd, buf.data(), buf.size()); + ::close(fd); + if (n <= 0) { return std::nullopt; } + return std::string_view{buf.data(), static_cast(n)}; +} + +/** + * @brief Parse a single unsigned integer from a sysfs pseudo-file. + * + * @param path Path of the pseudo-file to read. + * @return The parsed value, or std::nullopt if the file is unparseable. + */ +[[nodiscard]] inline std::optional read_u64_file(std::string const& path) noexcept +{ + // A uint64 counter is at most 20 digits, so 32 bytes is ample (20 digits + newline) + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return std::nullopt; } + std::uint64_t value{}; + // from_chars is locale-independent, non-allocating, and non-throwing, and stops at the trailing + // newline. + [[maybe_unused]] auto const [_, ec] = + std::from_chars(content->data(), content->data() + content->size(), value); + if (ec != std::errc{}) { return std::nullopt; } + return value; +} + +/** + * @brief Check whether a network interface is up. + * + * @param name Interface name. + * @return True only when operstate is "up" or "unknown". False for any other state or if it cannot + * be read. + */ +[[nodiscard]] inline bool iface_is_up(std::string const& name) +{ + auto const path = std::string{constants::sysfs_net} + "/" + name + "/operstate"; + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return false; } + std::string_view state = content.value(); + while (!state.empty() && (state.back() == '\n' || state.back() == ' ')) { + state.remove_suffix(1); + } + // Some working NICs report "unknown" (loopback or virtual NICs), so this state should be + // accepted. The other states (down, notpresent, lowerlayerdown, testing, dormant) should be + // excluded. + return state == "up" || state == "unknown"; +} + +/** + * @brief Enumerate the default set of interfaces to monitor. + * + * @return All "up" non-loopback interfaces under `/sys/class/net`, sorted for a stable order. + */ +[[nodiscard]] inline std::vector default_interfaces() +{ + std::vector result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto name = entry.path().filename().string(); + if (name == "lo") { continue; } + if (!iface_is_up(name)) { continue; } + result.push_back(std::move(name)); + } + std::sort(result.begin(), result.end()); + return result; +} + /** * @brief Read the current cumulative byte counters for every network interface. * @@ -33,75 +144,49 @@ struct NicCounters { * @return A map from interface name to its cumulative counters. Empty if `/sys/class/net` is * unavailable. */ -std::map read_nic_counters(); +[[nodiscard]] inline std::map read_nic_counters() +{ + std::map result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto const name = entry.path().filename().string(); + auto const stats = entry.path() / "statistics"; + auto const rx = read_u64_file((stats / "rx_bytes").string()); + auto const tx = read_u64_file((stats / "tx_bytes").string()); + // If an interface exposes no parseable byte counters, skip it. + if (!rx.has_value() || !tx.has_value()) { continue; } + result.emplace(name, NicCounters{rx.value(), tx.value()}); + } + return result; +} /** - * @brief Samples NIC bandwidth and emits it as NVTX counter groups. + * @brief Convert a pair of cumulative counter samples into receive and transmit rates in MiB/s. * - * On `start()`, a background `std::jthread` differences the kernel byte counters at a fixed - * frequency and emits one NVTX counter group per monitored interface (named `nic_MiBps.`) - * plus a summed `nic_MiBps.total`, in the `libkvikio` NVTX domain. Each group carries the receive - * and transmit rates (`rx_MiBps`, `tx_MiBps`) sampled together, so they render as a single grouped - * counter track on the Nsight Systems timeline. + * @param prev The earlier counter sample. + * @param cur The later counter sample. + * @param dt_seconds Elapsed wall-clock seconds between the two samples. + * @return Rates in MiB/s. A direction whose counter did not advance yields zero for that direction + * (this guards the first sample as well as a counter wrap or reset). Both rates are zero when + * @p dt_seconds is not positive. */ -class NicBandwidthMonitor { - public: - /** - * @brief Construct a monitor. - * - * @param freq_hz Sampling frequency in hertz. Must be positive. - * @param interfaces Interfaces to monitor. If empty, all UP non-loopback interfaces are selected - * once at `start()`. - * - * @exception std::invalid_argument if @p freq_hz is not positive. - */ - explicit NicBandwidthMonitor(double freq_hz = 20.0, std::vector interfaces = {}); - - /** - * @brief Stop the sampling thread if it is running. - */ - ~NicBandwidthMonitor(); - - // Non-copyable - NicBandwidthMonitor(NicBandwidthMonitor const&) = delete; - NicBandwidthMonitor& operator=(NicBandwidthMonitor const&) = delete; - - /** - * @brief Select interfaces (if not given), register NVTX counters, and launch the sampling - * thread. Has no effect if already running. - */ - void start(); - - /** - * @brief Signal the sampling thread to stop and join it. Has no effect if not running. - */ - void stop(); - - /** - * @brief Whether the sampling thread is currently running. - */ - [[nodiscard]] bool running() const noexcept; - - /** - * @brief The interfaces being monitored (populated at `start()`). - */ - [[nodiscard]] std::vector const& interfaces() const noexcept; - - private: - void run(std::stop_token stop_token); - - double _freq_hz; - std::vector _interfaces; - - // NVTX counter ids, parallel to `_interfaces`, plus the total counter id. - std::vector _counter_ids; - std::uint64_t _total_counter_id{0}; - - // Declared after the members used by `run()`, so the jthread's auto-join on destruction happens - // before those members are destroyed. - std::jthread _thread; - - bool _running{false}; -}; +[[nodiscard]] inline NicRates compute_rates(NicCounters const& prev, + NicCounters const& cur, + double dt_seconds) noexcept +{ + NicRates rates{0.0, 0.0}; + if (dt_seconds <= 0.0) { return rates; } + if (cur.rx_bytes >= prev.rx_bytes) { + auto const delta = cur.rx_bytes - prev.rx_bytes; + rates.rx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; + } + if (cur.tx_bytes >= prev.tx_bytes) { + auto const delta = cur.tx_bytes - prev.tx_bytes; + rates.tx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; + } + return rates; +} } // namespace kvikio::experimental diff --git a/cpp/nsys_plugins/nic/CMakeLists.txt b/cpp/nsys_plugins/nic/CMakeLists.txt new file mode 100644 index 0000000000..84b69a4941 --- /dev/null +++ b/cpp/nsys_plugins/nic/CMakeLists.txt @@ -0,0 +1,56 @@ +# ============================================================================= +# cmake-format: off +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# cmake-format: on +# ============================================================================= + +# The plugin is a standalone Nsight Systems collector. It reads NVTX headers only (header-only) and +# libdl, so it deliberately does not link libkvikio and is fully relocatable. +set(NSYS_PLUGIN_INSTALL_DIR share/kvikio/nsys-plugins/kvikio_nic) + +# The NIC-reading core (nic_monitor.cpp) is compiled directly into the plugin so it links no shared +# library beyond NVTX (header-only) and libdl. +add_executable(kvikio_nic_nsys_plugin kvikio_nic_nsys_plugin.cpp nic_monitor.cpp) + +set_target_properties( + kvikio_nic_nsys_plugin + PROPERTIES CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + INSTALL_RPATH "" +) + +target_link_libraries(kvikio_nic_nsys_plugin PRIVATE nvtx3::nvtx3-cpp ${CMAKE_DL_LIBS}) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options( + kvikio_nic_nsys_plugin PRIVATE "$<$:-Wall;-Werror;-Wno-unknown-pragmas>" + ) +endif() + +# Install the binary and its manifest side by side into a directory named after the plugin, matching +# how nsys discovers plugins under NSYS_PLUGIN_SEARCH_DIRS. +# +# By default the install rules are EXCLUDE_FROM_ALL, so a plain `cmake --install` (the base conda +# libkvikio package) skips them and the plugin ships only via `cmake --install --component +# nsys_plugin` (the dedicated conda output). The libkvikio wheel sets +# KvikIO_NSYS_PLUGIN_INSTALL_DEFAULT so the plugin is part of the default install and is packaged +# without needing component selection in scikit-build-core. +if(KvikIO_NSYS_PLUGIN_INSTALL_DEFAULT) + set(_kvikio_nsys_plugin_exclude "") +else() + set(_kvikio_nsys_plugin_exclude "EXCLUDE_FROM_ALL") +endif() + +install( + TARGETS kvikio_nic_nsys_plugin + COMPONENT nsys_plugin + DESTINATION ${NSYS_PLUGIN_INSTALL_DIR} + ${_kvikio_nsys_plugin_exclude} +) +install( + FILES nsys-plugin.yaml + COMPONENT nsys_plugin + DESTINATION ${NSYS_PLUGIN_INSTALL_DIR} + ${_kvikio_nsys_plugin_exclude} +) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp new file mode 100644 index 0000000000..7aba670b25 --- /dev/null +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -0,0 +1,326 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// Nsight Systems plugin that samples per-interface network bandwidth and emits it as NVTX counter +// groups, so the bandwidth curve lands on the same timeline as CUDA and KvikIO activity. +// +// nsys spawns this executable as a collector for the duration of a profiling session, enabled with +// `--enable=kvikio_nic[,args]` and discovered via `NSYS_PLUGIN_SEARCH_DIRS` (see nsys-plugin.yaml). +// It links only NVTX (header-only) and libdl, so it is standalone and does not depend on libkvikio. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "nic_monitor.hpp" + +using kvikio::experimental::compute_rates; +using kvikio::experimental::default_interfaces; +using kvikio::experimental::NicRates; +using kvikio::experimental::read_nic_counters; +namespace constants = kvikio::experimental::constants; + +namespace { + +/** + * @brief Tag type naming this plugin's NVTX domain. + * + * The domain is separate from libkvikio's own domain because the plugin runs as its own process; it + * is created through the nvtx3 C++ registry (`nvtx3::domain::get`) rather than `nvtxDomainCreateA`. + */ +struct kvikio_nic_domain { + static constexpr char const* name{"KvikIO NIC"}; +}; + +// Set from a signal handler to request a clean shutdown. `volatile sig_atomic_t` is the only type +// an async-signal-safe handler may touch. +volatile std::sig_atomic_t g_stop = 0; + +/** + * @brief Parsed command line configuration. + */ +struct Config { + std::chrono::microseconds interval{50000}; ///< Sampling interval. + std::optional device_filter; ///< If set, monitor interfaces matching this regex. +}; + +/** + * @brief Print usage to stderr and terminate the process. + * + * @param prog Program name (argv[0]). + * @param code Process exit code. + */ +[[noreturn]] void print_help_and_exit(char const* prog, int code) +{ + std::fprintf(stderr, + "Usage: %s [options]\n" + " -i | --interval Sampling interval in microseconds (default 50000)\n" + " -d | --device Interface name regex (default: up or unknown, non-loopback)\n" + " -h | --help Print this help message\n", + prog); + std::exit(code); +} + +/** + * @brief Parse the command line into a Config, exiting on `--help` or a malformed argument. + * + * Accepts `-i N` / `--interval N` / `--interval=N` (and the `-iN` short form) plus the equivalent + * `-d` / `--device` forms, matching how nsys forwards `--enable=kvikio_nic,-d,eth0,-i,50000`. + * + * @param argc Argument count. + * @param argv Argument vector. + * @return The parsed configuration. + */ +Config parse_args(int argc, char** argv) +{ + Config config; + for (int i = 1; i < argc; ++i) { + std::string_view const arg{argv[i]}; + std::string_view name; + std::optional inline_value; + if (arg.starts_with("--")) { + auto const eq = arg.find('='); + if (eq == std::string_view::npos) { + name = arg; + } else { + name = arg.substr(0, eq); + inline_value = arg.substr(eq + 1); + } + } else if (arg.size() >= 2 && arg.front() == '-') { + name = arg.substr(0, 2); + if (arg.size() > 2) { inline_value = arg.substr(2); } + } else { + std::fprintf(stderr, + "kvikio_nic: unexpected argument '%.*s'\n", + static_cast(arg.size()), + arg.data()); + print_help_and_exit(argv[0], 2); + } + + // Take this option's value from the inline form (`--interval=N`, `-iN`) or the next argument. + auto take_value = [&]() -> std::string { + if (inline_value.has_value()) { return std::string{inline_value.value()}; } + if (i + 1 < argc) { return std::string{argv[++i]}; } + std::fprintf(stderr, + "kvikio_nic: option '%.*s' requires a value\n", + static_cast(name.size()), + name.data()); + print_help_and_exit(argv[0], 2); + }; + + if (name == "-h" || name == "--help") { + print_help_and_exit(argv[0], 0); + } else if (name == "-i" || name == "--interval") { + auto const value = take_value(); + long long parsed = 0; + auto const [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + auto const fully_parsed = (ec == std::errc{} && ptr == value.data() + value.size()); + if (!fully_parsed || parsed <= 0) { + std::fprintf(stderr, + "kvikio_nic: invalid interval '%s' (expected a positive integer)\n", + value.c_str()); + print_help_and_exit(argv[0], 2); + } + config.interval = std::chrono::microseconds{parsed}; + } else if (name == "-d" || name == "--device") { + auto const value = take_value(); + try { + config.device_filter = std::regex{value}; + } catch (std::regex_error const& e) { + std::fprintf( + stderr, "kvikio_nic: invalid device regex '%s' (%s)\n", value.c_str(), e.what()); + print_help_and_exit(argv[0], 2); + } + } else { + std::fprintf( + stderr, "kvikio_nic: unknown option '%.*s'\n", static_cast(name.size()), name.data()); + print_help_and_exit(argv[0], 2); + } + } + return config; +} + +/** + * @brief Choose the interfaces to monitor. + * + * @param config Parsed configuration. + * @return With no `--device` filter, all up (or unknown) non-loopback interfaces. With a filter, + * all interfaces whose name matches the regex, bypassing the up check so an explicit request is + * honored. Sorted for a stable order. + */ +std::vector select_interfaces(Config const& config) +{ + if (!config.device_filter.has_value()) { return default_interfaces(); } + std::vector result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto name = entry.path().filename().string(); + if (std::regex_match(name, config.device_filter.value())) { result.push_back(std::move(name)); } + } + std::sort(result.begin(), result.end()); + return result; +} + +/** + * @brief Build the counter semantics that carry the rate unit for a whole counter group. + * + * Keeping the unit here (rather than in the schema field names) means the identifiers stay + * unit-neutral and a future unit change touches only this value. + * + * @return A populated counter-semantics extension describing MiB/s values. + */ +nvtxSemanticsCounter_t make_rate_semantics() +{ + nvtxSemanticsCounter_t sem{}; + sem.header.structSize = sizeof(nvtxSemanticsCounter_t); + sem.header.semanticId = NVTX_SEMANTIC_ID_COUNTERS_V1; + sem.header.version = NVTX_COUNTER_SEMANTIC_VERSION; + sem.header.next = nullptr; + sem.flags = NVTX_COUNTER_FLAGS_NONE; + sem.unit = "MiB/s"; + sem.unitScaleNumerator = 1; + sem.unitScaleDenominator = 1; + sem.limitType = NVTX_COUNTER_LIMIT_UNDEFINED; + return sem; +} + +/** + * @brief Register the unit-neutral {rx, tx} payload schema shared by every counter group. + * + * @param domain NVTX domain handle. + * @return The schema id to pass as nvtxCounterAttr_t::schemaId. + */ +std::uint64_t register_rate_schema(nvtxDomainHandle_t domain) +{ + nvtxPayloadSchemaEntry_t const entries[] = { + {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "rx"}, + {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "tx"}, + }; + nvtxPayloadSchemaAttr_t attr{}; + attr.fieldMask = NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_TYPE | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_ENTRIES | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NUM_ENTRIES | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_STATIC_SIZE; + attr.type = NVTX_PAYLOAD_SCHEMA_TYPE_STATIC; + attr.entries = entries; + attr.numEntries = 2; + attr.payloadStaticSize = sizeof(NicRates); + return nvtxPayloadSchemaRegister(domain, &attr); +} + +/** + * @brief Register an NVTX counter group in the given domain. + * + * @param domain NVTX domain handle. + * @param name Counter group name (for example "nic_bandwidth.eth0"). + * @param schema_id Payload schema id from register_rate_schema. + * @return The counter id to pass to nvtxCounterSample(). + */ +std::uint64_t register_counter(nvtxDomainHandle_t domain, + std::string const& name, + std::uint64_t schema_id) +{ + // The semantics apply to the whole group and must outlive this call, so keep them static. + static nvtxSemanticsCounter_t const rate_semantics = make_rate_semantics(); + nvtxCounterAttr_t attr{}; + attr.structSize = sizeof(nvtxCounterAttr_t); + attr.schemaId = schema_id; + attr.name = name.c_str(); + attr.scopeId = NVTX_SCOPE_CURRENT_VM; + attr.semantics = &rate_semantics.header; + attr.counterId = NVTX_COUNTER_ID_NONE; + return nvtxCounterRegister(domain, &attr); +} + +} // namespace + +extern "C" { +// nsys stops the collector by sending SIGTERM (then SIGKILL after a grace period). Catch it so the +// loop breaks and the process exits cleanly with code 0 instead of an abnormal termination; SIGINT +// gives the same clean exit for standalone Ctrl-C runs. +static void kvikio_nic_handle_signal(int /*signum*/) { g_stop = 1; } +} + +int main(int argc, char** argv) +{ + auto const config = parse_args(argc, argv); + auto const interfaces = select_interfaces(config); + if (interfaces.empty()) { + std::fprintf(stderr, "kvikio_nic: no matching network interfaces to monitor.\n"); + return 1; + } + + std::signal(SIGTERM, kvikio_nic_handle_signal); + std::signal(SIGINT, kvikio_nic_handle_signal); + + nvtxDomainHandle_t domain = nvtx3::domain::get(); + auto const schema_id = register_rate_schema(domain); + + std::vector counter_ids; + counter_ids.reserve(interfaces.size()); + for (auto const& name : interfaces) { + counter_ids.push_back(register_counter(domain, "nic_bandwidth." + name, schema_id)); + } + auto const total_counter_id = register_counter(domain, "nic_bandwidth.total", schema_id); + + std::fprintf(stderr, + "kvikio_nic: sampling %zu interface(s) every %lld us\n", + interfaces.size(), + static_cast(config.interval.count())); + + using clock = std::chrono::steady_clock; + auto prev = read_nic_counters(); + auto prev_t = clock::now(); + auto next_t = prev_t; + + while (g_stop == 0) { + // Absolute deadlines keep the sampling frequency drift-free. A signal does not shorten the + // sleep (libstdc++ restarts it over EINTR), so g_stop is observed at the next tick. + next_t += config.interval; + std::this_thread::sleep_until(next_t); + + auto const now = clock::now(); + auto const cur = read_nic_counters(); + double const dt = std::chrono::duration{now - prev_t}.count(); + + NicRates total{0.0, 0.0}; + for (std::size_t i = 0; i < interfaces.size(); ++i) { + NicRates rates{0.0, 0.0}; + auto const itc = cur.find(interfaces[i]); + auto const itp = prev.find(interfaces[i]); + if (itc != cur.end() && itp != prev.end()) { + rates = compute_rates(itp->second, itc->second, dt); + } + total.rx += rates.rx; + total.tx += rates.tx; + nvtxCounterSample(domain, counter_ids[i], &rates, sizeof(rates)); + } + nvtxCounterSample(domain, total_counter_id, &total, sizeof(total)); + + prev = cur; + prev_t = now; + } + + return 0; +} diff --git a/cpp/nsys_plugins/nic/nic_monitor.cpp b/cpp/nsys_plugins/nic/nic_monitor.cpp new file mode 100644 index 0000000000..ef8e9f3172 --- /dev/null +++ b/cpp/nsys_plugins/nic/nic_monitor.cpp @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "nic_monitor.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace kvikio::experimental { + +namespace { + +/** + * @brief Read a sysfs pseudo-file in a single read(). + * + * @param path Path of the pseudo-file to read. + * @param buf Scratch buffer that backs the returned view. At most `buf.size()` bytes are read, so + * it never overflows. + * @return A view of the bytes read (into @p buf), or std::nullopt if the file cannot be read. The + * string view is not necessarily null terminated. + */ +[[nodiscard]] std::optional read_pseudo_file(char const* path, + std::span buf) noexcept +{ + auto const fd = ::open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { return std::nullopt; } + auto const n = ::read(fd, buf.data(), buf.size()); + ::close(fd); + if (n <= 0) { return std::nullopt; } + return std::string_view{buf.data(), static_cast(n)}; +} + +/** + * @brief Parse a single unsigned integer from a sysfs pseudo-file. + * + * @param path Path of the pseudo-file to read. + * @return The parsed value, or std::nullopt if the file is unparseable. + */ +[[nodiscard]] std::optional read_u64_file(std::string const& path) noexcept +{ + // A uint64 counter is at most 20 digits, so 32 bytes is ample (20 digits + newline) + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return std::nullopt; } + std::uint64_t value{}; + // from_chars is locale-independent, non-allocating, and non-throwing, and stops at the trailing + // newline. + [[maybe_unused]] auto const [_, ec] = + std::from_chars(content->data(), content->data() + content->size(), value); + if (ec != std::errc{}) { return std::nullopt; } + return value; +} + +} // namespace + +bool iface_is_up(std::string const& name) +{ + auto const path = std::string{constants::sysfs_net} + "/" + name + "/operstate"; + std::array buf; + auto const content = read_pseudo_file(path.c_str(), buf); + if (!content.has_value()) { return false; } + std::string_view state = content.value(); + while (!state.empty() && (state.back() == '\n' || state.back() == ' ')) { + state.remove_suffix(1); + } + // Some working NICs report "unknown" (loopback or virtual NICs), so this state should be + // accepted. The other states (down, notpresent, lowerlayerdown, testing, dormant) should be + // excluded. + return state == "up" || state == "unknown"; +} + +std::vector default_interfaces() +{ + std::vector result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto name = entry.path().filename().string(); + if (name == "lo") { continue; } + if (!iface_is_up(name)) { continue; } + result.push_back(std::move(name)); + } + std::sort(result.begin(), result.end()); + return result; +} + +std::map read_nic_counters() +{ + std::map result; + std::error_code ec; + std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + if (ec) { return result; } + for (auto const& entry : it) { + auto const name = entry.path().filename().string(); + auto const stats = entry.path() / "statistics"; + auto const rx = read_u64_file((stats / "rx_bytes").string()); + auto const tx = read_u64_file((stats / "tx_bytes").string()); + // If an interface exposes no parseable byte counters, skip it. + if (!rx.has_value() || !tx.has_value()) { continue; } + result.emplace(name, NicCounters{rx.value(), tx.value()}); + } + return result; +} + +NicRates compute_rates(NicCounters const& prev, NicCounters const& cur, double dt_seconds) noexcept +{ + NicRates rates{0.0, 0.0}; + if (dt_seconds <= 0.0) { return rates; } + if (cur.rx_bytes >= prev.rx_bytes) { + auto const delta = cur.rx_bytes - prev.rx_bytes; + rates.rx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; + } + if (cur.tx_bytes >= prev.tx_bytes) { + auto const delta = cur.tx_bytes - prev.tx_bytes; + rates.tx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; + } + return rates; +} + +} // namespace kvikio::experimental diff --git a/cpp/nsys_plugins/nic/nic_monitor.hpp b/cpp/nsys_plugins/nic/nic_monitor.hpp new file mode 100644 index 0000000000..990a7cc867 --- /dev/null +++ b/cpp/nsys_plugins/nic/nic_monitor.hpp @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include +#include + +// Internal support code for the KvikIO Nsight Systems NIC plugin. It reads network byte counters +// from sysfs and converts them to rates. It is compiled directly into the plugin and its test; it +// is not part of libkvikio and is not an installed public header. + +namespace kvikio::experimental { + +namespace constants { +inline constexpr double bytes_per_mib = 1024.0 * 1024.0; +inline constexpr char const* sysfs_net = "/sys/class/net"; +} // namespace constants + +/** + * @brief Cumulative byte counters for a single network interface. + * + * The values are the kernel's running totals since boot, as exposed by + * `/sys/class/net//statistics/{rx,tx}_bytes`. + */ +struct NicCounters { + std::uint64_t rx_bytes; ///< Total bytes received since boot. + std::uint64_t tx_bytes; ///< Total bytes transmitted since boot. +}; + +/** + * @brief One bandwidth sample for a single interface: the receive and transmit rates. + * + * The field names are deliberately unit-neutral. `compute_rates` produces the values in MiB/s, but + * the unit is not baked into the type; a consumer attaches it separately (for example as NVTX + * counter semantics), so switching units later does not ripple into these names. + */ +struct NicRates { + double rx; ///< Receive rate. + double tx; ///< Transmit rate. +}; + +/** + * @brief Check whether a network interface is up. + * + * @param name Interface name. + * @return True only when operstate is "up" or "unknown". False for any other state or if it cannot + * be read. + */ +[[nodiscard]] bool iface_is_up(std::string const& name); + +/** + * @brief Enumerate the default set of interfaces to monitor. + * + * @return All "up" non-loopback interfaces under `/sys/class/net`, sorted for a stable order. + */ +[[nodiscard]] std::vector default_interfaces(); + +/** + * @brief Read the current cumulative byte counters for every network interface. + * + * Reads `/sys/class/net//statistics/{rx,tx}_bytes`. Interfaces whose counters cannot be read + * are skipped rather than reported as an error. + * + * @return A map from interface name to its cumulative counters. Empty if `/sys/class/net` is + * unavailable. + */ +[[nodiscard]] std::map read_nic_counters(); + +/** + * @brief Convert a pair of cumulative counter samples into receive and transmit rates in MiB/s. + * + * @param prev The earlier counter sample. + * @param cur The later counter sample. + * @param dt_seconds Elapsed wall-clock seconds between the two samples. + * @return Rates in MiB/s. A direction whose counter did not advance yields zero for that direction + * (this guards the first sample as well as a counter wrap or reset). Both rates are zero when + * @p dt_seconds is not positive. + */ +[[nodiscard]] NicRates compute_rates(NicCounters const& prev, + NicCounters const& cur, + double dt_seconds) noexcept; + +} // namespace kvikio::experimental diff --git a/cpp/nsys_plugins/nic/nsys-plugin.yaml b/cpp/nsys_plugins/nic/nsys-plugin.yaml new file mode 100644 index 0000000000..16e16ea1c7 --- /dev/null +++ b/cpp/nsys_plugins/nic/nsys-plugin.yaml @@ -0,0 +1,17 @@ +PluginName: kvikio_nic +FeatureName: Collect KvikIO NIC bandwidth +ExecutablePath: kvikio_nic_nsys_plugin +Description: Per-interface RX/TX network bandwidth from /sys/class/net, as NVTX counter groups. +ExtendedDescription: | + Command line options: + -i | --interval Sampling interval in microseconds (default 50000) + -d | --device Interface name regex (default: up or unknown, non-loopback) + -h | --help Print this help message + + With no --device filter, all up (or unknown) non-loopback interfaces are monitored. + A --device regex selects interfaces by name and bypasses the up check, so an explicit + request is always honored. +ExtendedDescriptionForGui: | + Options: + -i | --interval Sampling interval in microseconds (default 50000) + -d | --device Interface name regex (default: up or unknown, non-loopback) diff --git a/cpp/src/experimental/nic_monitor.cpp b/cpp/src/experimental/nic_monitor.cpp deleted file mode 100644 index a12c330d4e..0000000000 --- a/cpp/src/experimental/nic_monitor.cpp +++ /dev/null @@ -1,279 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include -#include - -namespace kvikio::experimental { - -namespace { - -namespace constants { -constexpr double bytes_per_mib = 1024.0 * 1024.0; -constexpr char const* sysfs_net = "/sys/class/net"; -} // namespace constants - -/** - * @brief Read a sysfs pseudo-file in a single read(). - * - * @param path Path of the pseudo-file to read. - * @param buf Scratch buffer that backs the returned view. At most `buf.size()` bytes are read, so - * it never overflows. - * @return A view of the bytes read (into @p buf), or std::nullopt if the file cannot be read. The - * string view is not necessarily null terminated. - */ -std::optional read_pseudo_file(char const* path, std::span buf) noexcept -{ - auto const fd = ::open(path, O_RDONLY | O_CLOEXEC); - if (fd < 0) { return std::nullopt; } - auto const n = ::read(fd, buf.data(), buf.size()); - ::close(fd); - if (n <= 0) { return std::nullopt; } - return std::string_view{buf.data(), static_cast(n)}; -} - -/** - * @brief Parse a single unsigned integer from a sysfs pseudo-file. - * - * @param path Path of the pseudo-file to read. - * @return The parsed value, or std::nullopt if the file is unparseable. - */ -std::optional read_u64_file(std::string const& path) noexcept -{ - // A uint64 counter is at most 20 digits, so 32 bytes is ample (20 digits + newline) - std::array buf; - auto const content = read_pseudo_file(path.c_str(), buf); - if (!content.has_value()) { return std::nullopt; } - std::uint64_t value{}; - // from_chars is locale-independent, non-allocating, and non-throwing, and stops at the trailing - // newline. - [[maybe_unused]] auto const [_, ec] = - std::from_chars(content->data(), content->data() + content->size(), value); - if (ec != std::errc{}) { return std::nullopt; } - return value; -} - -/** - * @brief Check whether a network interface is up. - * - * @param name Interface name. - * @return True only when operstate is "up" or "unknown". False for any other state or if it cannot - * be read. - */ -bool iface_is_up(std::string const& name) -{ - auto const path = std::string{constants::sysfs_net} + "/" + name + "/operstate"; - std::array buf; - auto const content = read_pseudo_file(path.c_str(), buf); - if (!content.has_value()) { return false; } - std::string_view state = content.value(); - while (!state.empty() && (state.back() == '\n' || state.back() == ' ')) { - state.remove_suffix(1); - } - // Some working NICs report "unknown" (loopback or virtual NICs), so this state should be - // accepted. The other states (down, notpresent, lowerlayerdown, testing, dormant) should be - // excluded. - return state == "up" || state == "unknown"; -} - -/** - * @brief Enumerate the default set of interfaces to monitor. - * - * @return All "up" non-loopback interfaces under `/sys/class/net`, sorted for a stable order. - */ -std::vector default_interfaces() -{ - std::vector result; - std::error_code ec; - std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; - if (ec) { return result; } - for (auto const& entry : it) { - auto name = entry.path().filename().string(); - if (name == "lo") { continue; } - if (!iface_is_up(name)) { continue; } - result.push_back(std::move(name)); - } - std::sort(result.begin(), result.end()); - return result; -} - -// One sample of a counter group: receive and transmit rates for one interface. -struct NicRates { - double rx_mibps; - double tx_mibps; -}; - -/** - * @brief Register the {rx_MiBps, tx_MiBps} payload schema for the counter groups. - * - * @param domain NVTX domain handle. - * @return The schema id to pass as nvtxCounterAttr_t::schemaId. - */ -std::uint64_t register_rate_schema(nvtxDomainHandle_t domain) -{ - nvtxPayloadSchemaEntry_t const entries[] = { - {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "rx_MiBps"}, - {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "tx_MiBps"}, - }; - nvtxPayloadSchemaAttr_t attr{}; - attr.fieldMask = NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_TYPE | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_ENTRIES | - NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NUM_ENTRIES | - NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_STATIC_SIZE; - attr.type = NVTX_PAYLOAD_SCHEMA_TYPE_STATIC; - attr.entries = entries; - attr.numEntries = 2; - attr.payloadStaticSize = sizeof(NicRates); - return nvtxPayloadSchemaRegister(domain, &attr); -} - -/** - * @brief Register an NVTX counter group in the given domain using a payload schema. - * - * @param domain NVTX domain handle. - * @param name Counter name (for example "nic_MiBps.eth0"). - * @param schema_id Payload schema id from register_rate_schema. - * @return The counter id to pass to nvtxCounterSample(). - */ -std::uint64_t register_counter(nvtxDomainHandle_t domain, - std::string const& name, - std::uint64_t schema_id) -{ - nvtxCounterAttr_t attr{}; - attr.structSize = sizeof(nvtxCounterAttr_t); - attr.schemaId = schema_id; - attr.name = name.c_str(); - attr.scopeId = NVTX_SCOPE_NONE; - attr.counterId = NVTX_COUNTER_ID_NONE; - return nvtxCounterRegister(domain, &attr); -} - -} // namespace - -std::map read_nic_counters() -{ - std::map result; - std::error_code ec; - std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; - if (ec) { return result; } - for (auto const& entry : it) { - auto const name = entry.path().filename().string(); - auto const stats = entry.path() / "statistics"; - auto const rx = read_u64_file((stats / "rx_bytes").string()); - auto const tx = read_u64_file((stats / "tx_bytes").string()); - // If an interface exposes no parseable byte counters, skip it. - if (!rx.has_value() || !tx.has_value()) { continue; } - result.emplace(name, NicCounters{rx.value(), tx.value()}); - } - return result; -} - -NicBandwidthMonitor::NicBandwidthMonitor(double freq_hz, std::vector interfaces) - : _freq_hz{freq_hz}, _interfaces{std::move(interfaces)} -{ - KVIKIO_EXPECT(freq_hz > 0.0, "freq_hz must be positive", std::invalid_argument); -} - -NicBandwidthMonitor::~NicBandwidthMonitor() { stop(); } - -void NicBandwidthMonitor::start() -{ - if (_running) { return; } - if (_interfaces.empty()) { _interfaces = default_interfaces(); } - - nvtxDomainHandle_t domain = nvtx3::domain::get(); - auto const schema_id = register_rate_schema(domain); - _counter_ids.clear(); - _counter_ids.reserve(_interfaces.size()); - for (auto const& name : _interfaces) { - _counter_ids.push_back(register_counter(domain, "nic_MiBps." + name, schema_id)); - } - _total_counter_id = register_counter(domain, "nic_MiBps.total", schema_id); - - _running = true; - _thread = std::jthread{[this](std::stop_token stop_token) { run(stop_token); }}; -} - -void NicBandwidthMonitor::stop() -{ - if (!_running) { return; } - _thread.request_stop(); - if (_thread.joinable()) { _thread.join(); } - _running = false; -} - -bool NicBandwidthMonitor::running() const noexcept { return _running; } - -std::vector const& NicBandwidthMonitor::interfaces() const noexcept -{ - return _interfaces; -} - -void NicBandwidthMonitor::run(std::stop_token stop_token) -{ - nvtxDomainHandle_t domain = nvtx3::domain::get(); - using clock = std::chrono::steady_clock; - auto const interval = std::chrono::duration{1.0 / _freq_hz}; - - auto prev = read_nic_counters(); - auto prev_t = clock::now(); - - while (!stop_token.stop_requested()) { - std::this_thread::sleep_for(std::chrono::duration_cast(interval)); - - auto const now = clock::now(); - auto const cur = read_nic_counters(); - double const dt = std::chrono::duration{now - prev_t}.count(); - - NicRates total{0.0, 0.0}; - for (std::size_t i = 0; i < _interfaces.size(); ++i) { - NicRates rates{0.0, 0.0}; - auto const itc = cur.find(_interfaces[i]); - auto const itp = prev.find(_interfaces[i]); - // Guard the first tick and a missing interface; guard each direction's counter wrap/reset - // independently. - if (dt > 0.0 && itc != cur.end() && itp != prev.end()) { - if (itc->second.rx_bytes >= itp->second.rx_bytes) { - auto const delta = itc->second.rx_bytes - itp->second.rx_bytes; - rates.rx_mibps = static_cast(delta) / dt / constants::bytes_per_mib; - } - if (itc->second.tx_bytes >= itp->second.tx_bytes) { - auto const delta = itc->second.tx_bytes - itp->second.tx_bytes; - rates.tx_mibps = static_cast(delta) / dt / constants::bytes_per_mib; - } - } - total.rx_mibps += rates.rx_mibps; - total.tx_mibps += rates.tx_mibps; - nvtxCounterSample(domain, _counter_ids[i], &rates, sizeof(rates)); - } - nvtxCounterSample(domain, _total_counter_id, &total, sizeof(total)); - - prev = cur; - prev_t = now; - } -} - -} // namespace kvikio::experimental diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 280899e2c2..6226118c26 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -82,7 +82,30 @@ kvikio_add_test(NAME MMAP_TEST SOURCES test_mmap.cpp) kvikio_add_test(NAME CONCURRENT_REQUEST_LIMITER_TEST SOURCES test_concurrent_request_limiter.cpp) -kvikio_add_test(NAME NIC_MONITOR_TEST SOURCES test_nic_monitor.cpp) +# The NIC monitor core lives with the Nsight Systems plugin (nsys_plugins/nic) and does not depend +# on libkvikio. Its test compiles that core directly and links only GoogleTest, rather than going +# through kvikio_add_test (which links kvikio and CUDA). +add_executable( + NIC_MONITOR_TEST test_nic_monitor.cpp ${KvikIO_SOURCE_DIR}/nsys_plugins/nic/nic_monitor.cpp +) +set_target_properties( + NIC_MONITOR_TEST + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON +) +target_include_directories( + NIC_MONITOR_TEST PRIVATE "$" +) +target_link_libraries( + NIC_MONITOR_TEST PRIVATE GTest::gmock GTest::gmock_main GTest::gtest GTest::gtest_main +) +rapids_test_add( + NAME NIC_MONITOR_TEST + COMMAND NIC_MONITOR_TEST + GPUS 1 + INSTALL_COMPONENT_SET testing +) if(KvikIO_REMOTE_SUPPORT) kvikio_add_test(NAME REMOTE_HANDLE_TEST SOURCES test_remote_handle.cpp utils/env.cpp) diff --git a/cpp/tests/test_nic_monitor.cpp b/cpp/tests/test_nic_monitor.cpp index faaeb6cf4d..6786be327f 100644 --- a/cpp/tests/test_nic_monitor.cpp +++ b/cpp/tests/test_nic_monitor.cpp @@ -1,55 +1,64 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ -#include -#include -#include -#include +#include #include -#include +#include "nic_monitor.hpp" -using kvikio::experimental::NicBandwidthMonitor; +using kvikio::experimental::compute_rates; +using kvikio::experimental::default_interfaces; +using kvikio::experimental::iface_is_up; +using kvikio::experimental::NicCounters; using kvikio::experimental::read_nic_counters; +using kvikio::experimental::constants::bytes_per_mib; TEST(NicMonitor, ReadCountersIncludesLoopback) { - auto counters = read_nic_counters(); + auto const counters = read_nic_counters(); // The loopback interface is present on essentially every Linux host. ASSERT_TRUE(counters.find("lo") != counters.end()); } -TEST(NicMonitor, RejectsNonPositiveFrequency) +TEST(NicMonitor, IfaceIsUpAcceptsLoopbackAndRejectsMissing) { - EXPECT_THROW(NicBandwidthMonitor(0.0), std::invalid_argument); - EXPECT_THROW(NicBandwidthMonitor(-1.0), std::invalid_argument); + // Loopback reports operstate "unknown", which the up/unknown policy accepts. + EXPECT_TRUE(iface_is_up("lo")); + // A name that cannot be read must be treated as not up rather than accepted. + EXPECT_FALSE(iface_is_up("kvikio_no_such_iface")); } -TEST(NicMonitor, StartStopExplicitInterface) +TEST(NicMonitor, DefaultInterfacesExcludeLoopback) { - NicBandwidthMonitor monitor{200.0, std::vector{"lo"}}; - EXPECT_FALSE(monitor.running()); - monitor.start(); - EXPECT_TRUE(monitor.running()); - std::this_thread::sleep_for(std::chrono::milliseconds{100}); - monitor.stop(); - EXPECT_FALSE(monitor.running()); - EXPECT_EQ(monitor.interfaces().size(), 1U); + for (auto const& name : default_interfaces()) { + EXPECT_NE(name, "lo"); + } } -TEST(NicMonitor, DefaultSelectsInterfacesAndStopIsIdempotent) +TEST(NicMonitor, ComputeRatesConvertsDeltasToMiBps) { - NicBandwidthMonitor monitor{100.0}; - monitor.start(); - std::this_thread::sleep_for(std::chrono::milliseconds{50}); - // `lo` is excluded from the default set; the monitor still runs cleanly. - for (auto const& name : monitor.interfaces()) { - EXPECT_NE(name, "lo"); - } - monitor.stop(); - monitor.stop(); // Safe to call more than once. - EXPECT_FALSE(monitor.running()); + NicCounters const prev{0, 0}; + NicCounters const cur{static_cast(bytes_per_mib) * 2, + static_cast(bytes_per_mib) * 6}; + auto const rates = compute_rates(prev, cur, 2.0); + // 2 MiB received over 2 s -> 1 MiB/s; 6 MiB transmitted over 2 s -> 3 MiB/s. + EXPECT_DOUBLE_EQ(rates.rx, 1.0); + EXPECT_DOUBLE_EQ(rates.tx, 3.0); +} + +TEST(NicMonitor, ComputeRatesGuardsWrapAndNonPositiveInterval) +{ + NicCounters const high{1000, 1000}; + NicCounters const low{10, 10}; + // A counter that appears to go backwards (wrap or reset) yields zero for that direction. + auto const wrapped = compute_rates(high, low, 1.0); + EXPECT_DOUBLE_EQ(wrapped.rx, 0.0); + EXPECT_DOUBLE_EQ(wrapped.tx, 0.0); + // A non-positive elapsed time yields zero rather than dividing by zero. + auto const no_time = compute_rates(low, high, 0.0); + EXPECT_DOUBLE_EQ(no_time.rx, 0.0); + EXPECT_DOUBLE_EQ(no_time.tx, 0.0); } diff --git a/python/kvikio/kvikio/_lib/CMakeLists.txt b/python/kvikio/kvikio/_lib/CMakeLists.txt index 73d06968eb..d98cf5b047 100644 --- a/python/kvikio/kvikio/_lib/CMakeLists.txt +++ b/python/kvikio/kvikio/_lib/CMakeLists.txt @@ -7,7 +7,7 @@ # Set the list of Cython files to build, one .so per file set(cython_modules arr.pyx buffer.pyx defaults.pyx cufile_driver.pyx file_handle.pyx future.pyx - mmap.pyx nic_monitor.pyx stream.pyx + mmap.pyx stream.pyx ) if(KvikIO_REMOTE_SUPPORT) diff --git a/python/kvikio/kvikio/_lib/nic_monitor.pyx b/python/kvikio/kvikio/_lib/nic_monitor.pyx deleted file mode 100644 index 0cdaa9e730..0000000000 --- a/python/kvikio/kvikio/_lib/nic_monitor.pyx +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# distutils: language = c++ -# cython: language_level=3 - -from cython.operator cimport dereference as deref - -from libcpp cimport bool -from libcpp.memory cimport make_unique, unique_ptr -from libcpp.string cimport string -from libcpp.utility cimport move -from libcpp.vector cimport vector - - -cdef extern from "" \ - namespace "kvikio::experimental" nogil: - cdef cppclass cpp_NicBandwidthMonitor "kvikio::experimental::NicBandwidthMonitor": - cpp_NicBandwidthMonitor(double freq_hz, vector[string] interfaces) except + - void start() except + - void stop() except + - bool running() - const vector[string]& interfaces() - - -cdef class NicBandwidthMonitor: - """Samples NIC receive bandwidth and emits it as NVTX counters. - - A background C++ thread differences the kernel byte counters at a fixed - frequency and emits one NVTX float64 counter per interface (named - ``nic_rx_MiBps.``) plus a summed ``nic_rx_MiBps.total``. The sampling - runs entirely in C++ and never holds the Python GIL. - - Parameters - ---------- - freq_hz - Sampling frequency in hertz. Must be positive. - interfaces - Interface names to monitor. If ``None`` or empty, all UP non-loopback - interfaces are selected when :meth:`start` is called. - """ - cdef unique_ptr[cpp_NicBandwidthMonitor] _handle - - def __init__(self, double freq_hz=20.0, interfaces=None): - cdef vector[string] cpp_ifaces - if interfaces is not None: - for iface in interfaces: - cpp_ifaces.push_back(str(iface).encode()) - with nogil: - self._handle = make_unique[cpp_NicBandwidthMonitor]( - freq_hz, move(cpp_ifaces) - ) - - def start(self) -> None: - """Register the NVTX counters and launch the sampling thread.""" - with nogil: - deref(self._handle).start() - - def stop(self) -> None: - """Stop and join the sampling thread. Safe to call more than once.""" - with nogil: - deref(self._handle).stop() - - def running(self) -> bool: - """Whether the sampling thread is currently running.""" - cdef bool result - with nogil: - result = deref(self._handle).running() - return result - - def interfaces(self) -> list: - """The interfaces being monitored (populated after :meth:`start`).""" - cdef vector[string] result - with nogil: - result = deref(self._handle).interfaces() - return [iface.decode() for iface in result] - - def __enter__(self) -> "NicBandwidthMonitor": - self.start() - return self - - def __exit__(self, *exc) -> None: - self.stop() diff --git a/python/kvikio/kvikio/tools/__init__.py b/python/kvikio/kvikio/tools/__init__.py deleted file mode 100644 index 250e79de4c..0000000000 --- a/python/kvikio/kvikio/tools/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""KvikIO tools: profiling and diagnostic utilities. - -This subpackage holds standalone tools that help measure and profile KvikIO -I/O. The first tool is :mod:`kvikio.tools.remote_io_monitor`, which samples NIC -bandwidth and publishes it as an NVTX counter for the Nsight Systems timeline. -""" diff --git a/python/kvikio/kvikio/tools/remote_io_monitor.py b/python/kvikio/kvikio/tools/remote_io_monitor.py deleted file mode 100644 index 38d69a48a2..0000000000 --- a/python/kvikio/kvikio/tools/remote_io_monitor.py +++ /dev/null @@ -1,517 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""NIC bandwidth monitor for KvikIO remote I/O. - -Samples per-interface network byte counters while an application runs and -reports the receive rate in MiB/s. There are two independent outputs: - -* **NVTX counter track** (default): emitted by the native C++ engine - (:class:`kvikio._lib.nic_monitor.NicBandwidthMonitor`) so the bandwidth curve - renders on the Nsight Systems timeline next to CUDA and KvikIO activity. The - C++ sampling thread never holds the Python GIL. -* **CSV time series** (``--csv``): a dependency-free, pure-Python sampler that - needs neither nsys nor the native engine, useful for a quick log and for - testing. - -Run it as a launcher that wraps any command (the application goes after ``--``):: - - nsys profile --trace=cuda,nvtx,osrt --trace-fork-before-exec=true \\ - --force-overwrite=true --output=netio \\ - python -m kvikio.tools.remote_io_monitor --freq-hz 50 -- - -For embedded use inside your own Python process, the native engine is a context -manager: ``with kvikio._lib.nic_monitor.NicBandwidthMonitor(20.0): run_query()`` -(GIL-immune, and the NVTX track lands in the app's own rows). -""" - -from __future__ import annotations - -import abc -import argparse -import enum -import os -import signal -import subprocess -import sys -import time -from collections.abc import Callable, Iterable, Sequence -from typing import Final, NamedTuple, Optional - -_SYSFS_NET: Final[str] = "/sys/class/net" -_MIB: Final[float] = float(1 << 20) - - -def _read_text(path: str) -> str: - """Read and return the full contents of a small text file.""" - with open(path) as f: - return f.read() - - -# --------------------------------------------------------------------------- -# Counter sources (used by the pure-Python CSV path; the native NVTX engine -# reads /sys directly in C++). -# --------------------------------------------------------------------------- - - -class NicCounters(NamedTuple): - """Cumulative byte counters for a single network interface. - - Attributes - ---------- - rx_bytes : int - Total bytes received since boot. - tx_bytes : int - Total bytes transmitted since boot. - """ - - rx_bytes: int - tx_bytes: int - - -class NicCounterSourceKind(enum.Enum): - """Selects the implementation that reads NIC byte counters. - - Attributes - ---------- - AUTO : int - Use ``SYSFS`` when ``/sys/class/net`` is available, otherwise ``PSUTIL``. - SYSFS : int - Read ``/sys/class/net//statistics/{rx,tx}_bytes`` directly. - PSUTIL : int - Read counters via ``psutil.net_io_counters(pernic=True)``. - """ - - AUTO = 0 - SYSFS = 1 - PSUTIL = 2 - - @staticmethod - def parse(name: str) -> NicCounterSourceKind: - """Return the kind matching a case-insensitive name.""" - return NicCounterSourceKind[name.strip().upper()] - - -class NicCounterSource(abc.ABC): - """Reads cumulative NIC byte counters, one entry per interface.""" - - @abc.abstractmethod - def read_counters(self) -> dict[str, NicCounters]: - """Return current cumulative counters keyed by interface name.""" - - @staticmethod - def create( - kind: NicCounterSourceKind = NicCounterSourceKind.AUTO, - ) -> NicCounterSource: - """Create a counter source. - - Parameters - ---------- - kind - Which implementation to use. ``AUTO`` prefers the sysfs reader when - ``/sys/class/net`` exists and falls back to psutil. - - Returns - ------- - NicCounterSource - A ready-to-use source instance. - - Raises - ------ - ValueError - If ``kind`` is not a known source kind. - """ - if kind is NicCounterSourceKind.AUTO: - kind = ( - NicCounterSourceKind.SYSFS - if os.path.isdir(_SYSFS_NET) - else NicCounterSourceKind.PSUTIL - ) - if kind is NicCounterSourceKind.SYSFS: - return SysfsCounterSource() - if kind is NicCounterSourceKind.PSUTIL: - return PsutilCounterSource() - raise ValueError(f"unknown counter source kind: {kind!r}") - - -class SysfsCounterSource(NicCounterSource): - """Counter source backed by ``/sys/class/net//statistics``.""" - - def read_counters(self) -> dict[str, NicCounters]: - """Read RX/TX byte counters for every interface under sysfs.""" - result: dict[str, NicCounters] = {} - try: - names = os.listdir(_SYSFS_NET) - except OSError: - return result - for name in names: - stats = os.path.join(_SYSFS_NET, name, "statistics") - try: - rx = int(_read_text(os.path.join(stats, "rx_bytes"))) - tx = int(_read_text(os.path.join(stats, "tx_bytes"))) - except (OSError, ValueError): - # Interface vanished or exposes no byte counters; skip it. - continue - result[name] = NicCounters(rx, tx) - return result - - -class PsutilCounterSource(NicCounterSource): - """Counter source backed by ``psutil.net_io_counters(pernic=True)``. - - ``psutil`` is imported lazily, so it is only required when this source is - used (the default ``sysfs``/``auto`` path has no such dependency). - """ - - def __init__(self) -> None: - try: - import psutil - except ImportError as exc: - raise RuntimeError( - "PsutilCounterSource requires psutil; install it with " - "'pip install psutil', or use the default sysfs source." - ) from exc - self._psutil = psutil - - def read_counters(self) -> dict[str, NicCounters]: - """Read RX/TX byte counters for every interface psutil reports.""" - per_nic = self._psutil.net_io_counters(pernic=True) - return { - name: NicCounters(stat.bytes_recv, stat.bytes_sent) - for name, stat in per_nic.items() - } - - -# --------------------------------------------------------------------------- -# CSV path: a pure-Python sampler (no native engine, no nsys). -# --------------------------------------------------------------------------- - - -class BandwidthSample(NamedTuple): - """One bandwidth reading across the monitored interfaces. - - Attributes - ---------- - monotonic_s : float - ``time.monotonic()`` timestamp when the reading was taken. - rates_mibps : dict[str, float] - Receive rate in MiB/s for each monitored interface. - total_mibps : float - Sum of ``rates_mibps`` across interfaces, kept separate so that - ``"total"`` is never mistaken for a real interface name. - """ - - monotonic_s: float - rates_mibps: dict[str, float] - total_mibps: float - - -class Sink(abc.ABC): - """Destination for :class:`BandwidthSample` records.""" - - @abc.abstractmethod - def open(self, interfaces: Sequence[str]) -> None: - """Prepare the sink for the given fixed set of interfaces.""" - - @abc.abstractmethod - def emit(self, sample: BandwidthSample) -> None: - """Write one sample to the destination.""" - - @abc.abstractmethod - def close(self) -> None: - """Release any resources held by the sink.""" - - -class CsvSink(Sink): - """Appends samples to a CSV file, one column per interface plus total.""" - - def __init__(self, path: str) -> None: - self._path = path - self._file = None - self._interfaces: list[str] = [] - - def open(self, interfaces: Sequence[str]) -> None: - """Open the file and write the header row.""" - self._interfaces = list(interfaces) - self._file = open(self._path, "w", buffering=1) - header = ["monotonic_s", "total_mibps", *self._interfaces] - self._file.write(",".join(header) + "\n") - - def emit(self, sample: BandwidthSample) -> None: - """Append one row for ``sample``.""" - if self._file is None: - return - row = [f"{sample.monotonic_s:.6f}", f"{sample.total_mibps:.3f}"] - row += [f"{sample.rates_mibps.get(n, 0.0):.3f}" for n in self._interfaces] - self._file.write(",".join(row) + "\n") - - def close(self) -> None: - """Close the file.""" - if self._file is not None: - self._file.close() - self._file = None - - -def _iface_is_up(name: str) -> bool: - """Return whether an interface is administratively up. - - Reads ``/sys/class/net//operstate``. When the state cannot be read the - interface is kept, since an unknown state is not a reason to drop it. - """ - try: - state = _read_text(os.path.join(_SYSFS_NET, name, "operstate")).strip() - except OSError: - return True - return state != "down" - - -def _select_interfaces( - available: Iterable[str], requested: Optional[Sequence[str]] -) -> list[str]: - """Choose which interfaces to monitor. - - When ``requested`` is given it is used verbatim. Otherwise all UP - non-loopback interfaces are selected; traffic is never used as a filter - because an idle-at-start interface can become active later. - """ - if requested: - return list(requested) - return sorted(n for n in available if n != "lo" and _iface_is_up(n)) - - -class CsvBandwidthLogger: - """Pure-Python sampler that writes per-interface MiB/s to a CSV file. - - This path needs neither the native engine nor nsys, so it is testable in - pure Python and works as a dependency-free fallback. - - Parameters - ---------- - source - Counter source to read. - path - CSV file to write. - freq_hz - Sampling frequency in hertz. - interfaces - Interfaces to log. ``None`` selects all UP non-loopback interfaces at - :meth:`open` time. - """ - - def __init__( - self, - source: NicCounterSource, - path: str, - *, - freq_hz: float = 20.0, - interfaces: Optional[Sequence[str]] = None, - ) -> None: - if freq_hz <= 0.0: - raise ValueError("freq_hz must be positive") - self._source = source - self._sink = CsvSink(path) - self._freq_hz = float(freq_hz) - self._requested = list(interfaces) if interfaces is not None else None - self._interfaces: list[str] = [] - self._prev: dict[str, NicCounters] = {} - self._prev_t = 0.0 - self._opened = False - - def open(self) -> None: - """Select interfaces, take a baseline reading, and open the CSV file.""" - if self._opened: - return - counters = self._source.read_counters() - self._interfaces = _select_interfaces(counters.keys(), self._requested) - self._prev = counters - self._prev_t = time.monotonic() - self._sink.open(self._interfaces) - self._opened = True - - def close(self) -> None: - """Close the CSV file. Safe to call more than once.""" - if not self._opened: - return - self._sink.close() - self._opened = False - - def sample(self) -> BandwidthSample: - """Read the source and return the rate since the previous sample.""" - now = time.monotonic() - counters = self._source.read_counters() - dt = now - self._prev_t - rates: dict[str, float] = {} - for name in self._interfaces: - cur = counters.get(name) - prev = self._prev.get(name) - if cur is None or prev is None or dt <= 0.0: - rates[name] = 0.0 - else: - delta = max(0, cur.rx_bytes - prev.rx_bytes) - rates[name] = delta / dt / _MIB - self._prev = counters - self._prev_t = now - return BandwidthSample(now, rates, sum(rates.values())) - - def run_until(self, stop: Callable[[], bool]) -> None: - """Sample and write at ``freq_hz`` until ``stop()`` is true. - - Writes one final sample after the loop to capture the tail/drain. - """ - if not self._opened: - raise RuntimeError("CsvBandwidthLogger.run_until requires open()") - interval = 1.0 / self._freq_hz - next_t = time.monotonic() - while not stop(): - self._sink.emit(self.sample()) - next_t += interval - time.sleep(max(0.0, next_t - time.monotonic())) - self._sink.emit(self.sample()) - - -# --------------------------------------------------------------------------- -# NVTX path: the native C++ engine. -# --------------------------------------------------------------------------- - - -def _make_native_monitor(freq_hz: float, interfaces: Optional[Sequence[str]]): - """Construct the native NVTX engine, with a clear error if unavailable.""" - try: - from kvikio._lib.nic_monitor import NicBandwidthMonitor - except ImportError as exc: - raise RuntimeError( - "The native NIC monitor is unavailable; build KvikIO " - "(build-all) to enable the NVTX counter track, or run with " - "--no-nvtx --csv for a pure-Python CSV log." - ) from exc - return NicBandwidthMonitor(freq_hz, list(interfaces) if interfaces else None) - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - -def _build_arg_parser() -> argparse.ArgumentParser: - """Build the CLI argument parser (monitor flags only; app goes after --).""" - parser = argparse.ArgumentParser( - prog="python -m kvikio.tools.remote_io_monitor", - description=( - "Sample NIC bandwidth while running an application. By default it " - "emits an NVTX counter track (native engine); add --csv for a " - "pure-Python CSV log. Put the application command after '--'." - ), - ) - parser.add_argument( - "--iface", - action="append", - metavar="NAME", - help="Interface to monitor; repeatable. Default: all UP non-loopback.", - ) - parser.add_argument( - "--freq-hz", - type=float, - default=20.0, - help="Sampling frequency in hertz (default: %(default)s).", - ) - parser.add_argument( - "--no-nvtx", - action="store_true", - help="Do not emit the NVTX counter track (use with --csv).", - ) - parser.add_argument( - "--csv", - metavar="PATH", - help="Also write samples to this CSV file (pure-Python, no nsys needed).", - ) - parser.add_argument( - "--counter-source", - default="auto", - choices=["auto", "sysfs", "psutil"], - help="Counter source for the CSV path (default: %(default)s).", - ) - return parser - - -def _split_argv(argv: Sequence[str]) -> tuple[list[str], list[str]]: - """Split ``argv`` at the first ``--`` into (monitor args, app argv).""" - argv = list(argv) - if "--" not in argv: - return argv, [] - idx = argv.index("--") - return argv[:idx], argv[idx + 1 :] - - -def main(argv: Optional[Sequence[str]] = None) -> int: - """Run an application under the NIC monitor (CLI entry point). - - Parameters - ---------- - argv - Argument list excluding the program name. Defaults to ``sys.argv[1:]``. - - Returns - ------- - int - The application's exit code, or 1 on a usage error. - """ - monitor_argv, app_argv = _split_argv( - sys.argv[1:] if argv is None else argv - ) - args = _build_arg_parser().parse_args(monitor_argv) - if not app_argv: - print("error: no application command given after '--'", file=sys.stderr) - return 1 - - nvtx_enabled = not args.no_nvtx - csv_enabled = args.csv is not None - if not nvtx_enabled and not csv_enabled: - print( - "error: --no-nvtx given without --csv leaves nothing to do", - file=sys.stderr, - ) - return 1 - - # Build outputs before launching the app so a failure (e.g. native engine - # not built) does not leave an orphaned child process. - monitor = _make_native_monitor(args.freq_hz, args.iface) if nvtx_enabled else None - csv_logger = None - if csv_enabled: - source = NicCounterSource.create( - NicCounterSourceKind.parse(args.counter_source) - ) - csv_logger = CsvBandwidthLogger( - source, args.csv, freq_hz=args.freq_hz, interfaces=args.iface - ) - csv_logger.open() - - if monitor is not None: - monitor.start() - proc = subprocess.Popen(app_argv) - - def _forward(signum, frame) -> None: - # Pass the signal to the child so it can shut down; the loop ends when - # the child exits. - proc.send_signal(signum) - - old_int = signal.signal(signal.SIGINT, _forward) - old_term = signal.signal(signal.SIGTERM, _forward) - try: - if csv_logger is not None: - csv_logger.run_until(lambda: proc.poll() is not None) - else: - interval = min(0.05, 1.0 / args.freq_hz) - while proc.poll() is None: - time.sleep(interval) - finally: - if monitor is not None: - monitor.stop() - if csv_logger is not None: - csv_logger.close() - signal.signal(signal.SIGINT, old_int) - signal.signal(signal.SIGTERM, old_term) - return proc.returncode if proc.returncode is not None else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python/kvikio/tests/test_remote_io_monitor.py b/python/kvikio/tests/test_remote_io_monitor.py deleted file mode 100644 index 0786db5c67..0000000000 --- a/python/kvikio/tests/test_remote_io_monitor.py +++ /dev/null @@ -1,138 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import sys - -import pytest - -from kvikio.tools import remote_io_monitor as rim - - -class FakeSource(rim.NicCounterSource): - """Counter source whose RX/TX grow by a fixed step on every read.""" - - def __init__(self, names=("eth0", "eth1"), step=1 << 20): - self._names = list(names) - self._step = step - self._calls = 0 - - def read_counters(self): - self._calls += 1 - base = self._calls * self._step - return {n: rim.NicCounters(base, base) for n in self._names} - - -def test_source_kind_parse(): - assert rim.NicCounterSourceKind.parse("auto") is rim.NicCounterSourceKind.AUTO - assert rim.NicCounterSourceKind.parse("SYSFS") is rim.NicCounterSourceKind.SYSFS - assert rim.NicCounterSourceKind.parse("PsUtil") is rim.NicCounterSourceKind.PSUTIL - - -def test_create_auto_returns_a_source(): - source = rim.NicCounterSource.create() - assert isinstance(source, rim.NicCounterSource) - counters = source.read_counters() - assert isinstance(counters, dict) - for name, c in counters.items(): - assert isinstance(name, str) - assert isinstance(c, rim.NicCounters) - - -@pytest.mark.skipif( - not sys.platform.startswith("linux"), reason="sysfs is Linux only" -) -def test_sysfs_source_reports_loopback(): - counters = rim.SysfsCounterSource().read_counters() - assert "lo" in counters - - -def test_select_interfaces_respects_request(): - chosen = rim._select_interfaces(["lo", "eth0", "eth1"], ["eth1"]) - assert chosen == ["eth1"] - - -def test_csv_logger_sample_math(tmp_path): - logger = rim.CsvBandwidthLogger( - FakeSource(), - str(tmp_path / "bw.csv"), - freq_hz=50.0, - interfaces=["eth0", "eth1"], - ) - logger.open() - sample = logger.sample() - assert set(sample.rates_mibps) == {"eth0", "eth1"} - assert all(rate > 0.0 for rate in sample.rates_mibps.values()) - assert sample.total_mibps == pytest.approx(sum(sample.rates_mibps.values())) - logger.close() - - -def test_csv_logger_writes_csv(tmp_path): - csv_path = tmp_path / "bw.csv" - logger = rim.CsvBandwidthLogger( - FakeSource(), str(csv_path), freq_hz=200.0, interfaces=["eth0", "eth1"] - ) - logger.open() - counter = {"n": 0} - - def stop(): - counter["n"] += 1 - return counter["n"] > 3 - - logger.run_until(stop) - logger.close() - - lines = csv_path.read_text().strip().splitlines() - assert lines[0] == "monotonic_s,total_mibps,eth0,eth1" - assert len(lines) >= 2 # header + at least one row - - -def test_csv_logger_rejects_bad_freq(tmp_path): - with pytest.raises(ValueError): - rim.CsvBandwidthLogger(FakeSource(), str(tmp_path / "x.csv"), freq_hz=0.0) - - -def test_cli_runs_app_and_writes_csv(tmp_path): - csv_path = tmp_path / "cli.csv" - rc = rim.main( - [ - "--no-nvtx", - "--csv", - str(csv_path), - "--freq-hz", - "100", - "--counter-source", - "auto", - "--", - sys.executable, - "-c", - "import time; time.sleep(0.15)", - ] - ) - assert rc == 0 - assert csv_path.exists() - lines = csv_path.read_text().strip().splitlines() - assert lines[0].startswith("monotonic_s,total_mibps,") - - -def test_cli_requires_app_command(): - assert rim.main(["--no-nvtx", "--csv", "x.csv"]) == 1 - - -def test_cli_no_output_is_error(): - assert rim.main(["--no-nvtx", "--", sys.executable, "-c", "pass"]) == 1 - - -def test_native_engine_if_built(): - nm = pytest.importorskip("kvikio._lib.nic_monitor") - import time - - monitor = nm.NicBandwidthMonitor(100.0, ["lo"]) - assert not monitor.running() - monitor.start() - assert monitor.running() - time.sleep(0.1) - monitor.stop() - assert not monitor.running() - assert monitor.interfaces() == ["lo"] diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 7a7de6e790..bc03dd58cf 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -32,5 +32,10 @@ unset(kvikio_FOUND) set(KvikIO_BUILD_BENCHMARKS OFF) set(KvikIO_BUILD_EXAMPLES OFF) set(KvikIO_BUILD_TESTS OFF) +# Build and ship the Nsight Systems NIC plugin in the libkvikio wheel. INSTALL_DEFAULT makes its +# install rules part of the default install so scikit-build-core packages them without needing to +# select a component (the base conda package keeps them excluded instead). +set(KvikIO_BUILD_NSYS_PLUGIN ON) +set(KvikIO_NSYS_PLUGIN_INSTALL_DEFAULT ON) add_subdirectory(../../cpp kvikio-cpp) diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py index 8051956848..74db43a7c9 100644 --- a/python/libkvikio/libkvikio/__init__.py +++ b/python/libkvikio/libkvikio/__init__.py @@ -1,7 +1,13 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libkvikio._version import __git_commit__, __version__ from libkvikio.load import load_library +from libkvikio.nsys import nsys_plugin_search_dir -__all__ = ["__git_commit__", "__version__", "load_library"] +__all__ = [ + "__git_commit__", + "__version__", + "load_library", + "nsys_plugin_search_dir", +] diff --git a/python/libkvikio/libkvikio/nsys.py b/python/libkvikio/libkvikio/nsys.py new file mode 100644 index 0000000000..8714361ae0 --- /dev/null +++ b/python/libkvikio/libkvikio/nsys.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os + + +def nsys_plugin_search_dir() -> str: + """Return the directory that holds KvikIO's bundled Nsight Systems plugins. + + When libkvikio is installed as a wheel, the ``kvikio_nic`` plugin (an executable + plus its ``nsys-plugin.yaml`` manifest) is packaged under this directory. Point + Nsight Systems at it by exporting the returned path on ``NSYS_PLUGIN_SEARCH_DIRS``, + for example:: + + export NSYS_PLUGIN_SEARCH_DIRS="$(python -c 'import libkvikio; print(libkvikio.nsys_plugin_search_dir())')" + nsys profile --enable=kvikio_nic ... + + Returns + ------- + str + The ``share/kvikio/nsys-plugins`` directory inside the installed + ``libkvikio`` package. The path is returned even if it does not exist, for + example when the plugin was not built. + """ + return os.path.join(os.path.dirname(__file__), "share", "kvikio", "nsys-plugins") From 552ad46fb38f19a83c0940fd6094be44024738c5 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 2 Jul 2026 16:41:35 -0400 Subject: [PATCH 03/16] Cleaup --- .../kvikio/experimental/nic_monitor.hpp | 192 ------------------ 1 file changed, 192 deletions(-) delete mode 100644 cpp/include/kvikio/experimental/nic_monitor.hpp diff --git a/cpp/include/kvikio/experimental/nic_monitor.hpp b/cpp/include/kvikio/experimental/nic_monitor.hpp deleted file mode 100644 index ded02e1428..0000000000 --- a/cpp/include/kvikio/experimental/nic_monitor.hpp +++ /dev/null @@ -1,192 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace kvikio::experimental { - -namespace constants { -inline constexpr double bytes_per_mib = 1024.0 * 1024.0; -inline constexpr char const* sysfs_net = "/sys/class/net"; -} // namespace constants - -/** - * @brief Cumulative byte counters for a single network interface. - * - * The values are the kernel's running totals since boot, as exposed by - * `/sys/class/net//statistics/{rx,tx}_bytes`. - */ -struct NicCounters { - std::uint64_t rx_bytes; ///< Total bytes received since boot. - std::uint64_t tx_bytes; ///< Total bytes transmitted since boot. -}; - -/** - * @brief One bandwidth sample for a single interface: the receive and transmit rates. - * - * The field names are deliberately unit-neutral. `compute_rates` produces the values in MiB/s, but - * the unit is not baked into the type; a consumer attaches it separately (for example as NVTX - * counter semantics), so switching units later does not ripple into these names. - */ -struct NicRates { - double rx; ///< Receive rate. - double tx; ///< Transmit rate. -}; - -/** - * @brief Read a sysfs pseudo-file in a single read(). - * - * @param path Path of the pseudo-file to read. - * @param buf Scratch buffer that backs the returned view. At most `buf.size()` bytes are read, so - * it never overflows. - * @return A view of the bytes read (into @p buf), or std::nullopt if the file cannot be read. The - * string view is not necessarily null terminated. - */ -[[nodiscard]] inline std::optional read_pseudo_file(char const* path, - std::span buf) noexcept -{ - auto const fd = ::open(path, O_RDONLY | O_CLOEXEC); - if (fd < 0) { return std::nullopt; } - auto const n = ::read(fd, buf.data(), buf.size()); - ::close(fd); - if (n <= 0) { return std::nullopt; } - return std::string_view{buf.data(), static_cast(n)}; -} - -/** - * @brief Parse a single unsigned integer from a sysfs pseudo-file. - * - * @param path Path of the pseudo-file to read. - * @return The parsed value, or std::nullopt if the file is unparseable. - */ -[[nodiscard]] inline std::optional read_u64_file(std::string const& path) noexcept -{ - // A uint64 counter is at most 20 digits, so 32 bytes is ample (20 digits + newline) - std::array buf; - auto const content = read_pseudo_file(path.c_str(), buf); - if (!content.has_value()) { return std::nullopt; } - std::uint64_t value{}; - // from_chars is locale-independent, non-allocating, and non-throwing, and stops at the trailing - // newline. - [[maybe_unused]] auto const [_, ec] = - std::from_chars(content->data(), content->data() + content->size(), value); - if (ec != std::errc{}) { return std::nullopt; } - return value; -} - -/** - * @brief Check whether a network interface is up. - * - * @param name Interface name. - * @return True only when operstate is "up" or "unknown". False for any other state or if it cannot - * be read. - */ -[[nodiscard]] inline bool iface_is_up(std::string const& name) -{ - auto const path = std::string{constants::sysfs_net} + "/" + name + "/operstate"; - std::array buf; - auto const content = read_pseudo_file(path.c_str(), buf); - if (!content.has_value()) { return false; } - std::string_view state = content.value(); - while (!state.empty() && (state.back() == '\n' || state.back() == ' ')) { - state.remove_suffix(1); - } - // Some working NICs report "unknown" (loopback or virtual NICs), so this state should be - // accepted. The other states (down, notpresent, lowerlayerdown, testing, dormant) should be - // excluded. - return state == "up" || state == "unknown"; -} - -/** - * @brief Enumerate the default set of interfaces to monitor. - * - * @return All "up" non-loopback interfaces under `/sys/class/net`, sorted for a stable order. - */ -[[nodiscard]] inline std::vector default_interfaces() -{ - std::vector result; - std::error_code ec; - std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; - if (ec) { return result; } - for (auto const& entry : it) { - auto name = entry.path().filename().string(); - if (name == "lo") { continue; } - if (!iface_is_up(name)) { continue; } - result.push_back(std::move(name)); - } - std::sort(result.begin(), result.end()); - return result; -} - -/** - * @brief Read the current cumulative byte counters for every network interface. - * - * Reads `/sys/class/net//statistics/{rx,tx}_bytes`. Interfaces whose counters cannot be read - * are skipped rather than reported as an error. - * - * @return A map from interface name to its cumulative counters. Empty if `/sys/class/net` is - * unavailable. - */ -[[nodiscard]] inline std::map read_nic_counters() -{ - std::map result; - std::error_code ec; - std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; - if (ec) { return result; } - for (auto const& entry : it) { - auto const name = entry.path().filename().string(); - auto const stats = entry.path() / "statistics"; - auto const rx = read_u64_file((stats / "rx_bytes").string()); - auto const tx = read_u64_file((stats / "tx_bytes").string()); - // If an interface exposes no parseable byte counters, skip it. - if (!rx.has_value() || !tx.has_value()) { continue; } - result.emplace(name, NicCounters{rx.value(), tx.value()}); - } - return result; -} - -/** - * @brief Convert a pair of cumulative counter samples into receive and transmit rates in MiB/s. - * - * @param prev The earlier counter sample. - * @param cur The later counter sample. - * @param dt_seconds Elapsed wall-clock seconds between the two samples. - * @return Rates in MiB/s. A direction whose counter did not advance yields zero for that direction - * (this guards the first sample as well as a counter wrap or reset). Both rates are zero when - * @p dt_seconds is not positive. - */ -[[nodiscard]] inline NicRates compute_rates(NicCounters const& prev, - NicCounters const& cur, - double dt_seconds) noexcept -{ - NicRates rates{0.0, 0.0}; - if (dt_seconds <= 0.0) { return rates; } - if (cur.rx_bytes >= prev.rx_bytes) { - auto const delta = cur.rx_bytes - prev.rx_bytes; - rates.rx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; - } - if (cur.tx_bytes >= prev.tx_bytes) { - auto const delta = cur.tx_bytes - prev.tx_bytes; - rates.tx = static_cast(delta) / dt_seconds / constants::bytes_per_mib; - } - return rates; -} - -} // namespace kvikio::experimental From 697dd91d81a68f277b1d4d7be3c7dcad062eba86 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 2 Jul 2026 16:46:57 -0400 Subject: [PATCH 04/16] Cleanup --- cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp | 10 +++++----- cpp/nsys_plugins/nic/nic_monitor.cpp | 4 ++-- cpp/nsys_plugins/nic/nic_monitor.hpp | 4 ++-- cpp/tests/test_nic_monitor.cpp | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp index 7aba670b25..635ac29455 100644 --- a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -34,11 +34,11 @@ #include "nic_monitor.hpp" -using kvikio::experimental::compute_rates; -using kvikio::experimental::default_interfaces; -using kvikio::experimental::NicRates; -using kvikio::experimental::read_nic_counters; -namespace constants = kvikio::experimental::constants; +using kvikio::nsys_plugin::compute_rates; +using kvikio::nsys_plugin::default_interfaces; +using kvikio::nsys_plugin::NicRates; +using kvikio::nsys_plugin::read_nic_counters; +namespace constants = kvikio::nsys_plugin::constants; namespace { diff --git a/cpp/nsys_plugins/nic/nic_monitor.cpp b/cpp/nsys_plugins/nic/nic_monitor.cpp index ef8e9f3172..622dc82b38 100644 --- a/cpp/nsys_plugins/nic/nic_monitor.cpp +++ b/cpp/nsys_plugins/nic/nic_monitor.cpp @@ -17,7 +17,7 @@ #include #include -namespace kvikio::experimental { +namespace kvikio::nsys_plugin { namespace { @@ -129,4 +129,4 @@ NicRates compute_rates(NicCounters const& prev, NicCounters const& cur, double d return rates; } -} // namespace kvikio::experimental +} // namespace kvikio::nsys_plugin diff --git a/cpp/nsys_plugins/nic/nic_monitor.hpp b/cpp/nsys_plugins/nic/nic_monitor.hpp index 990a7cc867..7ad7b93a32 100644 --- a/cpp/nsys_plugins/nic/nic_monitor.hpp +++ b/cpp/nsys_plugins/nic/nic_monitor.hpp @@ -13,7 +13,7 @@ // from sysfs and converts them to rates. It is compiled directly into the plugin and its test; it // is not part of libkvikio and is not an installed public header. -namespace kvikio::experimental { +namespace kvikio::nsys_plugin { namespace constants { inline constexpr double bytes_per_mib = 1024.0 * 1024.0; @@ -84,4 +84,4 @@ struct NicRates { NicCounters const& cur, double dt_seconds) noexcept; -} // namespace kvikio::experimental +} // namespace kvikio::nsys_plugin diff --git a/cpp/tests/test_nic_monitor.cpp b/cpp/tests/test_nic_monitor.cpp index 6786be327f..35addf7da2 100644 --- a/cpp/tests/test_nic_monitor.cpp +++ b/cpp/tests/test_nic_monitor.cpp @@ -9,12 +9,12 @@ #include "nic_monitor.hpp" -using kvikio::experimental::compute_rates; -using kvikio::experimental::default_interfaces; -using kvikio::experimental::iface_is_up; -using kvikio::experimental::NicCounters; -using kvikio::experimental::read_nic_counters; -using kvikio::experimental::constants::bytes_per_mib; +using kvikio::nsys_plugin::compute_rates; +using kvikio::nsys_plugin::default_interfaces; +using kvikio::nsys_plugin::iface_is_up; +using kvikio::nsys_plugin::NicCounters; +using kvikio::nsys_plugin::read_nic_counters; +using kvikio::nsys_plugin::constants::bytes_per_mib; TEST(NicMonitor, ReadCountersIncludesLoopback) { From a9f6ce76c814496702f5393397fb3c5b07e70650 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Thu, 2 Jul 2026 19:00:00 -0400 Subject: [PATCH 05/16] Update --- .../nic/kvikio_nic_nsys_plugin.cpp | 19 ++++---- cpp/nsys_plugins/nic/nic_monitor.cpp | 44 ++++++++++++----- cpp/nsys_plugins/nic/nic_monitor.hpp | 48 ++++++++++++++----- cpp/tests/test_nic_monitor.cpp | 18 +++++-- 4 files changed, 92 insertions(+), 37 deletions(-) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp index 635ac29455..2fb35b6abe 100644 --- a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -18,13 +18,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include @@ -36,8 +36,8 @@ using kvikio::nsys_plugin::compute_rates; using kvikio::nsys_plugin::default_interfaces; +using kvikio::nsys_plugin::NicCounterReader; using kvikio::nsys_plugin::NicRates; -using kvikio::nsys_plugin::read_nic_counters; namespace constants = kvikio::nsys_plugin::constants; namespace { @@ -290,7 +290,10 @@ int main(int argc, char** argv) static_cast(config.interval.count())); using clock = std::chrono::steady_clock; - auto prev = read_nic_counters(); + // The sysfs paths of the selected interfaces are precomputed once; each tick then reads only + // those counters, independent of how many other interfaces exist on the host. + NicCounterReader const reader{interfaces}; + auto prev = reader.read(); auto prev_t = clock::now(); auto next_t = prev_t; @@ -301,16 +304,14 @@ int main(int argc, char** argv) std::this_thread::sleep_until(next_t); auto const now = clock::now(); - auto const cur = read_nic_counters(); + auto cur = reader.read(); double const dt = std::chrono::duration{now - prev_t}.count(); NicRates total{0.0, 0.0}; for (std::size_t i = 0; i < interfaces.size(); ++i) { NicRates rates{0.0, 0.0}; - auto const itc = cur.find(interfaces[i]); - auto const itp = prev.find(interfaces[i]); - if (itc != cur.end() && itp != prev.end()) { - rates = compute_rates(itp->second, itc->second, dt); + if (prev[i].has_value() && cur[i].has_value()) { + rates = compute_rates(prev[i].value(), cur[i].value(), dt); } total.rx += rates.rx; total.tx += rates.tx; @@ -318,7 +319,7 @@ int main(int argc, char** argv) } nvtxCounterSample(domain, total_counter_id, &total, sizeof(total)); - prev = cur; + prev = std::move(cur); prev_t = now; } diff --git a/cpp/nsys_plugins/nic/nic_monitor.cpp b/cpp/nsys_plugins/nic/nic_monitor.cpp index 622dc82b38..5b8fd027e3 100644 --- a/cpp/nsys_plugins/nic/nic_monitor.cpp +++ b/cpp/nsys_plugins/nic/nic_monitor.cpp @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -96,24 +98,40 @@ std::vector default_interfaces() return result; } -std::map read_nic_counters() +NicCounterReader::NicCounterReader(std::vector interfaces) + : _interfaces{std::move(interfaces)} { - std::map result; - std::error_code ec; - std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; - if (ec) { return result; } - for (auto const& entry : it) { - auto const name = entry.path().filename().string(); - auto const stats = entry.path() / "statistics"; - auto const rx = read_u64_file((stats / "rx_bytes").string()); - auto const tx = read_u64_file((stats / "tx_bytes").string()); - // If an interface exposes no parseable byte counters, skip it. - if (!rx.has_value() || !tx.has_value()) { continue; } - result.emplace(name, NicCounters{rx.value(), tx.value()}); + _rx_paths.reserve(_interfaces.size()); + _tx_paths.reserve(_interfaces.size()); + for (auto const& name : _interfaces) { + auto const stats = std::string{constants::sysfs_net} + "/" + name + "/statistics/"; + _rx_paths.push_back(stats + "rx_bytes"); + _tx_paths.push_back(stats + "tx_bytes"); + } +} + +std::vector> NicCounterReader::read() const +{ + std::vector> result; + result.reserve(_interfaces.size()); + for (std::size_t i = 0; i < _interfaces.size(); ++i) { + auto const rx = read_u64_file(_rx_paths[i]); + auto const tx = read_u64_file(_tx_paths[i]); + if (rx.has_value() && tx.has_value()) { + result.emplace_back(NicCounters{rx.value(), tx.value()}); + } else { + // Report an unreadable interface (for example one that was removed) as missing. + result.emplace_back(std::nullopt); + } } return result; } +std::vector const& NicCounterReader::interfaces() const noexcept +{ + return _interfaces; +} + NicRates compute_rates(NicCounters const& prev, NicCounters const& cur, double dt_seconds) noexcept { NicRates rates{0.0, 0.0}; diff --git a/cpp/nsys_plugins/nic/nic_monitor.hpp b/cpp/nsys_plugins/nic/nic_monitor.hpp index 7ad7b93a32..39b085973a 100644 --- a/cpp/nsys_plugins/nic/nic_monitor.hpp +++ b/cpp/nsys_plugins/nic/nic_monitor.hpp @@ -5,12 +5,12 @@ #pragma once #include -#include +#include #include #include // Internal support code for the KvikIO Nsight Systems NIC plugin. It reads network byte counters -// from sysfs and converts them to rates. It is compiled directly into the plugin and its test; it +// from sysfs and converts them to rates. It is compiled directly into the plugin and its test. It // is not part of libkvikio and is not an installed public header. namespace kvikio::nsys_plugin { @@ -60,15 +60,41 @@ struct NicRates { [[nodiscard]] std::vector default_interfaces(); /** - * @brief Read the current cumulative byte counters for every network interface. + * @brief Reads the byte counters of a fixed set of network interfaces. * - * Reads `/sys/class/net//statistics/{rx,tx}_bytes`. Interfaces whose counters cannot be read - * are skipped rather than reported as an error. - * - * @return A map from interface name to its cumulative counters. Empty if `/sys/class/net` is - * unavailable. + * The two sysfs paths per interface are precomputed once at construction, so each `read()` performs + * only the 2N pseudo-file reads, with no directory iteration and no path construction. This keeps + * the per-tick footprint of the sampling loop minimal and independent of how many other interfaces + * exist on the host. */ -[[nodiscard]] std::map read_nic_counters(); +class NicCounterReader { + public: + /** + * @brief Construct a reader over the given interfaces. + * + * @param interfaces Interface names to read, in the order `read()` reports them. + */ + explicit NicCounterReader(std::vector interfaces); + + /** + * @brief Read the current cumulative byte counters of every configured interface. + * + * @return One entry per interface, in constructor order. An entry is std::nullopt when either of + * its counters is unreadable (for example the interface was removed). + */ + [[nodiscard]] std::vector> read() const; + + /** + * @brief The interfaces this reader was constructed with. + */ + [[nodiscard]] std::vector const& interfaces() const noexcept; + + private: + std::vector _interfaces; + // Precomputed `/sys/class/net//statistics/{rx,tx}_bytes` paths, parallel to `_interfaces`. + std::vector _rx_paths; + std::vector _tx_paths; +}; /** * @brief Convert a pair of cumulative counter samples into receive and transmit rates in MiB/s. @@ -77,8 +103,8 @@ struct NicRates { * @param cur The later counter sample. * @param dt_seconds Elapsed wall-clock seconds between the two samples. * @return Rates in MiB/s. A direction whose counter did not advance yields zero for that direction - * (this guards the first sample as well as a counter wrap or reset). Both rates are zero when - * @p dt_seconds is not positive. + * (this guards the first sample as well as a counter wrap or reset). Both rates are zero when @p + * dt_seconds is not positive. */ [[nodiscard]] NicRates compute_rates(NicCounters const& prev, NicCounters const& cur, diff --git a/cpp/tests/test_nic_monitor.cpp b/cpp/tests/test_nic_monitor.cpp index 35addf7da2..60709a0164 100644 --- a/cpp/tests/test_nic_monitor.cpp +++ b/cpp/tests/test_nic_monitor.cpp @@ -12,15 +12,25 @@ using kvikio::nsys_plugin::compute_rates; using kvikio::nsys_plugin::default_interfaces; using kvikio::nsys_plugin::iface_is_up; +using kvikio::nsys_plugin::NicCounterReader; using kvikio::nsys_plugin::NicCounters; -using kvikio::nsys_plugin::read_nic_counters; using kvikio::nsys_plugin::constants::bytes_per_mib; -TEST(NicMonitor, ReadCountersIncludesLoopback) +TEST(NicMonitor, ReaderReadsLoopback) { - auto const counters = read_nic_counters(); + NicCounterReader const reader{{"lo"}}; + auto const counters = reader.read(); + ASSERT_EQ(counters.size(), 1U); // The loopback interface is present on essentially every Linux host. - ASSERT_TRUE(counters.find("lo") != counters.end()); + EXPECT_TRUE(counters[0].has_value()); +} + +TEST(NicMonitor, ReaderReportsMissingInterfaceWithoutValue) +{ + NicCounterReader const reader{{"kvikio_no_such_iface"}}; + auto const counters = reader.read(); + ASSERT_EQ(counters.size(), 1U); + EXPECT_FALSE(counters[0].has_value()); } TEST(NicMonitor, IfaceIsUpAcceptsLoopbackAndRejectsMissing) From 92b28d656db22d045c9b7e257dc7995595193f8d Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Fri, 3 Jul 2026 23:49:28 -0400 Subject: [PATCH 06/16] Update --- .../nic/kvikio_nic_nsys_plugin.cpp | 194 +++++++++++------- cpp/nsys_plugins/nic/nic_monitor.cpp | 15 +- cpp/nsys_plugins/nic/nic_monitor.hpp | 19 +- cpp/nsys_plugins/nic/nsys-plugin.yaml | 12 +- cpp/tests/test_nic_monitor.cpp | 11 +- 5 files changed, 156 insertions(+), 95 deletions(-) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp index 2fb35b6abe..0c792ae026 100644 --- a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -4,11 +4,11 @@ */ // Nsight Systems plugin that samples per-interface network bandwidth and emits it as NVTX counter -// groups, so the bandwidth curve lands on the same timeline as CUDA and KvikIO activity. +// groups, so the bandwidth curve lands on the same timeline with the application activity. // // nsys spawns this executable as a collector for the duration of a profiling session, enabled with -// `--enable=kvikio_nic[,args]` and discovered via `NSYS_PLUGIN_SEARCH_DIRS` (see nsys-plugin.yaml). -// It links only NVTX (header-only) and libdl, so it is standalone and does not depend on libkvikio. +// `--enable=kvikio_nic[,args]` and discovered via `NSYS_PLUGIN_SEARCH_DIRS`. It depends only on +// NVTX headers and libdl, so it is standalone and does not depend on libkvikio. #include #include @@ -45,22 +45,20 @@ namespace { /** * @brief Tag type naming this plugin's NVTX domain. * - * The domain is separate from libkvikio's own domain because the plugin runs as its own process; it - * is created through the nvtx3 C++ registry (`nvtx3::domain::get`) rather than `nvtxDomainCreateA`. + * The domain is separate from libkvikio's own domain because the plugin runs its own process. */ struct kvikio_nic_domain { static constexpr char const* name{"KvikIO NIC"}; }; -// Set from a signal handler to request a clean shutdown. `volatile sig_atomic_t` is the only type -// an async-signal-safe handler may touch. +// Set from a signal handler for a clean shutdown. volatile std::sig_atomic_t g_stop = 0; /** * @brief Parsed command line configuration. */ struct Config { - std::chrono::microseconds interval{50000}; ///< Sampling interval. + std::chrono::microseconds interval{20000}; ///< Sampling interval. std::optional device_filter; ///< If set, monitor interfaces matching this regex. }; @@ -74,7 +72,7 @@ struct Config { { std::fprintf(stderr, "Usage: %s [options]\n" - " -i | --interval Sampling interval in microseconds (default 50000)\n" + " -i | --interval Sampling interval in microseconds (default 20000)\n" " -d | --device Interface name regex (default: up or unknown, non-loopback)\n" " -h | --help Print this help message\n", prog); @@ -84,11 +82,19 @@ struct Config { /** * @brief Parse the command line into a Config, exiting on `--help` or a malformed argument. * - * Accepts `-i N` / `--interval N` / `--interval=N` (and the `-iN` short form) plus the equivalent - * `-d` / `--device` forms, matching how nsys forwards `--enable=kvikio_nic,-d,eth0,-i,50000`. + * Accepts arguments of these forms to be more consistent with nsys built-in sample: + * - `-i N` + * - `--interval N` + * - `--interval=N` + * - `-iN` * - * @param argc Argument count. - * @param argv Argument vector. + * Note that on the nsys command line the plugin arguments follow the plugin name as a comma + * separated list (commas only, no spaces). These are therefore equivalent: + * - `--enable=kvikio_nic,-d,eth0,-i,20000` + * - `--enable=kvikio_nic,--device=eth0,--interval=20000` + * + * @param argc Program argument count. + * @param argv Program argument vector. * @return The parsed configuration. */ Config parse_args(int argc, char** argv) @@ -101,14 +107,20 @@ Config parse_args(int argc, char** argv) if (arg.starts_with("--")) { auto const eq = arg.find('='); if (eq == std::string_view::npos) { - name = arg; + // Example: --interval 20000 + name = arg; // --interval } else { - name = arg.substr(0, eq); - inline_value = arg.substr(eq + 1); + // Example: --interval=20000 + name = arg.substr(0, eq); // --interval + inline_value = arg.substr(eq + 1); // 20000 } } else if (arg.size() >= 2 && arg.front() == '-') { - name = arg.substr(0, 2); - if (arg.size() > 2) { inline_value = arg.substr(2); } + // Example: -i 20000 or -i20000 + name = arg.substr(0, 2); // -i + if (arg.size() > 2) { + // Example: -i20000 + inline_value = arg.substr(2); // 20000 + } } else { std::fprintf(stderr, "kvikio_nic: unexpected argument '%.*s'\n", @@ -164,8 +176,8 @@ Config parse_args(int argc, char** argv) * @brief Choose the interfaces to monitor. * * @param config Parsed configuration. - * @return With no `--device` filter, all up (or unknown) non-loopback interfaces. With a filter, - * all interfaces whose name matches the regex, bypassing the up check so an explicit request is + * @return With no `--device` filter, all up or unknown, non-loopback interfaces. With a filter, all + * interfaces whose name matches the regex, bypassing the up check so an explicit request is * honored. Sorted for a stable order. */ std::vector select_interfaces(Config const& config) @@ -253,19 +265,102 @@ std::uint64_t register_counter(nvtxDomainHandle_t domain, return nvtxCounterRegister(domain, &attr); } +/** + * @brief Samples NIC byte counters at a fixed interval and emits them as NVTX counter groups. + * + * Construction registers one `nic_bandwidth.` counter group per interface plus + * `nic_bandwidth.total`, so no sample can be emitted for an unregistered counter. `run()` then + * differences the counters each tick until the stop flag is raised by the signal handler. + */ +class BandwidthCollector { + public: + /** + * @brief Register the NVTX schema and counter groups for the given interfaces. + * + * @param interfaces Interfaces to monitor. + * @param interval Sampling interval. + */ + BandwidthCollector(std::vector interfaces, std::chrono::microseconds interval) + : _interval{interval}, + _reader{std::move(interfaces)}, + _domain{nvtx3::domain::get()} + { + auto const schema_id = register_rate_schema(_domain); + _counter_ids.reserve(_reader.interfaces().size()); + for (auto const& name : _reader.interfaces()) { + _counter_ids.push_back(register_counter(_domain, "nic_bandwidth." + name, schema_id)); + } + _total_counter_id = register_counter(_domain, "nic_bandwidth.total", schema_id); + } + + /** + * @brief Run the sampling loop until @p stop becomes nonzero. + * + * @param stop Stop flag, set asynchronously by the signal handler. + */ + void run(volatile std::sig_atomic_t const& stop) const + { + auto const& interfaces = _reader.interfaces(); + std::fprintf(stderr, + "kvikio_nic: sampling %zu interface(s) every %lld us\n", + interfaces.size(), + static_cast(_interval.count())); + + using clock = std::chrono::steady_clock; + auto prev = _reader.read(); + auto prev_t = clock::now(); + auto next_t = prev_t; + + while (stop == 0) { + // Absolute deadlines keep the sampling frequency drift-free. A signal does not shorten the + // sleep (libstdc++ restarts it over EINTR), so the stop flag is observed at the next tick. + next_t += _interval; + std::this_thread::sleep_until(next_t); + + auto const now = clock::now(); + auto cur = _reader.read(); + auto const dt = std::chrono::duration{now - prev_t}.count(); + + NicRates total{0.0, 0.0}; + for (std::size_t i = 0; i < interfaces.size(); ++i) { + NicRates rates{0.0, 0.0}; + if (prev[i].has_value() && cur[i].has_value()) { + rates = compute_rates(prev[i].value(), cur[i].value(), dt); + } + total.rx += rates.rx; + total.tx += rates.tx; + nvtxCounterSample(_domain, _counter_ids[i], &rates, sizeof(rates)); + } + nvtxCounterSample(_domain, _total_counter_id, &total, sizeof(total)); + + prev = std::move(cur); + prev_t = now; + } + } + + private: + std::chrono::microseconds _interval; + NicCounterReader _reader; + nvtxDomainHandle_t _domain; + std::vector _counter_ids; + std::uint64_t _total_counter_id{0}; +}; + } // namespace +// C language linkage (extern) to be standard conformant. +// Also internal linkage (static) as a good practice. extern "C" { // nsys stops the collector by sending SIGTERM (then SIGKILL after a grace period). Catch it so the -// loop breaks and the process exits cleanly with code 0 instead of an abnormal termination; SIGINT -// gives the same clean exit for standalone Ctrl-C runs. +// loop breaks and the process exits cleanly with code 0 instead of an abnormal termination. Also +// catch SIGINT to have the same clean exit for Ctrl-C. static void kvikio_nic_handle_signal(int /*signum*/) { g_stop = 1; } } int main(int argc, char** argv) { - auto const config = parse_args(argc, argv); - auto const interfaces = select_interfaces(config); + auto const config = parse_args(argc, argv); + auto interfaces = select_interfaces(config); if (interfaces.empty()) { std::fprintf(stderr, "kvikio_nic: no matching network interfaces to monitor.\n"); return 1; @@ -274,54 +369,7 @@ int main(int argc, char** argv) std::signal(SIGTERM, kvikio_nic_handle_signal); std::signal(SIGINT, kvikio_nic_handle_signal); - nvtxDomainHandle_t domain = nvtx3::domain::get(); - auto const schema_id = register_rate_schema(domain); - - std::vector counter_ids; - counter_ids.reserve(interfaces.size()); - for (auto const& name : interfaces) { - counter_ids.push_back(register_counter(domain, "nic_bandwidth." + name, schema_id)); - } - auto const total_counter_id = register_counter(domain, "nic_bandwidth.total", schema_id); - - std::fprintf(stderr, - "kvikio_nic: sampling %zu interface(s) every %lld us\n", - interfaces.size(), - static_cast(config.interval.count())); - - using clock = std::chrono::steady_clock; - // The sysfs paths of the selected interfaces are precomputed once; each tick then reads only - // those counters, independent of how many other interfaces exist on the host. - NicCounterReader const reader{interfaces}; - auto prev = reader.read(); - auto prev_t = clock::now(); - auto next_t = prev_t; - - while (g_stop == 0) { - // Absolute deadlines keep the sampling frequency drift-free. A signal does not shorten the - // sleep (libstdc++ restarts it over EINTR), so g_stop is observed at the next tick. - next_t += config.interval; - std::this_thread::sleep_until(next_t); - - auto const now = clock::now(); - auto cur = reader.read(); - double const dt = std::chrono::duration{now - prev_t}.count(); - - NicRates total{0.0, 0.0}; - for (std::size_t i = 0; i < interfaces.size(); ++i) { - NicRates rates{0.0, 0.0}; - if (prev[i].has_value() && cur[i].has_value()) { - rates = compute_rates(prev[i].value(), cur[i].value(), dt); - } - total.rx += rates.rx; - total.tx += rates.tx; - nvtxCounterSample(domain, counter_ids[i], &rates, sizeof(rates)); - } - nvtxCounterSample(domain, total_counter_id, &total, sizeof(total)); - - prev = std::move(cur); - prev_t = now; - } - + BandwidthCollector const collector{std::move(interfaces), config.interval}; + collector.run(g_stop); return 0; } diff --git a/cpp/nsys_plugins/nic/nic_monitor.cpp b/cpp/nsys_plugins/nic/nic_monitor.cpp index 5b8fd027e3..9a0507d727 100644 --- a/cpp/nsys_plugins/nic/nic_monitor.cpp +++ b/cpp/nsys_plugins/nic/nic_monitor.cpp @@ -60,6 +60,10 @@ namespace { // newline. [[maybe_unused]] auto const [_, ec] = std::from_chars(content->data(), content->data() + content->size(), value); + // Note: ec type is std::errc, which is an enum class and has no bool operator. In C++26, the + // return value of type `std::from_chars_result` will have bool operator. + // TODO: In C++26, remove structured binding and use the result's bool operator to check success + // directly. if (ec != std::errc{}) { return std::nullopt; } return value; } @@ -69,16 +73,17 @@ namespace { bool iface_is_up(std::string const& name) { auto const path = std::string{constants::sysfs_net} + "/" + name + "/operstate"; + // The longest operstate value is "lowerlayerdown" (14 chars + newline), so 16 bytes is ample. std::array buf; auto const content = read_pseudo_file(path.c_str(), buf); if (!content.has_value()) { return false; } std::string_view state = content.value(); + // Remove any trailing newline or space character while (!state.empty() && (state.back() == '\n' || state.back() == ' ')) { state.remove_suffix(1); } - // Some working NICs report "unknown" (loopback or virtual NICs), so this state should be - // accepted. The other states (down, notpresent, lowerlayerdown, testing, dormant) should be - // excluded. + // Some working NICs report "unknown" (virtual NICs), so this state should be accepted. The other + // states (down, notpresent, lowerlayerdown, testing, dormant) should be excluded. return state == "up" || state == "unknown"; } @@ -90,6 +95,7 @@ std::vector default_interfaces() if (ec) { return result; } for (auto const& entry : it) { auto name = entry.path().filename().string(); + // Ignore loopback interface if (name == "lo") { continue; } if (!iface_is_up(name)) { continue; } result.push_back(std::move(name)); @@ -120,7 +126,8 @@ std::vector> NicCounterReader::read() const if (rx.has_value() && tx.has_value()) { result.emplace_back(NicCounters{rx.value(), tx.value()}); } else { - // Report an unreadable interface (for example one that was removed) as missing. + // Report an unreadable interface (for example one that was removed or never existed) as + // missing. result.emplace_back(std::nullopt); } } diff --git a/cpp/nsys_plugins/nic/nic_monitor.hpp b/cpp/nsys_plugins/nic/nic_monitor.hpp index 39b085973a..e23ce0cd9d 100644 --- a/cpp/nsys_plugins/nic/nic_monitor.hpp +++ b/cpp/nsys_plugins/nic/nic_monitor.hpp @@ -9,9 +9,9 @@ #include #include -// Internal support code for the KvikIO Nsight Systems NIC plugin. It reads network byte counters -// from sysfs and converts them to rates. It is compiled directly into the plugin and its test. It -// is not part of libkvikio and is not an installed public header. +// Internal support code for the KvikIO Nsight Systems NIC plugin. Reads network byte counters from +// sysfs and converts them to rates. Compiled directly into the plugin and its test. Not part of +// libkvikio binary or public header. namespace kvikio::nsys_plugin { @@ -34,9 +34,7 @@ struct NicCounters { /** * @brief One bandwidth sample for a single interface: the receive and transmit rates. * - * The field names are deliberately unit-neutral. `compute_rates` produces the values in MiB/s, but - * the unit is not baked into the type; a consumer attaches it separately (for example as NVTX - * counter semantics), so switching units later does not ripple into these names. + * The field names are deliberately unit-neutral. `compute_rates` produces the values in MiB/s. */ struct NicRates { double rx; ///< Receive rate. @@ -62,16 +60,15 @@ struct NicRates { /** * @brief Reads the byte counters of a fixed set of network interfaces. * - * The two sysfs paths per interface are precomputed once at construction, so each `read()` performs - * only the 2N pseudo-file reads, with no directory iteration and no path construction. This keeps - * the per-tick footprint of the sampling loop minimal and independent of how many other interfaces - * exist on the host. + * The two sysfs paths (rx and tx) per interface are precomputed once at construction. */ class NicCounterReader { public: /** * @brief Construct a reader over the given interfaces. * + * The names are taken as-is, without filtering or validation, so loopback ("lo") is allowed. + * * @param interfaces Interface names to read, in the order `read()` reports them. */ explicit NicCounterReader(std::vector interfaces); @@ -80,7 +77,7 @@ class NicCounterReader { * @brief Read the current cumulative byte counters of every configured interface. * * @return One entry per interface, in constructor order. An entry is std::nullopt when either of - * its counters is unreadable (for example the interface was removed). + * its counters is unreadable (for example the interface was removed or never existed). */ [[nodiscard]] std::vector> read() const; diff --git a/cpp/nsys_plugins/nic/nsys-plugin.yaml b/cpp/nsys_plugins/nic/nsys-plugin.yaml index 16e16ea1c7..9da7513e1a 100644 --- a/cpp/nsys_plugins/nic/nsys-plugin.yaml +++ b/cpp/nsys_plugins/nic/nsys-plugin.yaml @@ -4,14 +4,22 @@ ExecutablePath: kvikio_nic_nsys_plugin Description: Per-interface RX/TX network bandwidth from /sys/class/net, as NVTX counter groups. ExtendedDescription: | Command line options: - -i | --interval Sampling interval in microseconds (default 50000) + -i | --interval Sampling interval in microseconds (default 20000) -d | --device Interface name regex (default: up or unknown, non-loopback) -h | --help Print this help message + Plugin arguments follow the plugin name as a comma separated list (commas only, no + spaces); each comma separated token is passed verbatim as one argument. These are + equivalent: + --enable=kvikio_nic,-d,eth0,-i,20000 + --enable=kvikio_nic,--device=eth0,--interval=20000 + A literal comma inside an argument (for example in a --device regex) must be escaped + as '\,'. + With no --device filter, all up (or unknown) non-loopback interfaces are monitored. A --device regex selects interfaces by name and bypasses the up check, so an explicit request is always honored. ExtendedDescriptionForGui: | Options: - -i | --interval Sampling interval in microseconds (default 50000) + -i | --interval Sampling interval in microseconds (default 20000) -d | --device Interface name regex (default: up or unknown, non-loopback) diff --git a/cpp/tests/test_nic_monitor.cpp b/cpp/tests/test_nic_monitor.cpp index 60709a0164..d298a88260 100644 --- a/cpp/tests/test_nic_monitor.cpp +++ b/cpp/tests/test_nic_monitor.cpp @@ -27,7 +27,7 @@ TEST(NicMonitor, ReaderReadsLoopback) TEST(NicMonitor, ReaderReportsMissingInterfaceWithoutValue) { - NicCounterReader const reader{{"kvikio_no_such_iface"}}; + NicCounterReader const reader{{"nonexistent_iface"}}; auto const counters = reader.read(); ASSERT_EQ(counters.size(), 1U); EXPECT_FALSE(counters[0].has_value()); @@ -38,7 +38,7 @@ TEST(NicMonitor, IfaceIsUpAcceptsLoopbackAndRejectsMissing) // Loopback reports operstate "unknown", which the up/unknown policy accepts. EXPECT_TRUE(iface_is_up("lo")); // A name that cannot be read must be treated as not up rather than accepted. - EXPECT_FALSE(iface_is_up("kvikio_no_such_iface")); + EXPECT_FALSE(iface_is_up("nonexistent_iface")); } TEST(NicMonitor, DefaultInterfacesExcludeLoopback) @@ -54,7 +54,8 @@ TEST(NicMonitor, ComputeRatesConvertsDeltasToMiBps) NicCounters const cur{static_cast(bytes_per_mib) * 2, static_cast(bytes_per_mib) * 6}; auto const rates = compute_rates(prev, cur, 2.0); - // 2 MiB received over 2 s -> 1 MiB/s; 6 MiB transmitted over 2 s -> 3 MiB/s. + // 2 MiB received over 2 s -> 1 MiB/s + // 6 MiB transmitted over 2 s -> 3 MiB/s. EXPECT_DOUBLE_EQ(rates.rx, 1.0); EXPECT_DOUBLE_EQ(rates.tx, 3.0); } @@ -63,11 +64,11 @@ TEST(NicMonitor, ComputeRatesGuardsWrapAndNonPositiveInterval) { NicCounters const high{1000, 1000}; NicCounters const low{10, 10}; - // A counter that appears to go backwards (wrap or reset) yields zero for that direction. + // A counter that appears to go backwards (wrap or reset) yields zero. auto const wrapped = compute_rates(high, low, 1.0); EXPECT_DOUBLE_EQ(wrapped.rx, 0.0); EXPECT_DOUBLE_EQ(wrapped.tx, 0.0); - // A non-positive elapsed time yields zero rather than dividing by zero. + // A non-positive elapsed time yields zero. auto const no_time = compute_rates(low, high, 0.0); EXPECT_DOUBLE_EQ(no_time.rx, 0.0); EXPECT_DOUBLE_EQ(no_time.tx, 0.0); From 49b83eb593f25a68c7e31dfd7e0535cb687d6c5d Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 09:45:59 -0400 Subject: [PATCH 07/16] Improve impl --- .../nic/kvikio_nic_nsys_plugin.cpp | 93 ++++++++++--------- 1 file changed, 51 insertions(+), 42 deletions(-) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp index 0c792ae026..d03e834bcc 100644 --- a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -11,6 +11,7 @@ // NVTX headers and libdl, so it is standalone and does not depend on libkvikio. #include +#include #include #include #include @@ -38,7 +39,6 @@ using kvikio::nsys_plugin::compute_rates; using kvikio::nsys_plugin::default_interfaces; using kvikio::nsys_plugin::NicCounterReader; using kvikio::nsys_plugin::NicRates; -namespace constants = kvikio::nsys_plugin::constants; namespace { @@ -54,6 +54,13 @@ struct kvikio_nic_domain { // Set from a signal handler for a clean shutdown. volatile std::sig_atomic_t g_stop = 0; +namespace constants { +// Exit codes +constexpr int exit_success = 0; +constexpr int exit_no_interfaces = 1; +constexpr int exit_usage_error = 2; +} // namespace constants + /** * @brief Parsed command line configuration. */ @@ -126,7 +133,7 @@ Config parse_args(int argc, char** argv) "kvikio_nic: unexpected argument '%.*s'\n", static_cast(arg.size()), arg.data()); - print_help_and_exit(argv[0], 2); + print_help_and_exit(argv[0], constants::exit_usage_error); } // Take this option's value from the inline form (`--interval=N`, `-iN`) or the next argument. @@ -137,11 +144,11 @@ Config parse_args(int argc, char** argv) "kvikio_nic: option '%.*s' requires a value\n", static_cast(name.size()), name.data()); - print_help_and_exit(argv[0], 2); + print_help_and_exit(argv[0], constants::exit_usage_error); }; if (name == "-h" || name == "--help") { - print_help_and_exit(argv[0], 0); + print_help_and_exit(argv[0], constants::exit_success); } else if (name == "-i" || name == "--interval") { auto const value = take_value(); long long parsed = 0; @@ -151,7 +158,7 @@ Config parse_args(int argc, char** argv) std::fprintf(stderr, "kvikio_nic: invalid interval '%s' (expected a positive integer)\n", value.c_str()); - print_help_and_exit(argv[0], 2); + print_help_and_exit(argv[0], constants::exit_usage_error); } config.interval = std::chrono::microseconds{parsed}; } else if (name == "-d" || name == "--device") { @@ -161,12 +168,12 @@ Config parse_args(int argc, char** argv) } catch (std::regex_error const& e) { std::fprintf( stderr, "kvikio_nic: invalid device regex '%s' (%s)\n", value.c_str(), e.what()); - print_help_and_exit(argv[0], 2); + print_help_and_exit(argv[0], constants::exit_usage_error); } } else { std::fprintf( stderr, "kvikio_nic: unknown option '%.*s'\n", static_cast(name.size()), name.data()); - print_help_and_exit(argv[0], 2); + print_help_and_exit(argv[0], constants::exit_usage_error); } } return config; @@ -185,7 +192,8 @@ std::vector select_interfaces(Config const& config) if (!config.device_filter.has_value()) { return default_interfaces(); } std::vector result; std::error_code ec; - std::filesystem::directory_iterator it{std::filesystem::path{constants::sysfs_net}, ec}; + std::filesystem::directory_iterator it{ + std::filesystem::path{kvikio::nsys_plugin::constants::sysfs_net}, ec}; if (ec) { return result; } for (auto const& entry : it) { auto name = entry.path().filename().string(); @@ -196,12 +204,9 @@ std::vector select_interfaces(Config const& config) } /** - * @brief Build the counter semantics that carry the rate unit for a whole counter group. - * - * Keeping the unit here (rather than in the schema field names) means the identifiers stay - * unit-neutral and a future unit change touches only this value. + * @brief Build the counter semantics that specify the rate unit. * - * @return A populated counter-semantics extension describing MiB/s values. + * @return A populated counter-semantics object that specifies the rate unit. */ nvtxSemanticsCounter_t make_rate_semantics() { @@ -219,24 +224,29 @@ nvtxSemanticsCounter_t make_rate_semantics() } /** - * @brief Register the unit-neutral {rx, tx} payload schema shared by every counter group. + * @brief Register the {rx, tx} payload schema. * * @param domain NVTX domain handle. * @return The schema id to pass as nvtxCounterAttr_t::schemaId. */ std::uint64_t register_rate_schema(nvtxDomainHandle_t domain) { - nvtxPayloadSchemaEntry_t const entries[] = { - {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "rx"}, - {.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "tx"}, + std::array const entries = { + nvtxPayloadSchemaEntry_t{ + .type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "rx", .description = "Receive rate"}, + nvtxPayloadSchemaEntry_t{ + .type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "tx", .description = "Transmit rate"}, }; nvtxPayloadSchemaAttr_t attr{}; - attr.fieldMask = NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_TYPE | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_ENTRIES | + attr.fieldMask = NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NAME | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_TYPE | + NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_FLAGS | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_ENTRIES | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NUM_ENTRIES | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_STATIC_SIZE; + attr.name = "NIC bandwidth"; attr.type = NVTX_PAYLOAD_SCHEMA_TYPE_STATIC; - attr.entries = entries; - attr.numEntries = 2; + attr.flags = NVTX_PAYLOAD_SCHEMA_FLAG_COUNTER_GROUP; + attr.entries = entries.data(); + attr.numEntries = entries.size(); attr.payloadStaticSize = sizeof(NicRates); return nvtxPayloadSchemaRegister(domain, &attr); } @@ -253,8 +263,7 @@ std::uint64_t register_counter(nvtxDomainHandle_t domain, std::string const& name, std::uint64_t schema_id) { - // The semantics apply to the whole group and must outlive this call, so keep them static. - static nvtxSemanticsCounter_t const rate_semantics = make_rate_semantics(); + nvtxSemanticsCounter_t const rate_semantics = make_rate_semantics(); nvtxCounterAttr_t attr{}; attr.structSize = sizeof(nvtxCounterAttr_t); attr.schemaId = schema_id; @@ -269,8 +278,8 @@ std::uint64_t register_counter(nvtxDomainHandle_t domain, * @brief Samples NIC byte counters at a fixed interval and emits them as NVTX counter groups. * * Construction registers one `nic_bandwidth.` counter group per interface plus - * `nic_bandwidth.total`, so no sample can be emitted for an unregistered counter. `run()` then - * differences the counters each tick until the stop flag is raised by the signal handler. + * `nic_bandwidth.total`. `run()` then differences the counters each tick until the stop flag is + * raised by the signal handler. */ class BandwidthCollector { public: @@ -306,26 +315,26 @@ class BandwidthCollector { interfaces.size(), static_cast(_interval.count())); - using clock = std::chrono::steady_clock; - auto prev = _reader.read(); - auto prev_t = clock::now(); - auto next_t = prev_t; + using clock = std::chrono::steady_clock; + auto prev_counters = _reader.read(); + auto prev_time = clock::now(); + auto next_deadline = prev_time; while (stop == 0) { - // Absolute deadlines keep the sampling frequency drift-free. A signal does not shorten the - // sleep (libstdc++ restarts it over EINTR), so the stop flag is observed at the next tick. - next_t += _interval; - std::this_thread::sleep_until(next_t); + // Use sleep_until (instead of sleep_for) to minimize sampling frequency drift. + // A signal does not interrupt the sleep. + next_deadline += _interval; + std::this_thread::sleep_until(next_deadline); - auto const now = clock::now(); - auto cur = _reader.read(); - auto const dt = std::chrono::duration{now - prev_t}.count(); + auto const now = clock::now(); + auto cur_counters = _reader.read(); + auto const dt = std::chrono::duration{now - prev_time}.count(); NicRates total{0.0, 0.0}; for (std::size_t i = 0; i < interfaces.size(); ++i) { NicRates rates{0.0, 0.0}; - if (prev[i].has_value() && cur[i].has_value()) { - rates = compute_rates(prev[i].value(), cur[i].value(), dt); + if (prev_counters[i].has_value() && cur_counters[i].has_value()) { + rates = compute_rates(prev_counters[i].value(), cur_counters[i].value(), dt); } total.rx += rates.rx; total.tx += rates.tx; @@ -333,8 +342,8 @@ class BandwidthCollector { } nvtxCounterSample(_domain, _total_counter_id, &total, sizeof(total)); - prev = std::move(cur); - prev_t = now; + prev_counters = std::move(cur_counters); + prev_time = now; } } @@ -350,10 +359,10 @@ class BandwidthCollector { // C language linkage (extern) to be standard conformant. // Also internal linkage (static) as a good practice. -extern "C" { // nsys stops the collector by sending SIGTERM (then SIGKILL after a grace period). Catch it so the // loop breaks and the process exits cleanly with code 0 instead of an abnormal termination. Also // catch SIGINT to have the same clean exit for Ctrl-C. +extern "C" { static void kvikio_nic_handle_signal(int /*signum*/) { g_stop = 1; } } @@ -363,7 +372,7 @@ int main(int argc, char** argv) auto interfaces = select_interfaces(config); if (interfaces.empty()) { std::fprintf(stderr, "kvikio_nic: no matching network interfaces to monitor.\n"); - return 1; + return constants::exit_no_interfaces; } std::signal(SIGTERM, kvikio_nic_handle_signal); @@ -371,5 +380,5 @@ int main(int argc, char** argv) BandwidthCollector const collector{std::move(interfaces), config.interval}; collector.run(g_stop); - return 0; + return constants::exit_success; } From b426750047fc08a20b9c8d48efe0166492f2e14d Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 10:03:35 -0400 Subject: [PATCH 08/16] Further improve impl --- .../nic/kvikio_nic_nsys_plugin.cpp | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp index d03e834bcc..b5bdf0ec74 100644 --- a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include @@ -59,6 +61,18 @@ namespace constants { constexpr int exit_success = 0; constexpr int exit_no_interfaces = 1; constexpr int exit_usage_error = 2; + +constexpr nvtxSemanticsCounter_t rate_semantics{ + .header = {.structSize = sizeof(nvtxSemanticsCounter_t), + .semanticId = NVTX_SEMANTIC_ID_COUNTERS_V1, + .version = NVTX_COUNTER_SEMANTIC_VERSION, + .next = nullptr}, + .flags = NVTX_COUNTER_FLAGS_NONE, + .unit = "MiB/s", + .unitScaleNumerator = 1, + .unitScaleDenominator = 1, + .limitType = NVTX_COUNTER_LIMIT_UNDEFINED, +}; } // namespace constants /** @@ -203,26 +217,6 @@ std::vector select_interfaces(Config const& config) return result; } -/** - * @brief Build the counter semantics that specify the rate unit. - * - * @return A populated counter-semantics object that specifies the rate unit. - */ -nvtxSemanticsCounter_t make_rate_semantics() -{ - nvtxSemanticsCounter_t sem{}; - sem.header.structSize = sizeof(nvtxSemanticsCounter_t); - sem.header.semanticId = NVTX_SEMANTIC_ID_COUNTERS_V1; - sem.header.version = NVTX_COUNTER_SEMANTIC_VERSION; - sem.header.next = nullptr; - sem.flags = NVTX_COUNTER_FLAGS_NONE; - sem.unit = "MiB/s"; - sem.unitScaleNumerator = 1; - sem.unitScaleDenominator = 1; - sem.limitType = NVTX_COUNTER_LIMIT_UNDEFINED; - return sem; -} - /** * @brief Register the {rx, tx} payload schema. * @@ -231,11 +225,16 @@ nvtxSemanticsCounter_t make_rate_semantics() */ std::uint64_t register_rate_schema(nvtxDomainHandle_t domain) { + static_assert(std::is_standard_layout_v); std::array const entries = { - nvtxPayloadSchemaEntry_t{ - .type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "rx", .description = "Receive rate"}, - nvtxPayloadSchemaEntry_t{ - .type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, .name = "tx", .description = "Transmit rate"}, + nvtxPayloadSchemaEntry_t{.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, + .name = "rx", + .description = "Receive rate", + .offset = offsetof(NicRates, rx)}, + nvtxPayloadSchemaEntry_t{.type = NVTX_PAYLOAD_ENTRY_TYPE_DOUBLE, + .name = "tx", + .description = "Transmit rate", + .offset = offsetof(NicRates, tx)}, }; nvtxPayloadSchemaAttr_t attr{}; attr.fieldMask = NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_NAME | NVTX_PAYLOAD_SCHEMA_ATTR_FIELD_TYPE | @@ -263,13 +262,12 @@ std::uint64_t register_counter(nvtxDomainHandle_t domain, std::string const& name, std::uint64_t schema_id) { - nvtxSemanticsCounter_t const rate_semantics = make_rate_semantics(); nvtxCounterAttr_t attr{}; attr.structSize = sizeof(nvtxCounterAttr_t); attr.schemaId = schema_id; attr.name = name.c_str(); attr.scopeId = NVTX_SCOPE_CURRENT_VM; - attr.semantics = &rate_semantics.header; + attr.semantics = &constants::rate_semantics.header; attr.counterId = NVTX_COUNTER_ID_NONE; return nvtxCounterRegister(domain, &attr); } From 163cfaad449164554fb1b43d8300a323e1913401 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 10:19:33 -0400 Subject: [PATCH 09/16] Improve cmake scripts --- cpp/CMakeLists.txt | 8 +-- cpp/tests/CMakeLists.txt | 92 ++++++++++++++++++++++++--------- python/libkvikio/CMakeLists.txt | 4 +- 3 files changed, 71 insertions(+), 33 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4be0fc2e3a..32a5aed3c6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -52,13 +52,7 @@ option(KvikIO_BUILD_BENCHMARKS "Configure CMake to build benchmarks" OFF) option(KvikIO_BUILD_EXAMPLES "Configure CMake to build examples" ON) option(KvikIO_BUILD_TESTS "Configure CMake to build tests" ON) option(KvikIO_REMOTE_SUPPORT "Configure CMake to build with remote IO support" ON) - -# The Nsight Systems NIC plugin reads /sys/class/net, so it is only meaningful on Linux. -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - option(KvikIO_BUILD_NSYS_PLUGIN "Configure CMake to build the Nsight Systems NIC plugin" ON) -else() - option(KvikIO_BUILD_NSYS_PLUGIN "Configure CMake to build the Nsight Systems NIC plugin" OFF) -endif() +option(KvikIO_BUILD_NSYS_PLUGIN "Configure CMake to build the Nsight Systems NIC plugin" ON) # ################################################################################################## # * conda environment ------------------------------------------------------------------------------ diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 6226118c26..d1c7685008 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -62,6 +62,73 @@ function(kvikio_add_test) ) endfunction() +#[=======================================================================[.rst: +kvikio_add_nsys_plugin_test +--------------------------- + +Create a test for the support code of a KvikIO Nsight Systems plugin. + +Plugin support code (under ``cpp/nsys_plugins/``) deliberately does not depend on +libkvikio or CUDA, so this test links only GoogleTest and compiles the plugin sources directly, +rather than going through ``kvikio_add_test``. + +.. code-block:: cmake + + kvikio_add_nsys_plugin_test(NAME SOURCES PLUGIN_SOURCES ) + + ``NAME`` + Test name. Single-value argument. + + ``SOURCES`` + List of test source files, relative to ``cpp/tests``. Multi-value argument. + + ``PLUGIN_SOURCES`` + List of plugin source files, relative to ``cpp/nsys_plugins`` (for example + ``nic/nic_monitor.cpp``). The parent directory of each file is added as an include + directory. Multi-value argument. +#]=======================================================================] +function(kvikio_add_nsys_plugin_test) + cmake_parse_arguments( + _KVIKIO # prefix + "" # optional + "NAME" # single value + "SOURCES;PLUGIN_SOURCES" # multi-value + ${ARGN} + ) + + if(DEFINED _KVIKIO_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unknown argument: ${_KVIKIO_UNPARSED_ARGUMENTS}") + endif() + + set(_plugin_root "${KvikIO_SOURCE_DIR}/nsys_plugins") + set(_plugin_sources "") + set(_plugin_include_dirs "") + foreach(_src IN LISTS _KVIKIO_PLUGIN_SOURCES) + list(APPEND _plugin_sources "${_plugin_root}/${_src}") + cmake_path(GET _src PARENT_PATH _src_dir) + list(APPEND _plugin_include_dirs "$") + endforeach() + list(REMOVE_DUPLICATES _plugin_include_dirs) + + add_executable(${_KVIKIO_NAME} ${_KVIKIO_SOURCES} ${_plugin_sources}) + set_target_properties( + ${_KVIKIO_NAME} + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED ON + ) + target_include_directories(${_KVIKIO_NAME} PRIVATE ${_plugin_include_dirs}) + target_link_libraries( + ${_KVIKIO_NAME} PRIVATE GTest::gmock GTest::gmock_main GTest::gtest GTest::gtest_main + ) + + rapids_test_add( + NAME ${_KVIKIO_NAME} + COMMAND ${_KVIKIO_NAME} + INSTALL_COMPONENT_SET testing + ) +endfunction() + kvikio_add_test(NAME BASIC_IO_TEST SOURCES test_basic_io.cpp utils/env.cpp) kvikio_add_test(NAME BOUNCE_BUFFER_TEST SOURCES test_bounce_buffer.cpp) @@ -82,29 +149,8 @@ kvikio_add_test(NAME MMAP_TEST SOURCES test_mmap.cpp) kvikio_add_test(NAME CONCURRENT_REQUEST_LIMITER_TEST SOURCES test_concurrent_request_limiter.cpp) -# The NIC monitor core lives with the Nsight Systems plugin (nsys_plugins/nic) and does not depend -# on libkvikio. Its test compiles that core directly and links only GoogleTest, rather than going -# through kvikio_add_test (which links kvikio and CUDA). -add_executable( - NIC_MONITOR_TEST test_nic_monitor.cpp ${KvikIO_SOURCE_DIR}/nsys_plugins/nic/nic_monitor.cpp -) -set_target_properties( - NIC_MONITOR_TEST - PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" - CXX_STANDARD 20 - CXX_STANDARD_REQUIRED ON -) -target_include_directories( - NIC_MONITOR_TEST PRIVATE "$" -) -target_link_libraries( - NIC_MONITOR_TEST PRIVATE GTest::gmock GTest::gmock_main GTest::gtest GTest::gtest_main -) -rapids_test_add( - NAME NIC_MONITOR_TEST - COMMAND NIC_MONITOR_TEST - GPUS 1 - INSTALL_COMPONENT_SET testing +kvikio_add_nsys_plugin_test( + NAME NIC_MONITOR_TEST SOURCES test_nic_monitor.cpp PLUGIN_SOURCES nic/nic_monitor.cpp ) if(KvikIO_REMOTE_SUPPORT) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index bc03dd58cf..c794a3be88 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -32,9 +32,7 @@ unset(kvikio_FOUND) set(KvikIO_BUILD_BENCHMARKS OFF) set(KvikIO_BUILD_EXAMPLES OFF) set(KvikIO_BUILD_TESTS OFF) -# Build and ship the Nsight Systems NIC plugin in the libkvikio wheel. INSTALL_DEFAULT makes its -# install rules part of the default install so scikit-build-core packages them without needing to -# select a component (the base conda package keeps them excluded instead). +# Build and ship the Nsight Systems NIC plugin in the libkvikio wheel. set(KvikIO_BUILD_NSYS_PLUGIN ON) set(KvikIO_NSYS_PLUGIN_INSTALL_DEFAULT ON) From 586bfdb505f36251bdb33dba1751f1344c4b8929 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 10:24:07 -0400 Subject: [PATCH 10/16] Cleanup --- conda/recipes/libkvikio/recipe.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/conda/recipes/libkvikio/recipe.yaml b/conda/recipes/libkvikio/recipe.yaml index f731c46f3d..c46afffab4 100644 --- a/conda/recipes/libkvikio/recipe.yaml +++ b/conda/recipes/libkvikio/recipe.yaml @@ -152,9 +152,6 @@ outputs: license: ${{ load_from_file("python/libkvikio/pyproject.toml").project.license }} summary: libkvikio tests - # The plugin is a standalone executable (NVTX header-only plus libdl); it links neither libkvikio - # nor CUDA. The requirements below mirror the libkvikio-tests output so the same validated build - # environment and libstdc++ runtime baseline apply; they can be slimmed once separately verified. - package: name: libkvikio-nsys-plugin version: ${{ version }} From 7c035dcd9b192a671548ba3e589419bf803bc986 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 10:51:39 -0400 Subject: [PATCH 11/16] Cleanup --- conda/recipes/libkvikio/recipe.yaml | 35 ++--------------------------- cpp/nsys_plugins/nic/CMakeLists.txt | 16 +++---------- python/libkvikio/CMakeLists.txt | 2 -- 3 files changed, 5 insertions(+), 48 deletions(-) diff --git a/conda/recipes/libkvikio/recipe.yaml b/conda/recipes/libkvikio/recipe.yaml index c46afffab4..f6d2444ab0 100644 --- a/conda/recipes/libkvikio/recipe.yaml +++ b/conda/recipes/libkvikio/recipe.yaml @@ -112,6 +112,8 @@ outputs: tests: - script: - test -f $PREFIX/include/kvikio/file_handle.hpp + - test -x $PREFIX/share/kvikio/nsys-plugins/kvikio_nic/kvikio_nic_nsys_plugin + - test -f $PREFIX/share/kvikio/nsys-plugins/kvikio_nic/nsys-plugin.yaml about: homepage: ${{ load_from_file("python/libkvikio/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libkvikio/pyproject.toml").project.license }} @@ -151,36 +153,3 @@ outputs: homepage: ${{ load_from_file("python/libkvikio/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libkvikio/pyproject.toml").project.license }} summary: libkvikio tests - - - package: - name: libkvikio-nsys-plugin - version: ${{ version }} - build: - string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} - dynamic_linking: - overlinking_behavior: "error" - script: - content: | - cmake --install cpp/build --component nsys_plugin - requirements: - build: - - cmake ${{ cmake_version }} - - ${{ compiler("c") }} - host: - - ${{ pin_subpackage("libkvikio", exact=True) }} - - cuda-version =${{ cuda_version }} - run: - - if: cuda_version >= "13.0" - then: - - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - else: - - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="12.2.0a0") }} - ignore_run_exports: - by_name: - - cuda-version - - libnuma - - libcufile - about: - homepage: ${{ load_from_file("python/libkvikio/pyproject.toml").project.urls.Homepage }} - license: ${{ load_from_file("python/libkvikio/pyproject.toml").project.license }} - summary: KvikIO Nsight Systems NIC bandwidth plugin diff --git a/cpp/nsys_plugins/nic/CMakeLists.txt b/cpp/nsys_plugins/nic/CMakeLists.txt index 84b69a4941..4c32608dd2 100644 --- a/cpp/nsys_plugins/nic/CMakeLists.txt +++ b/cpp/nsys_plugins/nic/CMakeLists.txt @@ -31,26 +31,16 @@ endif() # Install the binary and its manifest side by side into a directory named after the plugin, matching # how nsys discovers plugins under NSYS_PLUGIN_SEARCH_DIRS. # -# By default the install rules are EXCLUDE_FROM_ALL, so a plain `cmake --install` (the base conda -# libkvikio package) skips them and the plugin ships only via `cmake --install --component -# nsys_plugin` (the dedicated conda output). The libkvikio wheel sets -# KvikIO_NSYS_PLUGIN_INSTALL_DEFAULT so the plugin is part of the default install and is packaged -# without needing component selection in scikit-build-core. -if(KvikIO_NSYS_PLUGIN_INSTALL_DEFAULT) - set(_kvikio_nsys_plugin_exclude "") -else() - set(_kvikio_nsys_plugin_exclude "EXCLUDE_FROM_ALL") -endif() - +# The plugin is a user-facing tool, so it is part of the default install: both the base conda +# libkvikio package (plain `cmake --install`) and the libkvikio wheel ship it. The dedicated +# component remains available for targeted `cmake --install --component nsys_plugin` layouts. install( TARGETS kvikio_nic_nsys_plugin COMPONENT nsys_plugin DESTINATION ${NSYS_PLUGIN_INSTALL_DIR} - ${_kvikio_nsys_plugin_exclude} ) install( FILES nsys-plugin.yaml COMPONENT nsys_plugin DESTINATION ${NSYS_PLUGIN_INSTALL_DIR} - ${_kvikio_nsys_plugin_exclude} ) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index c794a3be88..cb66c48212 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -32,8 +32,6 @@ unset(kvikio_FOUND) set(KvikIO_BUILD_BENCHMARKS OFF) set(KvikIO_BUILD_EXAMPLES OFF) set(KvikIO_BUILD_TESTS OFF) -# Build and ship the Nsight Systems NIC plugin in the libkvikio wheel. set(KvikIO_BUILD_NSYS_PLUGIN ON) -set(KvikIO_NSYS_PLUGIN_INSTALL_DEFAULT ON) add_subdirectory(../../cpp kvikio-cpp) From 5a6b2e7afcb84d019ed5ef3fef92bc93e6c62a4b Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 14:54:34 -0400 Subject: [PATCH 12/16] Update --- docs/source/api.rst | 6 +++ python/kvikio/kvikio/__init__.py | 4 +- python/kvikio/kvikio/nsys.py | 65 ++++++++++++++++++++++++++++++ python/libkvikio/libkvikio/nsys.py | 16 ++------ 4 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 python/kvikio/kvikio/nsys.py diff --git a/docs/source/api.rst b/docs/source/api.rst index 71897a0769..433e35dbf5 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -79,3 +79,9 @@ Defaults .. autofunction:: set .. autofunction:: get + +Nsight Systems plugin +--------------------- +.. currentmodule:: kvikio.nsys + +.. autofunction:: nsys_plugin_search_dir diff --git a/python/kvikio/kvikio/__init__.py b/python/kvikio/kvikio/__init__.py index ee9d43d197..6c663d4576 100644 --- a/python/kvikio/kvikio/__init__.py +++ b/python/kvikio/kvikio/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # If libkvikio was installed as a wheel, we must request it to load the library symbols. @@ -23,6 +23,7 @@ get_page_cache_info, ) from kvikio.mmap import Mmap +from kvikio.nsys import nsys_plugin_search_dir from kvikio.remote_file import ( RemoteEndpointType, RemoteFile, @@ -44,6 +45,7 @@ "infer_remote_endpoint_type", "is_remote_file_available", "kvikio_deprecation_notice", + "nsys_plugin_search_dir", "RemoteEndpointType", "RemoteFile", "stream_register", diff --git a/python/kvikio/kvikio/nsys.py b/python/kvikio/kvikio/nsys.py new file mode 100644 index 0000000000..40acdc5e73 --- /dev/null +++ b/python/kvikio/kvikio/nsys.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + + +def nsys_plugin_search_dir() -> str: + """Return the directory that holds KvikIO's bundled Nsight Systems plugins. + + Works for both installation channels, which lay out the plugin differently. + + - pip (inside the ``libkvikio`` wheel, a Python package): + + .. code-block:: text + + site-packages/libkvikio/ + |-- __init__.py, load.py, nsys.py + |-- lib/libkvikio.so + |-- share/kvikio/nsys-plugins/ --> returned + |-- kvikio_nic/{kvikio_nic_nsys_plugin, nsys-plugin.yaml} + + - conda (the ``libkvikio`` conda package, C++ files only, no Python module): + + .. code-block:: text + + $CONDA_PREFIX/ + |-- lib/libkvikio.so + |-- include/kvikio/ + |-- share/kvikio/nsys-plugins/ --> returned + |-- kvikio_nic/{kvikio_nic_nsys_plugin, nsys-plugin.yaml} + + Point Nsight Systems at the returned path. This requires nsys 2026.2.1 or newer, + which introduced ``NSYS_PLUGIN_SEARCH_DIRS``: + + .. code-block:: bash + + export NSYS_PLUGIN_SEARCH_DIRS="$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')" + nsys profile --enable=kvikio_nic ... + + Older nsys versions do not search external plugin directories. There the + ``kvikio_nic`` directory must be copied or symlinked into the nsys installation + itself, next to the bundled plugins (write access to the installation is needed, + typically root): + + .. code-block:: bash + + sudo ln -s "$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')/kvikio_nic" \ + /opt/nvidia/nsight-systems//target-linux-x64/plugins/ + + Returns + ------- + str + The plugin search directory. The path is returned even if it does not exist, + for example when the plugin was not built. + """ + try: + import libkvikio + + # pip: the plugin ships inside the libkvikio wheel. + return libkvikio.nsys_plugin_search_dir() + except ModuleNotFoundError: + # conda: the libkvikio conda package installs the plugin into the + # environment prefix, which is sys.prefix in a conda environment. + return os.path.join(sys.prefix, "share", "kvikio", "nsys-plugins") diff --git a/python/libkvikio/libkvikio/nsys.py b/python/libkvikio/libkvikio/nsys.py index 8714361ae0..3cdf6d1bbb 100644 --- a/python/libkvikio/libkvikio/nsys.py +++ b/python/libkvikio/libkvikio/nsys.py @@ -5,21 +5,13 @@ def nsys_plugin_search_dir() -> str: - """Return the directory that holds KvikIO's bundled Nsight Systems plugins. - - When libkvikio is installed as a wheel, the ``kvikio_nic`` plugin (an executable - plus its ``nsys-plugin.yaml`` manifest) is packaged under this directory. Point - Nsight Systems at it by exporting the returned path on ``NSYS_PLUGIN_SEARCH_DIRS``, - for example:: - - export NSYS_PLUGIN_SEARCH_DIRS="$(python -c 'import libkvikio; print(libkvikio.nsys_plugin_search_dir())')" - nsys profile --enable=kvikio_nic ... + """Return the directory that holds KvikIO's bundled Nsight Systems plugins. Valid + only for pip installation. Returns ------- str - The ``share/kvikio/nsys-plugins`` directory inside the installed - ``libkvikio`` package. The path is returned even if it does not exist, for - example when the plugin was not built. + The ``share/kvikio/nsys-plugins`` directory inside the installed ``libkvikio`` + package. """ return os.path.join(os.path.dirname(__file__), "share", "kvikio", "nsys-plugins") From 41226eab85fd50865c7029a047bbe863599eb2e1 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 15:19:27 -0400 Subject: [PATCH 13/16] Cleanup --- cpp/nsys_plugins/nic/CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cpp/nsys_plugins/nic/CMakeLists.txt b/cpp/nsys_plugins/nic/CMakeLists.txt index 4c32608dd2..d787955183 100644 --- a/cpp/nsys_plugins/nic/CMakeLists.txt +++ b/cpp/nsys_plugins/nic/CMakeLists.txt @@ -28,12 +28,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") ) endif() -# Install the binary and its manifest side by side into a directory named after the plugin, matching -# how nsys discovers plugins under NSYS_PLUGIN_SEARCH_DIRS. -# -# The plugin is a user-facing tool, so it is part of the default install: both the base conda -# libkvikio package (plain `cmake --install`) and the libkvikio wheel ship it. The dedicated -# component remains available for targeted `cmake --install --component nsys_plugin` layouts. install( TARGETS kvikio_nic_nsys_plugin COMPONENT nsys_plugin From 422baaf6ac275a419ffa15a01199e87ae6143013 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Mon, 6 Jul 2026 15:24:46 -0400 Subject: [PATCH 14/16] Improve plugin doc --- cpp/nsys_plugins/nic/nsys-plugin.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/nsys_plugins/nic/nsys-plugin.yaml b/cpp/nsys_plugins/nic/nsys-plugin.yaml index 9da7513e1a..ebfcc38718 100644 --- a/cpp/nsys_plugins/nic/nsys-plugin.yaml +++ b/cpp/nsys_plugins/nic/nsys-plugin.yaml @@ -9,12 +9,11 @@ ExtendedDescription: | -h | --help Print this help message Plugin arguments follow the plugin name as a comma separated list (commas only, no - spaces); each comma separated token is passed verbatim as one argument. These are + spaces). Each comma separated token is passed verbatim as one argument. These are equivalent: --enable=kvikio_nic,-d,eth0,-i,20000 + --enable=kvikio_nic,-deth0,-i20000 --enable=kvikio_nic,--device=eth0,--interval=20000 - A literal comma inside an argument (for example in a --device regex) must be escaped - as '\,'. With no --device filter, all up (or unknown) non-loopback interfaces are monitored. A --device regex selects interfaces by name and bypasses the up check, so an explicit From bbafc7bfc9d9b44a1910ad20f00f228bd0ea9d5a Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 7 Jul 2026 00:52:22 +0000 Subject: [PATCH 15/16] Improve names on UI --- cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp index b5bdf0ec74..8b767df48a 100644 --- a/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp +++ b/cpp/nsys_plugins/nic/kvikio_nic_nsys_plugin.cpp @@ -254,7 +254,7 @@ std::uint64_t register_rate_schema(nvtxDomainHandle_t domain) * @brief Register an NVTX counter group in the given domain. * * @param domain NVTX domain handle. - * @param name Counter group name (for example "nic_bandwidth.eth0"). + * @param name Counter group name (for example "eth0"). * @param schema_id Payload schema id from register_rate_schema. * @return The counter id to pass to nvtxCounterSample(). */ @@ -275,9 +275,8 @@ std::uint64_t register_counter(nvtxDomainHandle_t domain, /** * @brief Samples NIC byte counters at a fixed interval and emits them as NVTX counter groups. * - * Construction registers one `nic_bandwidth.` counter group per interface plus - * `nic_bandwidth.total`. `run()` then differences the counters each tick until the stop flag is - * raised by the signal handler. + * Construction registers one `` counter group per interface plus `total`. `run()` then + * differences the counters each tick until the stop flag is raised by the signal handler. */ class BandwidthCollector { public: @@ -295,9 +294,9 @@ class BandwidthCollector { auto const schema_id = register_rate_schema(_domain); _counter_ids.reserve(_reader.interfaces().size()); for (auto const& name : _reader.interfaces()) { - _counter_ids.push_back(register_counter(_domain, "nic_bandwidth." + name, schema_id)); + _counter_ids.push_back(register_counter(_domain, name, schema_id)); } - _total_counter_id = register_counter(_domain, "nic_bandwidth.total", schema_id); + _total_counter_id = register_counter(_domain, "total", schema_id); } /** From 47d15284f2c5abf558230ac67c10df2846c74908 Mon Sep 17 00:00:00 2001 From: Tianyu Liu Date: Tue, 7 Jul 2026 01:35:43 -0400 Subject: [PATCH 16/16] UPdate doc --- docs/source/index.rst | 1 + docs/source/profiling.rst | 43 ++++++++++++++++++++++++++++++++++++ python/kvikio/kvikio/nsys.py | 18 +-------------- 3 files changed, 45 insertions(+), 17 deletions(-) create mode 100644 docs/source/profiling.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 9e302b5f44..a47d156043 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,5 +25,6 @@ Contents zarr remote_file runtime_settings + profiling api genindex diff --git a/docs/source/profiling.rst b/docs/source/profiling.rst new file mode 100644 index 0000000000..318cd55b3b --- /dev/null +++ b/docs/source/profiling.rst @@ -0,0 +1,43 @@ +Profiling +========= + +NIC bandwidth on the Nsight Systems timeline +-------------------------------------------- +KvikIO ships a standalone `Nsight Systems `_ (nsys) plugin named ``kvikio_nic`` that samples per-interface network bandwidth and places it on the nsys timeline. + +The plugin reads the Linux kernel byte counters (``/sys/class/net//statistics/{rx,tx}_bytes``) at a fixed interval and emits one NVTX counter group per monitored interface plus a ``total`` group, all under the ``KvikIO NIC`` NVTX domain. Each group carries the receive (``rx``) and transmit (``tx``) rates. + +Installation +------------ +The plugin ships by default on both packaging channels, inside the ``libkvikio`` conda package and inside the ``libkvikio`` wheel. It consists of an executable (``kvikio_nic_nsys_plugin``) and a manifest (``nsys-plugin.yaml``) placed together in a ``kvikio_nic`` directory. Use :py:func:`kvikio.nsys_plugin_search_dir` to locate that directory on either channel. + +Command line options +-------------------- + * ``-i | --interval``: Sampling interval in microseconds (default 20000, i.e. 50 Hz). + * ``-d | --device``: Interface name regex. If not given, all up (or unknown) non-loopback interfaces are monitored. An explicit regex selects interfaces by name and bypasses the up check, so an explicit request is always honored. + +On the nsys command line, plugin arguments follow the plugin name as a comma separated list. These are equivalent: + +.. code-block:: bash + + nsys profile --enable=kvikio_nic,-d,eth0,-i,20000 ... + nsys profile --enable=kvikio_nic,-deth0,-i20000 ... + nsys profile --enable=kvikio_nic,--device=eth0,--interval=20000 ... + +Usage with nsys 2026.2.1 or newer +--------------------------------- +Nsight Systems 2026.2.1 introduced ``NSYS_PLUGIN_SEARCH_DIRS`` for discovering third party plugins: + +.. code-block:: bash + + export NSYS_PLUGIN_SEARCH_DIRS="$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')" + nsys profile --enable=help # should list kvikio_nic + nsys profile --enable=kvikio_nic,--device=eth0 --trace=cuda,nvtx,osrt -o nsys_report my_application + +Usage with older nsys +--------------------- +Older nsys versions only discover plugins inside the nsys installation itself. Either copy or symlink the ``kvikio_nic`` directory next to the bundled plugins (write access to the installation is needed, typically root): + +.. code-block:: bash + + sudo ln -s "$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')/kvikio_nic" /opt/nvidia/nsight-systems//target-linux-x64/plugins/ diff --git a/python/kvikio/kvikio/nsys.py b/python/kvikio/kvikio/nsys.py index 40acdc5e73..ec14add748 100644 --- a/python/kvikio/kvikio/nsys.py +++ b/python/kvikio/kvikio/nsys.py @@ -30,23 +30,7 @@ def nsys_plugin_search_dir() -> str: |-- share/kvikio/nsys-plugins/ --> returned |-- kvikio_nic/{kvikio_nic_nsys_plugin, nsys-plugin.yaml} - Point Nsight Systems at the returned path. This requires nsys 2026.2.1 or newer, - which introduced ``NSYS_PLUGIN_SEARCH_DIRS``: - - .. code-block:: bash - - export NSYS_PLUGIN_SEARCH_DIRS="$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')" - nsys profile --enable=kvikio_nic ... - - Older nsys versions do not search external plugin directories. There the - ``kvikio_nic`` directory must be copied or symlinked into the nsys installation - itself, next to the bundled plugins (write access to the installation is needed, - typically root): - - .. code-block:: bash - - sudo ln -s "$(python -c 'import kvikio; print(kvikio.nsys_plugin_search_dir())')/kvikio_nic" \ - /opt/nvidia/nsight-systems//target-linux-x64/plugins/ + See :doc:`profiling ` for how to enable the plugin in Nsight Systems. Returns -------