diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ee9654166f9c..9e7825c9eaa5 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1232,6 +1232,7 @@ def getPytestBaseCommandLine( "--periodic-junit-xmlpath ${outputPath}/results.xml", "--periodic-batch-size=1", "--periodic-save-unfinished-test", + "--periodic-hang-traceback", ] if (perfMode) { diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index 1a5beb07cdf4..e455d2cfe4bd 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -1879,6 +1879,16 @@ def pytest_addoption(parser): "This helps identify which test was running when a timeout or crash occurs. " "Only used with --periodic-junit.", ) + parser.addoption( + "--periodic-hang-traceback", + action="store_true", + default=False, + help= + "Dump every thread's stack to hang_traceback.txt when a test overruns its " + "timeout or the process is signalled (default: False). This turns an empty " + "'Test terminated unexpectedly' record into a diagnosable hang stack. " + "Only used with --periodic-junit.", + ) @pytest.hookimpl(trylast=True) @@ -2010,6 +2020,8 @@ def pytest_configure(config): periodic_batch_size = config.getoption("--periodic-batch-size") periodic_save_unfinished_test = config.getoption( "--periodic-save-unfinished-test", default=False) + periodic_hang_traceback = config.getoption("--periodic-hang-traceback", + default=False) # Create output directory early (like --junitxml does) to avoid conflicts with other plugins # that may need to write to the same directory (e.g., pytest-split) @@ -2027,6 +2039,7 @@ def pytest_configure(config): 'warning': print_warning }, save_unfinished_test=periodic_save_unfinished_test, + dump_hang_traceback=periodic_hang_traceback, ) # Configure and register the reporter @@ -2039,6 +2052,7 @@ def pytest_configure(config): ) print_info(f" Batch size: {periodic_batch_size} tests") print_info(f" Save unfinished test: {periodic_save_unfinished_test}") + print_info(f" Hang traceback: {periodic_hang_traceback}") elif periodic and not output_dir: print_warning( "Warning: --periodic-junit requires --output-dir to be set. " diff --git a/tests/integration/defs/utils/periodic_junit.py b/tests/integration/defs/utils/periodic_junit.py index 68dfedfa17a7..79276cf65dbd 100644 --- a/tests/integration/defs/utils/periodic_junit.py +++ b/tests/integration/defs/utils/periodic_junit.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,23 +19,27 @@ pytest's built-in junitxml plugin for simplified test result handling. """ +import faulthandler import os import platform import signal +import threading import time import xml.etree.ElementTree as ET from datetime import datetime -from typing import Optional +from typing import Optional, TextIO try: from _pytest.config import Config from _pytest.junitxml import LogXML + from _pytest.nodes import Item from _pytest.reports import TestReport except ImportError: # Fallback for different pytest versions Config = None # type: ignore TestReport = None # type: ignore LogXML = None # type: ignore + Item = None # type: ignore class PeriodicJUnitXML: @@ -75,6 +79,10 @@ def __init__( logger=None, # Optional logger (info, warning functions) save_unfinished_test: bool = False, # Save unfinished test name in output-dir/unfinished_test.txt if True + dump_hang_traceback: + bool = False, # Dump a thread traceback to output-dir/hang_traceback.txt on a hang if True + hang_dump_fraction: + float = 0.9, # Fraction of a test's timeout after which to dump the hang traceback ): """ Initialize periodic reporter. @@ -88,12 +96,30 @@ def __init__( batch_size: Number of tests before triggering a save (default: 10) logger: Optional dictionary with 'info' and 'warning' functions for logging save_unfinished_test: If True, save unfinished test name in output-dir/unfinished_test.txt + dump_hang_traceback: If True, dump every thread's stack to + output-dir/hang_traceback.txt when a test overruns its timeout or the + process is signalled, so a hang leaves a diagnosable stack instead of an + empty "Test terminated unexpectedly" record. + hang_dump_fraction: Dump the traceback after this fraction of a test's + timeout elapses (default: 0.9), i.e. just before the test is killed. """ self.xmlpath = os.path.abspath(xmlpath) self.time_interval = interval self.batch_size = batch_size self.logger = logger or {} self.save_unfinished_test = save_unfinished_test + self.dump_hang_traceback = dump_hang_traceback + self.hang_dump_fraction = hang_dump_fraction + # Kept open for the process lifetime so faulthandler can write to it from the + # watchdog thread or a signal handler (both need a valid, live file object). + self._hang_file: Optional[TextIO] = None + # Per-test watchdog timer. A private threading.Timer is used instead of + # faulthandler.dump_traceback_later() on purpose: the latter is a single + # process-global timer shared with pytest-timeout / pytest's own faulthandler + # plugin, so arming/cancelling it here could clobber (or be clobbered by) + # theirs. Our own timer owns no shared state. + self._hang_timer: Optional[threading.Timer] = None + self._hang_lock = threading.Lock() self.completed_tests = 0 self.last_save_time = time.time() @@ -137,6 +163,11 @@ def pytest_configure(self, config: Config): # Register signal handlers for graceful shutdown self._register_signal_handlers() + # Set up hang-traceback dumping (after signal handlers, so the handler can + # reuse the open file object). + if self.dump_hang_traceback: + self._setup_hang_dump() + self._log_info(f"PeriodicJUnitXML: Initialized at {self.xmlpath} " "(lightweight mode - fast collection, batch processing)") @@ -221,6 +252,7 @@ def pytest_runtest_logreport(self, report: TestReport): def pytest_sessionfinish(self): """Generate final report at session end.""" + self._cancel_hang_timer() try: self._generate_report(is_final=True) except Exception as e: @@ -327,6 +359,115 @@ def _generate_report(self, is_final=False): pass raise + def _hang_traceback_path(self) -> str: + """Path of the sidecar file that captures thread stacks on a hang.""" + return os.path.join(os.path.dirname(self.xmlpath), "hang_traceback.txt") + + def _setup_hang_dump(self) -> None: + """Open the sidecar file and register a C-level dump on SIGINT/SIGTERM. + + The file is kept open for the whole process lifetime because the watchdog + thread writes to it via ``faulthandler.dump_traceback(file=...)`` and needs + a live file object. ``faulthandler.register(..., chain=True)`` installs a + C-level handler that dumps every thread's stack on a wall-clock kill even + when the main thread is stuck in native C/CUDA code (a pure-Python signal + handler would be deferred until the interpreter regains control); chaining + keeps the existing Python handler that persists results. ``enable()`` is + deliberately NOT called -- it is a process-wide redirect of fatal-signal + dumps that would override pytest's own crash handling. + """ + try: + os.makedirs(os.path.dirname(self.xmlpath), exist_ok=True) + self._hang_file = open(self._hang_traceback_path(), + "a", + encoding="utf-8") + for signum in (signal.SIGINT, signal.SIGTERM): + try: + faulthandler.register(signum, + file=self._hang_file, + all_threads=True, + chain=True) + except (OSError, RuntimeError, ValueError): + pass + self._log_info( + f"Hang-traceback dumping enabled -> {self._hang_traceback_path()}" + ) + except OSError as e: + self._log_warning(f"Could not enable hang-traceback dumping: {e}") + self._hang_file = None + + def _effective_timeout(self, item: "Item") -> Optional[float]: + """Resolve the timeout (seconds) in force for this test, or None. + + Prefers a per-test ``@pytest.mark.timeout`` / test-list override (positional + or ``timeout=`` keyword form), then falls back to the global ``--timeout``. + """ + marker = item.get_closest_marker("timeout") + if marker is not None: + candidate = marker.kwargs.get("timeout") if marker.kwargs else None + if candidate is None and marker.args: + candidate = marker.args[0] + try: + if candidate is not None: + return float(candidate) + except (TypeError, ValueError): + pass + try: + value = item.config.getoption("timeout", default=None) + except (ValueError, KeyError): + value = None + try: + return float(value) if value else None + except (TypeError, ValueError): + return None + + def _dump_hang(self, nodeid: str) -> None: + """Write every thread's stack to the sidecar file (called from the timer).""" + if self._hang_file is None: + return + try: + self._hang_file.write( + f"\n===== hang watchdog fired for {nodeid} =====\n") + faulthandler.dump_traceback(file=self._hang_file, all_threads=True) + self._hang_file.flush() + except (OSError, RuntimeError, ValueError): + pass + + def _cancel_hang_timer(self) -> None: + """Cancel the pending watchdog timer, if any.""" + with self._hang_lock: + timer = self._hang_timer + self._hang_timer = None + if timer is not None: + timer.cancel() + + def pytest_runtest_setup(self, item: "Item") -> None: + """Arm a watchdog that dumps all thread stacks just before a hang is killed. + + The timer is armed once for the whole item at setup and only cancelled when + the next test starts (or at session end), so a hang anywhere in setup, call, + or a fixture finalizer/teardown is captured. This also makes ``func_only`` + irrelevant: any phase overrunning the timeout dumps. + """ + if not self.dump_hang_traceback or self._hang_file is None: + return + self._cancel_hang_timer() # never leave a stale timer armed + timeout = self._effective_timeout(item) + if not timeout or timeout <= 0: + return + # Dump slightly before the test is killed. The timer thread can still run + # while the main thread is blocked in a CUDA/NCCL call, since torch releases + # the GIL around those blocking calls -- which is exactly when the per-test + # timeout cannot unwind the hang. + dump_after = max(1.0, timeout * self.hang_dump_fraction) + timer = threading.Timer(dump_after, + self._dump_hang, + args=(item.nodeid, )) + timer.daemon = True + with self._hang_lock: + self._hang_timer = timer + timer.start() + def _register_signal_handlers(self): """Register signal handlers for graceful shutdown on interruption.""" @@ -337,6 +478,13 @@ def signal_handler(signum, frame): f"\n\nReceived {signal_name} signal. Saving test results before exit..." ) + # The stack dump for this signal is handled at C level by the + # faulthandler.register(..., chain=True) installed in _setup_hang_dump + # (which runs even if the main thread is stuck in native code); just + # cancel the watchdog so it cannot also fire and interleave output. + if self.dump_hang_traceback: + self._cancel_hang_timer() + try: # Process any pending reports first if self.pending_reports: diff --git a/tests/integration/defs/utils/test_periodic_junit.py b/tests/integration/defs/utils/test_periodic_junit.py new file mode 100644 index 000000000000..3644cf96225c --- /dev/null +++ b/tests/integration/defs/utils/test_periodic_junit.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the hang-traceback dumping in PeriodicJUnitXML. + +These exercise the pure-Python wiring only (timeout resolution and the watchdog +that writes output-dir/hang_traceback.txt); they need no GPU or model access. +""" + +import os +import time + +from .periodic_junit import PeriodicJUnitXML + + +class _Marker: + def __init__(self, args=(), kwargs=None): + self.args = args + self.kwargs = kwargs or {} + + +class _Config: + def __init__(self, timeout=None): + self._timeout = timeout + + def getoption(self, name, default=None): + return self._timeout if name == "timeout" else default + + +class _Item: + def __init__(self, nodeid="pkg/test_x.py::test_y", timeout_opt=None, marker=None): + self.nodeid = nodeid + self.config = _Config(timeout_opt) + self._marker = marker + + def get_closest_marker(self, name): + return self._marker + + +def _make(tmp_path, **kwargs): + reporter = PeriodicJUnitXML( + xmlpath=os.path.join(tmp_path, "results.xml"), dump_hang_traceback=True, **kwargs + ) + reporter._setup_hang_dump() + return reporter + + +def test_effective_timeout_resolution(tmp_path): + reporter = _make(str(tmp_path)) + # positional marker, keyword marker, and the global --timeout all resolve. + assert reporter._effective_timeout(_Item(marker=_Marker(args=(90,)))) == 90.0 + assert reporter._effective_timeout(_Item(marker=_Marker(kwargs={"timeout": 45}))) == 45.0 + assert reporter._effective_timeout(_Item(timeout_opt=120)) == 120.0 + # no marker and no --timeout -> no watchdog. + assert reporter._effective_timeout(_Item()) is None + + +def test_hang_dumps_traceback(tmp_path): + reporter = _make(str(tmp_path), hang_dump_fraction=0.5) + # timeout 2s * 0.5 -> dump after ~1s. + reporter.pytest_runtest_setup(_Item(timeout_opt=2.0)) + time.sleep(1.4) + content = open(reporter._hang_traceback_path(), encoding="utf-8").read() + assert "hang watchdog fired for pkg/test_x.py::test_y" in content + assert "Thread" in content or 'File "' in content + + +def test_completed_test_is_not_dumped(tmp_path): + reporter = _make(str(tmp_path), hang_dump_fraction=0.5) + reporter.pytest_runtest_setup(_Item(timeout_opt=2.0)) + time.sleep(0.2) + reporter._cancel_hang_timer() # what the next setup / sessionfinish does + time.sleep(1.2) # past the would-be dump point + assert os.path.getsize(reporter._hang_traceback_path()) == 0 + + +def test_no_watchdog_without_timeout(tmp_path): + reporter = _make(str(tmp_path)) + reporter.pytest_runtest_setup(_Item()) # no timeout -> nothing armed + assert reporter._hang_timer is None