[None][test] Dump a stack traceback when a test hangs

jenkins/L0_Test.groovy

@@ -1232,6 +1232,7 @@ def getPytestBaseCommandLine(
         "--periodic-junit-xmlpath ${outputPath}/results.xml",
         "--periodic-batch-size=1",
         "--periodic-save-unfinished-test",
+        "--periodic-hang-traceback",
     ]
 
     if (perfMode) {

tests/integration/defs/conftest.py

@@ -1879,6 +1879,16 @@ def pytest_addoption(parser):
         "This helps identify which test was running when a timeout or crash occurs. "
         "Only used with --periodic-junit.",
     )
+    parser.addoption(
+        "--periodic-hang-traceback",
+        action="store_true",
+        default=False,
+        help=
+        "Dump every thread's stack to hang_traceback.txt when a test overruns its "
+        "timeout or the process is signalled (default: False). This turns an empty "
+        "'Test terminated unexpectedly' record into a diagnosable hang stack. "
+        "Only used with --periodic-junit.",
+    )
 
 
 @pytest.hookimpl(trylast=True)
@@ -2010,6 +2020,8 @@ def pytest_configure(config):
         periodic_batch_size = config.getoption("--periodic-batch-size")
         periodic_save_unfinished_test = config.getoption(
             "--periodic-save-unfinished-test", default=False)
+        periodic_hang_traceback = config.getoption("--periodic-hang-traceback",
+                                                   default=False)
 
         # Create output directory early (like --junitxml does) to avoid conflicts with other plugins
         # that may need to write to the same directory (e.g., pytest-split)
@@ -2027,6 +2039,7 @@ def pytest_configure(config):
                 'warning': print_warning
             },
             save_unfinished_test=periodic_save_unfinished_test,
+            dump_hang_traceback=periodic_hang_traceback,
         )
 
         # Configure and register the reporter
@@ -2039,6 +2052,7 @@ def pytest_configure(config):
         )
         print_info(f"  Batch size: {periodic_batch_size} tests")
         print_info(f"  Save unfinished test: {periodic_save_unfinished_test}")
+        print_info(f"  Hang traceback: {periodic_hang_traceback}")
     elif periodic and not output_dir:
         print_warning(
             "Warning: --periodic-junit requires --output-dir to be set. "

tests/integration/defs/utils/periodic_junit.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,23 +19,27 @@
 pytest's built-in junitxml plugin for simplified test result handling.
 """
 
+import faulthandler
 import os
 import platform
 import signal
+import threading
 import time
 import xml.etree.ElementTree as ET
 from datetime import datetime
-from typing import Optional
+from typing import Optional, TextIO
 
 try:
     from _pytest.config import Config
     from _pytest.junitxml import LogXML
+    from _pytest.nodes import Item
     from _pytest.reports import TestReport
 except ImportError:
     # Fallback for different pytest versions
     Config = None  # type: ignore
     TestReport = None  # type: ignore
     LogXML = None  # type: ignore
+    Item = None  # type: ignore
 
 
 class PeriodicJUnitXML:
@@ -75,6 +79,10 @@ def __init__(
             logger=None,  # Optional logger (info, warning functions)
             save_unfinished_test:
         bool = False,  # Save unfinished test name in output-dir/unfinished_test.txt if True
+            dump_hang_traceback:
+        bool = False,  # Dump a thread traceback to output-dir/hang_traceback.txt on a hang if True
+            hang_dump_fraction:
+        float = 0.9,  # Fraction of a test's timeout after which to dump the hang traceback
     ):
         """
         Initialize periodic reporter.
@@ -88,12 +96,30 @@ def __init__(
             batch_size: Number of tests before triggering a save (default: 10)
             logger: Optional dictionary with 'info' and 'warning' functions for logging
             save_unfinished_test: If True, save unfinished test name in output-dir/unfinished_test.txt
+            dump_hang_traceback: If True, dump every thread's stack to
+                output-dir/hang_traceback.txt when a test overruns its timeout or the
+                process is signalled, so a hang leaves a diagnosable stack instead of an
+                empty "Test terminated unexpectedly" record.
+            hang_dump_fraction: Dump the traceback after this fraction of a test's
+                timeout elapses (default: 0.9), i.e. just before the test is killed.
         """
         self.xmlpath = os.path.abspath(xmlpath)
         self.time_interval = interval
         self.batch_size = batch_size
         self.logger = logger or {}
         self.save_unfinished_test = save_unfinished_test
+        self.dump_hang_traceback = dump_hang_traceback
+        self.hang_dump_fraction = hang_dump_fraction
+        # Kept open for the process lifetime so faulthandler can write to it from the
+        # watchdog thread or a signal handler (both need a valid, live file object).
+        self._hang_file: Optional[TextIO] = None
+        # Per-test watchdog timer. A private threading.Timer is used instead of
+        # faulthandler.dump_traceback_later() on purpose: the latter is a single
+        # process-global timer shared with pytest-timeout / pytest's own faulthandler
+        # plugin, so arming/cancelling it here could clobber (or be clobbered by)
+        # theirs. Our own timer owns no shared state.
+        self._hang_timer: Optional[threading.Timer] = None
+        self._hang_lock = threading.Lock()
 
         self.completed_tests = 0
         self.last_save_time = time.time()
@@ -137,6 +163,11 @@ def pytest_configure(self, config: Config):
         # Register signal handlers for graceful shutdown
         self._register_signal_handlers()
 
+        # Set up hang-traceback dumping (after signal handlers, so the handler can
+        # reuse the open file object).
+        if self.dump_hang_traceback:
+            self._setup_hang_dump()
+
         self._log_info(f"PeriodicJUnitXML: Initialized at {self.xmlpath} "
                        "(lightweight mode - fast collection, batch processing)")
 
@@ -221,6 +252,7 @@ def pytest_runtest_logreport(self, report: TestReport):
 
     def pytest_sessionfinish(self):
         """Generate final report at session end."""
+        self._cancel_hang_timer()
         try:
             self._generate_report(is_final=True)
         except Exception as e:
@@ -327,6 +359,115 @@ def _generate_report(self, is_final=False):
                     pass
             raise
 
+    def _hang_traceback_path(self) -> str:
+        """Path of the sidecar file that captures thread stacks on a hang."""
+        return os.path.join(os.path.dirname(self.xmlpath), "hang_traceback.txt")
+
+    def _setup_hang_dump(self) -> None:
+        """Open the sidecar file and register a C-level dump on SIGINT/SIGTERM.
+
+        The file is kept open for the whole process lifetime because the watchdog
+        thread writes to it via ``faulthandler.dump_traceback(file=...)`` and needs
+        a live file object. ``faulthandler.register(..., chain=True)`` installs a
+        C-level handler that dumps every thread's stack on a wall-clock kill even
+        when the main thread is stuck in native C/CUDA code (a pure-Python signal
+        handler would be deferred until the interpreter regains control); chaining
+        keeps the existing Python handler that persists results. ``enable()`` is
+        deliberately NOT called -- it is a process-wide redirect of fatal-signal
+        dumps that would override pytest's own crash handling.
+        """
+        try:
+            os.makedirs(os.path.dirname(self.xmlpath), exist_ok=True)
+            self._hang_file = open(self._hang_traceback_path(),
+                                   "a",
+                                   encoding="utf-8")
+            for signum in (signal.SIGINT, signal.SIGTERM):
+                try:
+                    faulthandler.register(signum,
+                                          file=self._hang_file,
+                                          all_threads=True,
+                                          chain=True)
+                except (OSError, RuntimeError, ValueError):
+                    pass
+            self._log_info(
+                f"Hang-traceback dumping enabled -> {self._hang_traceback_path()}"
+            )
+        except OSError as e:
+            self._log_warning(f"Could not enable hang-traceback dumping: {e}")
+            self._hang_file = None
+
+    def _effective_timeout(self, item: "Item") -> Optional[float]:
+        """Resolve the timeout (seconds) in force for this test, or None.
+
+        Prefers a per-test ``@pytest.mark.timeout`` / test-list override (positional
+        or ``timeout=`` keyword form), then falls back to the global ``--timeout``.
+        """
+        marker = item.get_closest_marker("timeout")
+        if marker is not None:
+            candidate = marker.kwargs.get("timeout") if marker.kwargs else None
+            if candidate is None and marker.args:
+                candidate = marker.args[0]
+            try:
+                if candidate is not None:
+                    return float(candidate)
+            except (TypeError, ValueError):
+                pass
+        try:
+            value = item.config.getoption("timeout", default=None)
+        except (ValueError, KeyError):
+            value = None
+        try:
+            return float(value) if value else None
+        except (TypeError, ValueError):
+            return None
+
+    def _dump_hang(self, nodeid: str) -> None:
+        """Write every thread's stack to the sidecar file (called from the timer)."""
+        if self._hang_file is None:
+            return
+        try:
+            self._hang_file.write(
+                f"\n===== hang watchdog fired for {nodeid} =====\n")
+            faulthandler.dump_traceback(file=self._hang_file, all_threads=True)
+            self._hang_file.flush()
+        except (OSError, RuntimeError, ValueError):
+            pass
+
+    def _cancel_hang_timer(self) -> None:
+        """Cancel the pending watchdog timer, if any."""
+        with self._hang_lock:
+            timer = self._hang_timer
+            self._hang_timer = None
+        if timer is not None:
+            timer.cancel()
+
+    def pytest_runtest_setup(self, item: "Item") -> None:
+        """Arm a watchdog that dumps all thread stacks just before a hang is killed.
+
+        The timer is armed once for the whole item at setup and only cancelled when
+        the next test starts (or at session end), so a hang anywhere in setup, call,
+        or a fixture finalizer/teardown is captured. This also makes ``func_only``
+        irrelevant: any phase overrunning the timeout dumps.
+        """
+        if not self.dump_hang_traceback or self._hang_file is None:
+            return
+        self._cancel_hang_timer()  # never leave a stale timer armed
+        timeout = self._effective_timeout(item)
+        if not timeout or timeout <= 0:
+            return
+        # Dump slightly before the test is killed. The timer thread can still run
+        # while the main thread is blocked in a CUDA/NCCL call, since torch releases
+        # the GIL around those blocking calls -- which is exactly when the per-test
+        # timeout cannot unwind the hang.
+        dump_after = max(1.0, timeout * self.hang_dump_fraction)
+        timer = threading.Timer(dump_after,
+                                self._dump_hang,
+                                args=(item.nodeid, ))
+        timer.daemon = True
+        with self._hang_lock:
+            self._hang_timer = timer
+        timer.start()
+
     def _register_signal_handlers(self):
         """Register signal handlers for graceful shutdown on interruption."""
 
@@ -337,6 +478,13 @@ def signal_handler(signum, frame):
                 f"\n\nReceived {signal_name} signal. Saving test results before exit..."
             )
 
+            # The stack dump for this signal is handled at C level by the
+            # faulthandler.register(..., chain=True) installed in _setup_hang_dump
+            # (which runs even if the main thread is stuck in native code); just
+            # cancel the watchdog so it cannot also fire and interleave output.
+            if self.dump_hang_traceback:
+                self._cancel_hang_timer()
+
             try:
                 # Process any pending reports first
                 if self.pending_reports:

tests/integration/defs/utils/test_periodic_junit.py

@@ -0,0 +1,91 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for the hang-traceback dumping in PeriodicJUnitXML.
+
+These exercise the pure-Python wiring only (timeout resolution and the watchdog
+that writes output-dir/hang_traceback.txt); they need no GPU or model access.
+"""
+
+import os
+import time
+
+from .periodic_junit import PeriodicJUnitXML
+
+
+class _Marker:
+    def __init__(self, args=(), kwargs=None):
+        self.args = args
+        self.kwargs = kwargs or {}
+
+
+class _Config:
+    def __init__(self, timeout=None):
+        self._timeout = timeout
+
+    def getoption(self, name, default=None):
+        return self._timeout if name == "timeout" else default
+
+
+class _Item:
+    def __init__(self, nodeid="pkg/test_x.py::test_y", timeout_opt=None, marker=None):
+        self.nodeid = nodeid
+        self.config = _Config(timeout_opt)
+        self._marker = marker
+
+    def get_closest_marker(self, name):
+        return self._marker
+
+
+def _make(tmp_path, **kwargs):
+    reporter = PeriodicJUnitXML(
+        xmlpath=os.path.join(tmp_path, "results.xml"), dump_hang_traceback=True, **kwargs
+    )
+    reporter._setup_hang_dump()
+    return reporter
+
+
+def test_effective_timeout_resolution(tmp_path):
+    reporter = _make(str(tmp_path))
+    # positional marker, keyword marker, and the global --timeout all resolve.
+    assert reporter._effective_timeout(_Item(marker=_Marker(args=(90,)))) == 90.0
+    assert reporter._effective_timeout(_Item(marker=_Marker(kwargs={"timeout": 45}))) == 45.0
+    assert reporter._effective_timeout(_Item(timeout_opt=120)) == 120.0
+    # no marker and no --timeout -> no watchdog.
+    assert reporter._effective_timeout(_Item()) is None
+
+
+def test_hang_dumps_traceback(tmp_path):
+    reporter = _make(str(tmp_path), hang_dump_fraction=0.5)
+    # timeout 2s * 0.5 -> dump after ~1s.
+    reporter.pytest_runtest_setup(_Item(timeout_opt=2.0))
+    time.sleep(1.4)
+    content = open(reporter._hang_traceback_path(), encoding="utf-8").read()
+    assert "hang watchdog fired for pkg/test_x.py::test_y" in content
+    assert "Thread" in content or 'File "' in content
+
+
+def test_completed_test_is_not_dumped(tmp_path):
+    reporter = _make(str(tmp_path), hang_dump_fraction=0.5)
+    reporter.pytest_runtest_setup(_Item(timeout_opt=2.0))
+    time.sleep(0.2)
+    reporter._cancel_hang_timer()  # what the next setup / sessionfinish does
+    time.sleep(1.2)  # past the would-be dump point
+    assert os.path.getsize(reporter._hang_traceback_path()) == 0
+
+
+def test_no_watchdog_without_timeout(tmp_path):
+    reporter = _make(str(tmp_path))
+    reporter.pytest_runtest_setup(_Item())  # no timeout -> nothing armed
+    assert reporter._hang_timer is None