From afd44dcf6ca165842885705fac7c4da2c361ac66 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Mon, 6 Jul 2026 07:17:09 +0000 Subject: [PATCH 1/3] [TRTLLM-13409][test] fail fast + surface server logs when a perf-sanity server dies or never becomes healthy Post-merge CI analysis of TRTLLM-13409 showed the dominant remaining hang-shaped burn is a disagg/perf-sanity server that dies or wedges during startup: the observed pattern is "trtllm-serve disaggregated launched, died before binding, and nobody noticed" -- the benchmark client polls /health (connection refused) until the 90-minute per-test pytest timeout, the CTX/GEN/DISAGG babysitter ranks wait only for the benchmark-done file without ever polling their server process, 8-36 GPUs are held, and nothing in the CI log shows the server-side story. Harness defects compound this: - wait_for_endpoint_ready was called with the whole-test timeout (DEFAULT_TIMEOUT=10800s), so the pytest kill always fired first and the harness never reached its own timeout/reporting path; - its error-file scan ran every 300 loop iterations (5-30 min of wall time) and silently skipped missing files; - no failure path dumped the server log tail, and report_error's keyword scan iterated an already-exhausted file handle, so it never matched. Changes (process death is an event, not a timeout -- detect it as one, and keep timeouts only as backstops): - test_common.http_utils: new fail_if_proc_died() raising with the server log tail via report_error; called from wait_for_benchmark_ready()'s loop on the CTX/GEN and DISAGG_SERVER ranks (after the benchmark-done check, so a server exiting post-benchmark cannot fail a passing test). A dead server raises within one loop iteration with its log tail in the CI log; the rank exits nonzero, ending its SLURM step (srun --kill-on-bad-exit=1), and the remaining steps fail fast on the dead endpoint via the bounded ready-wait. - http_utils.wait_for_endpoint_ready: reuse the same helper for the dead-proc check; time-based error-file scanning (check_interval, default 30s); warn once per missing check_file; dump the tail of every check_file on ALL failure paths (dead server_proc, error keyword hit, ready timeout). - error_utils: report_error scans the buffered lines instead of the exhausted file handle (keyword context now actually works) and ALWAYS appends the log tail even when a keyword matched earlier (the first hit may be noise while the fatal error sits at end-of-file); check_error and report_error skip lines carrying known-benign markers ("[Autotuner]" warmup probe-OOMs legitimately contain "out of memory"), so the tighter scan cadence cannot fail healthy startups on expected warnings. - test_perf_sanity: bound the ready wait to min(self.timeout, server_ready_timeout(...)) with per-path defaults (aggregated 1800s; disaggregated 3600s, since the disagg /health only answers once EVERY ctx/gen worker finished model load + autotune + warmup). TRTLLM_TEST_SERVER_READY_TIMEOUT overrides both, read at call time and parsed defensively so a malformed value cannot break collection. A never-healthy server now fails in <=30-60 min with server logs attached instead of a bare 90-min TIMEOUT. Adds CPU-only unit tests (registered in l0_a10) covering the fail-fast paths, the babysitter liveness helper, the benign-marker filter, the always-tail behavior, and the report_error regression; a NO_PROXY fixture keeps the dead-endpoint tests correct behind corporate HTTP proxies. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- .../integration/defs/perf/test_perf_sanity.py | 74 +++++++- .../test_lists/test-db/l0_cpu_arm.yml | 1 + .../test_lists/test-db/l0_cpu_x86.yml | 1 + tests/test_common/error_utils.py | 38 ++++- tests/test_common/http_utils.py | 84 +++++++-- .../others/test_http_utils_fail_fast.py | 159 ++++++++++++++++++ 6 files changed, 330 insertions(+), 27 deletions(-) create mode 100644 tests/unittest/others/test_http_utils_fail_fast.py diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 0234e06dba7c..4cbb19646dd8 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -28,7 +28,7 @@ import pytest import yaml from test_common.error_utils import report_error -from test_common.http_utils import wait_for_endpoint_ready +from test_common.http_utils import fail_if_proc_died, wait_for_endpoint_ready from defs.trt_test_alternative import print_info from tensorrt_llm._utils import get_free_port @@ -96,6 +96,36 @@ def ensure_bench_serving_repo() -> str: DEFAULT_TIMEOUT = 10800 +# Defaults for the server *ready* wait, separate from the whole-test timeout: +# a server that is not healthy after this long is not going to be, and failing +# here (with server-log tails, see wait_for_endpoint_ready) instead of at the +# per-test pytest kill both saves GPU-hours and leaves a classifiable failure +# in the CI log. The disagg bound is larger because its /health only answers +# once EVERY ctx/gen worker has finished model load + autotune + warmup. +AGG_SERVER_READY_TIMEOUT = 1800 +DISAGG_SERVER_READY_TIMEOUT = 3600 + + +def server_ready_timeout(default: int) -> int: + """Ready-wait bound; overridable via TRTLLM_TEST_SERVER_READY_TIMEOUT. + + Read at call time (not import time) so the env var can be adjusted per + invocation, and parsed defensively so a malformed value cannot break + pytest collection of this module. + """ + raw = os.environ.get("TRTLLM_TEST_SERVER_READY_TIMEOUT") + if not raw: + return default + try: + timeout = int(raw) + except ValueError: + timeout = 0 + if timeout <= 0: + print_info(f"Invalid TRTLLM_TEST_SERVER_READY_TIMEOUT={raw!r}; using default {default}s") + return default + return timeout + + AGG_CONFIG_FOLDER = os.environ.get("AGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/aggregated") DISAGG_CONFIG_FOLDER = os.environ.get( "DISAGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/disaggregated" @@ -1095,7 +1125,7 @@ def run_cmd(self, server_idx: int) -> List[str]: wait_for_endpoint_ready( f"http://{server_hostname}:{server_port}/health", - timeout=self.timeout, + timeout=min(self.timeout, server_ready_timeout(AGG_SERVER_READY_TIMEOUT)), check_files=[server_file_path], server_proc=server_proc, ) @@ -1270,8 +1300,25 @@ def _get_disagg_server_hostname_and_port(self, server_idx: int) -> Tuple[str, in server_config = yaml.safe_load(f) return server_config["hostname"], server_config["port"] - def wait_for_benchmark_ready(self, benchmark_status_file: str): - """Wait for benchmark to complete.""" + def wait_for_benchmark_ready( + self, + benchmark_status_file: str, + server_proc: subprocess.Popen | None = None, + server_log: str | None = None, + ): + """Wait for benchmark to complete, failing fast if our server dies. + + The liveness check is event-driven (process exit), not a timeout: a + ctx/gen/disagg server that dies here raises within one loop iteration + with its log tail in the CI log, and the rank exits nonzero. Teardown + of the rest of the stage then follows from the launcher + (``srun --kill-on-bad-exit=1`` kills this rank's step) plus the + benchmark rank's bounded ready-wait failing fast on the dead endpoint + -- instead of every rank sitting in this loop for the full timeout. + + The benchmark-done check runs FIRST so a server exiting just after a + completed benchmark cannot fail an otherwise-passing test. + """ start_time = time.time() while True: if os.path.exists(benchmark_status_file): @@ -1279,6 +1326,11 @@ def wait_for_benchmark_ready(self, benchmark_status_file: str): f"Benchmark status file found, terminating server {self.disagg_serving_type}" ) break + fail_if_proc_died( + server_proc, + f"{self.disagg_serving_type} server", + [server_log] if server_log else None, + ) elapsed_time = time.time() - start_time print_info(f"Waiting for benchmark status file, elapsed time: {elapsed_time}s") if elapsed_time > self.timeout: @@ -1359,7 +1411,11 @@ def run_cmd(self, server_idx: int) -> List[str]: stdout=server_ctx, stderr=subprocess.STDOUT, ) - self.wait_for_benchmark_ready(benchmark_status_file) + self.wait_for_benchmark_ready( + benchmark_status_file, + server_proc=server_proc, + server_log=server_file_path, + ) finally: print_info(f"Server {self.disagg_serving_type} stopped") server_proc.terminate() @@ -1384,7 +1440,11 @@ def run_cmd(self, server_idx: int) -> List[str]: stdout=disagg_server_ctx, stderr=subprocess.STDOUT, ) - self.wait_for_benchmark_ready(benchmark_status_file) + self.wait_for_benchmark_ready( + benchmark_status_file, + server_proc=disagg_server_proc, + server_log=disagg_server_file_path, + ) finally: print_info(f"Disagg server {self.disagg_serving_type} stopped") disagg_server_proc.terminate() @@ -1398,7 +1458,7 @@ def run_cmd(self, server_idx: int) -> List[str]: wait_for_endpoint_ready( f"http://{disagg_server_hostname}:{disagg_server_port}/health", - timeout=self.timeout, + timeout=min(self.timeout, server_ready_timeout(DISAGG_SERVER_READY_TIMEOUT)), check_files=self.get_server_logs(server_idx), ) diff --git a/tests/integration/test_lists/test-db/l0_cpu_arm.yml b/tests/integration/test_lists/test-db/l0_cpu_arm.yml index ba693acaf756..07b1a2767da0 100644 --- a/tests/integration/test_lists/test-db/l0_cpu_arm.yml +++ b/tests/integration/test_lists/test-db/l0_cpu_arm.yml @@ -14,3 +14,4 @@ l0_cpu_arm: orchestrator: mpi tests: - unittest/executor/test_rpc.py + - unittest/others/test_http_utils_fail_fast.py diff --git a/tests/integration/test_lists/test-db/l0_cpu_x86.yml b/tests/integration/test_lists/test-db/l0_cpu_x86.yml index 9a39993347b8..a4f4ce532609 100644 --- a/tests/integration/test_lists/test-db/l0_cpu_x86.yml +++ b/tests/integration/test_lists/test-db/l0_cpu_x86.yml @@ -14,3 +14,4 @@ l0_cpu_x86: orchestrator: mpi tests: - unittest/executor/test_rpc.py + - unittest/others/test_http_utils_fail_fast.py diff --git a/tests/test_common/error_utils.py b/tests/test_common/error_utils.py index 9cb57aaf88f7..74ead0bc31b7 100644 --- a/tests/test_common/error_utils.py +++ b/tests/test_common/error_utils.py @@ -12,6 +12,22 @@ "PMI2_Init failed to initialize", "OSError", ] +# Autotuner warmup intentionally probes tactics that can OOM and logs e.g. +# "[Autotuner] Single-pair run failed ... CUDA out of memory ...". Only that +# specific marker+OOM combination is benign: an autotuner-prefixed line with +# any other error keyword (e.g. "[Autotuner] RuntimeError: ...") is a real +# failure and must still be reported. +AUTOTUNER_MARKER = "[Autotuner]" +AUTOTUNER_BENIGN_TEXTS = [ + "out of memory", +] + + +def is_benign_line(line: str) -> bool: + """True for lines expected during a HEALTHY run despite an error keyword.""" + return AUTOTUNER_MARKER in line and any(text in line for text in AUTOTUNER_BENIGN_TEXTS) + + SLURM_LOG_TAIL_LINES = 200 # Number of lines to print from slurm job logs ERROR_CONTEXT_LINES = 100 # Number of lines to print before and after error line @@ -23,6 +39,8 @@ def check_error(file_path: str) -> list[tuple[int, str]]: error_lines = [] with open(file_path, "r", errors="replace") as f: for line_idx, line in enumerate(f, start=1): + if is_benign_line(line): + continue for keyword in ERROR_KEYWORDS: if keyword in line: error_lines.append((line_idx, line.strip())) @@ -45,17 +63,22 @@ def report_error( for log_file in log_files: if not os.path.exists(log_file): messages.append(f"Failed to read {log_file}: Path doesn't exist") + continue all_lines = None error_lines = [] try: with open(log_file, "r", errors="replace") as f: all_lines = f.readlines() - for line_idx, line in enumerate(f, start=1): - for keyword in ERROR_KEYWORDS: - if keyword in line: - error_lines.append((line_idx, line.strip())) - break + # Scan the buffered lines (iterating the exhausted file handle + # after readlines() would yield nothing). + for line_idx, line in enumerate(all_lines, start=1): + if is_benign_line(line): + continue + for keyword in ERROR_KEYWORDS: + if keyword in line: + error_lines.append((line_idx, line.strip())) + break except Exception as e: all_lines = None error_lines = [] @@ -74,7 +97,10 @@ def report_error( end_idx = min(len(all_lines), first_idx + ERROR_CONTEXT_LINES) context_lines = all_lines[start_idx:end_idx] messages.append("".join(context_lines)) - else: + if all_lines is not None: + # ALWAYS include the end of the log, even when a keyword matched + # above: the first keyword hit may be benign noise while the + # actual fatal error sits at the end of the file. tail_content = "".join(all_lines[-tail_lines:]) if all_lines else "(empty)" messages.append(f"--- {log_file} [last {tail_lines} lines] ---") messages.append(tail_content) diff --git a/tests/test_common/http_utils.py b/tests/test_common/http_utils.py index 9628dff1809f..9ed7953ab202 100644 --- a/tests/test_common/http_utils.py +++ b/tests/test_common/http_utils.py @@ -1,9 +1,47 @@ +import os import subprocess import time import requests -from test_common.error_utils import check_error +from test_common.error_utils import check_error, report_error + + +def _fail_with_server_logs(message: str, check_files: list[str] | None): + """Raise RuntimeError(message), appending the tail of each server log. + + A server that never becomes healthy often prints its story only to its own + log file; without dumping it here the CI log shows nothing but the client + poll loop, making the failure unclassifiable. + """ + if check_files: + # report_error raises RuntimeError(message + error context / log tails). + report_error(message, check_files) + raise RuntimeError(message) + + +def fail_if_proc_died( + proc: subprocess.Popen | None, + what: str, + check_files: list[str] | None = None, +): + """Event-driven fail-fast: raise (with server-log tails) if ``proc`` exited. + + Process death is an event, not a timeout: a babysitter that checks its + child converts a dead server into an immediate, diagnosable failure + instead of burning GPUs until a timeout. The resulting nonzero rank exit + ends that rank's SLURM step (``srun --kill-on-bad-exit=1`` in the perf + scripts); the remaining steps then fail fast on the dead endpoint via the + bounded ready-wait. + """ + if proc is None: + return + exit_code = proc.poll() + if exit_code is not None: + _fail_with_server_logs( + f"{what} exited unexpectedly with code {exit_code} while it was still needed.", + check_files, + ) def wait_for_endpoint_ready( @@ -11,28 +49,44 @@ def wait_for_endpoint_ready( timeout: int = 300, check_files: list[str] | None = None, server_proc: subprocess.Popen | None = None, + check_interval: float = 30.0, ): + """Poll ``url`` until it returns 200, failing fast and loudly otherwise. + + Fail-fast paths (all of which dump the tails of ``check_files`` so the + server-side story lands in the CI log): + - ``server_proc`` exited -> no point polling a dead server for the + remaining timeout; + - an error keyword appears in a ``check_files`` log (scanned every + ``check_interval`` seconds; time-based rather than iteration-based); + - ``timeout`` elapses without the endpoint becoming ready. + """ start = time.monotonic() - iteration = 0 + next_file_check = start + check_interval + missing_warned: set[str] = set() while time.monotonic() - start < timeout: # Check server_proc if provided (singular) - if server_proc is not None: - exit_code = server_proc.poll() - if exit_code is not None: - raise RuntimeError( - f"Server process exited with code {exit_code} before becoming ready." - ) - - iteration += 1 - if check_files and iteration % 300 == 0: + fail_if_proc_died(server_proc, "Server process (before becoming ready)", check_files) + + if check_files and time.monotonic() >= next_file_check: + next_file_check = time.monotonic() + check_interval for check_file in check_files: + if not os.path.exists(check_file): + if check_file not in missing_warned: + missing_warned.add(check_file) + print( + f"[WARNING] server log {check_file} does not exist " + "(yet?); cannot scan it for errors" + ) + continue error_lines = check_error(check_file) if error_lines: error_lines_str = ", ".join( [f"line {line_idx}: {line_str}" for line_idx, line_str in error_lines] ) - raise RuntimeError( - f"Found error in server file {check_file}: {error_lines_str}" + _fail_with_server_logs( + f"Found error in server file {check_file}: {error_lines_str}", + check_files, ) try: time.sleep(1) @@ -41,7 +95,9 @@ def wait_for_endpoint_ready( return except Exception as err: print(f"endpoint {url} is not ready, with exception: {err}") - raise RuntimeError(f"Endpoint {url} did not become ready within {timeout} seconds") + _fail_with_server_logs( + f"Endpoint {url} did not become ready within {timeout} seconds", check_files + ) def wait_for_endpoint_down(url: str, timeout: int = 300): diff --git a/tests/unittest/others/test_http_utils_fail_fast.py b/tests/unittest/others/test_http_utils_fail_fast.py new file mode 100644 index 000000000000..472480f04fa0 --- /dev/null +++ b/tests/unittest/others/test_http_utils_fail_fast.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fail-fast + log-tail-dump behavior of wait_for_endpoint_ready (no GPU, no server).""" + +import subprocess + +import pytest +from test_common.error_utils import check_error, report_error +from test_common.http_utils import fail_if_proc_died, wait_for_endpoint_ready + +# Nothing listens here; connections are refused immediately. +DEAD_URL = "http://127.0.0.1:9/health" + + +@pytest.fixture(autouse=True) +def _no_proxy(monkeypatch): + """A corporate HTTP proxy answering 200 would make DEAD_URL look ready.""" + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setenv("no_proxy", "127.0.0.1,localhost") + monkeypatch.delenv("HTTP_PROXY", raising=False) + monkeypatch.delenv("http_proxy", raising=False) + + +def _server_log(tmp_path, content): + p = tmp_path / "server.log" + p.write_text(content) + return str(p) + + +def test_dead_server_proc_fails_fast_and_dumps_log(tmp_path): + log = _server_log(tmp_path, "loading weights...\nlast server words\n") + proc = subprocess.Popen(["true"]) + proc.wait() + with pytest.raises(RuntimeError) as e: + wait_for_endpoint_ready(DEAD_URL, timeout=30, check_files=[log], server_proc=proc) + msg = str(e.value) + assert "exited unexpectedly with code" in msg + # The server-side story must land in the failure message (CI log). + assert "last server words" in msg + + +def test_error_keyword_in_server_log_fails_fast(tmp_path): + log = _server_log(tmp_path, "starting\nRuntimeError: engine exploded\n") + with pytest.raises(RuntimeError) as e: + wait_for_endpoint_ready(DEAD_URL, timeout=30, check_files=[log], check_interval=0.0) + msg = str(e.value) + assert "Found error in server file" in msg + assert "engine exploded" in msg + + +def test_ready_timeout_dumps_log_tail(tmp_path): + log = _server_log(tmp_path, "clean init so far, no error keywords\n") + with pytest.raises(RuntimeError) as e: + # check_interval > timeout so the error-scan never trips; we want the + # timeout path itself. + wait_for_endpoint_ready(DEAD_URL, timeout=2, check_files=[log], check_interval=60.0) + msg = str(e.value) + assert "did not become ready within" in msg + assert "clean init so far" in msg + + +def test_missing_check_file_does_not_crash_the_wait(tmp_path): + missing = str(tmp_path / "not-written-yet.log") + with pytest.raises(RuntimeError) as e: + wait_for_endpoint_ready(DEAD_URL, timeout=2, check_files=[missing], check_interval=0.0) + # Reaches the timeout path (not a FileNotFoundError) and reports the + # missing log in the dump. + msg = str(e.value) + assert "did not become ready within" in msg + assert "Path doesn't exist" in msg + + +def test_fail_if_proc_died_raises_with_log_tail(tmp_path): + """Event-driven babysitter check: dead child -> immediate raise + log tail.""" + log = _server_log(tmp_path, "gen server init...\nfinal gen words\n") + proc = subprocess.Popen(["true"]) + proc.wait() + with pytest.raises(RuntimeError) as e: + fail_if_proc_died(proc, "GEN_0 server", [log]) + msg = str(e.value) + assert "GEN_0 server exited unexpectedly with code 0" in msg + assert "final gen words" in msg + + +def test_fail_if_proc_died_noop_when_alive_or_none(tmp_path): + fail_if_proc_died(None, "no server") # no-op + proc = subprocess.Popen(["sleep", "5"]) + try: + fail_if_proc_died(proc, "alive server") # no raise + finally: + proc.kill() + proc.wait() + + +def test_report_error_keyword_scan_fires(tmp_path): + """Regression: report_error scanned an exhausted handle; keywords never matched.""" + log = _server_log( + tmp_path, + "line one\nRuntimeError: boom\nline three\n", + ) + with pytest.raises(RuntimeError) as e: + report_error("wrapper message", [log]) + msg = str(e.value) + assert "wrapper message" in msg + assert "Error line 2" in msg + assert "RuntimeError: boom" in msg + + +def test_check_error_skips_benign_autotuner_lines(tmp_path): + """Autotuner warmup probe-OOMs are expected on healthy startups.""" + log = _server_log( + tmp_path, + "[Autotuner] Single-pair run failed: CUDA out of memory. Tried to allocate...\n" + "normal progress line\n", + ) + assert check_error(log) == [] + # A real error outside the benign marker still trips the scan. + log2 = _server_log(tmp_path, "RuntimeError: engine exploded\n") + assert len(check_error(log2)) == 1 + + +def test_autotuner_marker_does_not_hide_real_errors(tmp_path): + """Only the marker+OOM combination is benign; other autotuner errors are real.""" + log = _server_log( + tmp_path, + "[Autotuner] Single-pair run failed: CUDA out of memory. Tried to allocate...\n" + "[Autotuner] RuntimeError: invalid tactic configuration\n", + ) + hits = check_error(log) + assert [(idx, "RuntimeError" in line) for idx, line in hits] == [(2, True)] + # report_error's scanner applies the same boundary. + with pytest.raises(RuntimeError) as e: + report_error("wrapper", [log]) + assert "Error line 2" in str(e.value) + + +def test_report_error_always_appends_tail_even_on_keyword_hit(tmp_path): + """The first keyword hit may be noise; the fatal error can sit at EOF.""" + lines = ["ValueError: early benign-looking hit\n"] + lines += [f"filler {i}\n" for i in range(300)] + lines += ["the actual fatal last words\n"] + log = _server_log(tmp_path, "".join(lines)) + with pytest.raises(RuntimeError) as e: + report_error("wrapper", [log]) + msg = str(e.value) + assert "Error line 1" in msg # keyword context present + assert "the actual fatal last words" in msg # AND the tail From c51a4139f364115c69a6166ce0fabdb388460920 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Thu, 16 Jul 2026 08:39:27 +0000 Subject: [PATCH 2/3] [TRTLLM-13409][test] mark fail-fast http_utils tests cpu_only The CPU-Generic CI stages run pytest with -m 'cpu_only and not disabled'; without the marker all 10 tests were deselected and pytest exited with code 5, failing CPU-Generic-x86-1 and CPU-Generic-arm-1 in pipeline 47883. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tests/unittest/others/test_http_utils_fail_fast.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unittest/others/test_http_utils_fail_fast.py b/tests/unittest/others/test_http_utils_fail_fast.py index 472480f04fa0..f11996525eab 100644 --- a/tests/unittest/others/test_http_utils_fail_fast.py +++ b/tests/unittest/others/test_http_utils_fail_fast.py @@ -20,6 +20,10 @@ from test_common.error_utils import check_error, report_error from test_common.http_utils import fail_if_proc_died, wait_for_endpoint_ready +# The CPU-Generic CI stages select tests with `-m cpu_only`; without this +# marker every test here is deselected and pytest exits with code 5. +pytestmark = pytest.mark.cpu_only + # Nothing listens here; connections are refused immediately. DEAD_URL = "http://127.0.0.1:9/health" From baf66dc83650df5f2b71baf752189b6ead7bdc43 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Fri, 17 Jul 2026 04:29:08 +0000 Subject: [PATCH 3/3] [TRTLLM-13409][test] per-mode server-ready-timeout overrides Agg and disagg servers have very different init times (disagg /health answers only once every ctx/gen worker is up), so a single override var could not extend one without the other. Add TRTLLM_TEST_AGG_SERVER_READY_TIMEOUT and TRTLLM_TEST_DISAGG_SERVER_READY_TIMEOUT, with the existing TRTLLM_TEST_SERVER_READY_TIMEOUT as a shared fallback. Addresses brnguyen2's review comment on PR #16403. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- .../integration/defs/perf/test_perf_sanity.py | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 4cbb19646dd8..d5f2df9ba815 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -106,24 +106,34 @@ def ensure_bench_serving_repo() -> str: DISAGG_SERVER_READY_TIMEOUT = 3600 -def server_ready_timeout(default: int) -> int: - """Ready-wait bound; overridable via TRTLLM_TEST_SERVER_READY_TIMEOUT. +def server_ready_timeout(default: int, mode: str) -> int: + """Ready-wait bound for one serving mode ("AGG" or "DISAGG"). - Read at call time (not import time) so the env var can be adjusted per + Agg and disagg servers have very different init times (disagg's /health + answers only after every ctx/gen worker is up), so each mode has its own + override var, with the generic one as a shared fallback: + TRTLLM_TEST__SERVER_READY_TIMEOUT > TRTLLM_TEST_SERVER_READY_TIMEOUT + > the built-in per-mode default. + + Read at call time (not import time) so the env vars can be adjusted per invocation, and parsed defensively so a malformed value cannot break pytest collection of this module. """ - raw = os.environ.get("TRTLLM_TEST_SERVER_READY_TIMEOUT") - if not raw: - return default - try: - timeout = int(raw) - except ValueError: - timeout = 0 - if timeout <= 0: - print_info(f"Invalid TRTLLM_TEST_SERVER_READY_TIMEOUT={raw!r}; using default {default}s") - return default - return timeout + for var in ( + f"TRTLLM_TEST_{mode.upper()}_SERVER_READY_TIMEOUT", + "TRTLLM_TEST_SERVER_READY_TIMEOUT", + ): + raw = os.environ.get(var) + if not raw: + continue + try: + timeout = int(raw) + except ValueError: + timeout = 0 + if timeout > 0: + return timeout + print_info(f"Invalid {var}={raw!r}; ignoring it") + return default AGG_CONFIG_FOLDER = os.environ.get("AGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/aggregated") @@ -1125,7 +1135,9 @@ def run_cmd(self, server_idx: int) -> List[str]: wait_for_endpoint_ready( f"http://{server_hostname}:{server_port}/health", - timeout=min(self.timeout, server_ready_timeout(AGG_SERVER_READY_TIMEOUT)), + timeout=min( + self.timeout, server_ready_timeout(AGG_SERVER_READY_TIMEOUT, "AGG") + ), check_files=[server_file_path], server_proc=server_proc, ) @@ -1458,7 +1470,9 @@ def run_cmd(self, server_idx: int) -> List[str]: wait_for_endpoint_ready( f"http://{disagg_server_hostname}:{disagg_server_port}/health", - timeout=min(self.timeout, server_ready_timeout(DISAGG_SERVER_READY_TIMEOUT)), + timeout=min( + self.timeout, server_ready_timeout(DISAGG_SERVER_READY_TIMEOUT, "DISAGG") + ), check_files=self.get_server_logs(server_idx), )