From 350635a04b13be6bc4883beddd7519c129002439 Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Tue, 21 Jul 2026 05:28:17 -0700 Subject: [PATCH 1/4] [https://nvbugs/6426834][fix] Deflake test_kv_transfer: cap NIXL progress threads, pin UCX env test_transfer_worker_v2[tp4_pp1_to_tp2_pp2] (and other multi-worker cases) intermittently hit the 120s per-test timeout in CI. Root cause: each NIXL transfer agent spawns TRTLLM_NIXL_NUM_THREADS (default 8) busy-polling UCX progress threads, and a single case builds up to 8 TransferWorkers (64 spinning threads per pytest process). On CI nodes shared by up to 8 single-GPU Jenkins agents the resulting CPU oversubscription inflates agent construction from ~3s to ~27s each, so 8 agents alone exceed the 120s timeout depending on neighbor load. Measured on an H100 node with 4 concurrent pytest processes: 8 agents take ~178s with the default 8 progress threads vs ~23s with 1 thread. Fix, applied to test_kv_transfer.py and test_kv_transfer_mp.py: - setdefault TRTLLM_NIXL_NUM_THREADS=1: these tests verify transfer logic, not transfer-engine threading (sender-side KV_TRANSFER_NUM_THREADS worker threads are unchanged). - Force UCX_TLS=^ib,gdr_copy and drop UCX_NET_DEVICES instead of the previous setdefault, which was inert in CI (the CI agent bootstrap exports UCX_TLS before pytest starts) and broke local runs on clusters whose container runtime injects UCX_TLS/UCX_NET_DEVICES defaults. Un-waive the affected case. Validation: 8 concurrent whole-file runs x 5 iterations (40 runs) on one H100 node under CI-parity env all passed; before the fix the same setup timed out in the first iteration on every GPU. Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 4 +--- .../disaggregated/test_kv_transfer.py | 20 +++++++++++++++++-- .../disaggregated/test_kv_transfer_mp.py | 9 ++++++++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 80e697faf3be..066878706ad8 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -363,9 +363,7 @@ unittest/_torch/visual_gen/multi_gpu/test_ulysses_async.py::test_capture_smoke S unittest/_torch/visual_gen/test_attention_integration.py::test_sage_attention_self_attention[fp8-2-1560] SKIP (https://nvbugs/6198760) unittest/_torch/visual_gen/test_attention_integration.py::test_sage_attention_self_attention[int8-2-1560] SKIP (https://nvbugs/6198760) unittest/auto_deploy/multigpu/custom_ops SKIP (https://nvbugs/6403920) -unittest/disaggregated/test_kv_transfer.py SKIP (https://nvbugs/6403793) -unittest/disaggregated/test_kv_transfer.py::test_transfer_worker_v2[tp1_pp4_to_tp2_pp2] SKIP (https://nvbugs/6445316) -unittest/disaggregated/test_kv_transfer.py::test_transfer_worker_v2[tp4_pp1_to_tp2_pp2] SKIP (https://nvbugs/6426834) +unittest/executor/test_proxy_fast_death.py::test_pool_session_shutdown_never_blocks_after_release SKIP (https://nvbugs/6480574) unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) diff --git a/tests/unittest/disaggregated/test_kv_transfer.py b/tests/unittest/disaggregated/test_kv_transfer.py index 76a0d8db25b3..8a8c13f34f48 100644 --- a/tests/unittest/disaggregated/test_kv_transfer.py +++ b/tests/unittest/disaggregated/test_kv_transfer.py @@ -5,8 +5,24 @@ import time import uuid -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX config regardless of what the cluster/CI injects +# (enroot may inject UCX_TLS=rc_v,sm + UCX_NET_DEVICES=mlx5_*; the CI agent +# bootstrap exports UCX_TLS=tcp,cuda_copy,cuda_ipc): +# - exclude IB (no fabric assumed) and gdr_copy (UCX rcache SIGABRT at +# teardown); +# - drop UCX_NET_DEVICES, otherwise with ^ib the remaining transports cannot +# use the injected mlx5_* device list and UCX fails with "no active +# messages transport". +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +# Each NIXL agent spawns TRTLLM_NIXL_NUM_THREADS (default 8) busy-polling +# progress threads, and a single case builds up to 8 TransferWorkers (one per +# rank). On CI nodes shared with other single-GPU jobs the resulting CPU +# oversubscription inflates agent construction from ~3s to ~30s each, blowing +# the 120s per-test timeout intermittently (https://nvbugs/6426834). One +# progress thread is enough here: these tests verify transfer logic, not +# transfer-engine threading. +os.environ.setdefault("TRTLLM_NIXL_NUM_THREADS", "1") from dataclasses import dataclass from typing import List, Optional diff --git a/tests/unittest/disaggregated/test_kv_transfer_mp.py b/tests/unittest/disaggregated/test_kv_transfer_mp.py index 76e5f4aa7573..797a582526bb 100644 --- a/tests/unittest/disaggregated/test_kv_transfer_mp.py +++ b/tests/unittest/disaggregated/test_kv_transfer_mp.py @@ -8,7 +8,14 @@ import torch.multiprocessing as mp # Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX config regardless of what the cluster/CI injects; +# see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +# Limit NIXL busy-polling progress threads; see test_kv_transfer.py for the +# full rationale (intermittent 120s timeouts on shared CI nodes, +# https://nvbugs/6426834). +os.environ.setdefault("TRTLLM_NIXL_NUM_THREADS", "1") import tensorrt_llm import tensorrt_llm.bindings From fe0f929e028c31c1f42dd5c8efafcd6f57aec24b Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:25:25 +0000 Subject: [PATCH 2/4] [https://nvbugs/6426834][fix] Pin UCX/NIXL env in all transceiver unit tests Address review: assign TRTLLM_NIXL_NUM_THREADS explicitly instead of setdefault so an externally supplied value cannot defeat the single-progress-thread cap, and apply the same deterministic env (UCX_TLS=^ib,gdr_copy, drop UCX_NET_DEVICES, TRTLLM_NIXL_NUM_THREADS=1) to every unit test that creates real UCX/NIXL transfer agents, not just test_kv_transfer*.py. Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- tests/unittest/disaggregated/test_agent.py | 7 +++++-- .../unittest/disaggregated/test_agent_multi_backends.py | 8 ++++++-- .../test_cache_transceiver_single_process.py | 9 +++++---- tests/unittest/disaggregated/test_kv_transfer.py | 2 +- tests/unittest/disaggregated/test_kv_transfer_mp.py | 2 +- tests/unittest/disaggregated/test_mamba_transfer.py | 7 +++++++ .../disaggregated/test_py_cache_transceiver_mp.py | 7 +++++-- tests/unittest/others/test_kv_cache_transceiver.py | 7 +++++++ 8 files changed, 37 insertions(+), 12 deletions(-) diff --git a/tests/unittest/disaggregated/test_agent.py b/tests/unittest/disaggregated/test_agent.py index f2212a6c5f82..ad4688ef2f0d 100644 --- a/tests/unittest/disaggregated/test_agent.py +++ b/tests/unittest/disaggregated/test_agent.py @@ -6,8 +6,11 @@ import pytest import torch -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from tensorrt_llm import logger from tensorrt_llm._torch.disaggregation.base.agent import ( diff --git a/tests/unittest/disaggregated/test_agent_multi_backends.py b/tests/unittest/disaggregated/test_agent_multi_backends.py index d983af152542..c8fa0d0b85d6 100644 --- a/tests/unittest/disaggregated/test_agent_multi_backends.py +++ b/tests/unittest/disaggregated/test_agent_multi_backends.py @@ -3,8 +3,12 @@ import pytest -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. The subprocesses +# spawned below inherit these via os.environ.copy(). +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" def test_load_agent_missing_module(): diff --git a/tests/unittest/disaggregated/test_cache_transceiver_single_process.py b/tests/unittest/disaggregated/test_cache_transceiver_single_process.py index 651a347b2f08..4d3ba3779c0d 100644 --- a/tests/unittest/disaggregated/test_cache_transceiver_single_process.py +++ b/tests/unittest/disaggregated/test_cache_transceiver_single_process.py @@ -28,7 +28,11 @@ # Exclude UCX IB transport (avoid NIXL setup hangs without IB) and gdr_copy # (avoid SIGSEGV at process exit from UCX rcache cleanup; gdr_copy disabled # falls back to cuda_ipc / cuda_copy without affecting correctness). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from dataclasses import dataclass from typing import Dict, List, Optional @@ -55,9 +59,6 @@ AttentionTypeCpp = tensorrt_llm.bindings.internal.batch_manager.AttentionType -# Reduce NIXL threads for unit test: default 8 threads per agent causes heavy -# contention when creating multiple agents on a single GPU in the same process. -os.environ.setdefault("TRTLLM_NIXL_NUM_THREADS", "0") # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- diff --git a/tests/unittest/disaggregated/test_kv_transfer.py b/tests/unittest/disaggregated/test_kv_transfer.py index 8a8c13f34f48..821fcc431c03 100644 --- a/tests/unittest/disaggregated/test_kv_transfer.py +++ b/tests/unittest/disaggregated/test_kv_transfer.py @@ -22,7 +22,7 @@ # the 120s per-test timeout intermittently (https://nvbugs/6426834). One # progress thread is enough here: these tests verify transfer logic, not # transfer-engine threading. -os.environ.setdefault("TRTLLM_NIXL_NUM_THREADS", "1") +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from dataclasses import dataclass from typing import List, Optional diff --git a/tests/unittest/disaggregated/test_kv_transfer_mp.py b/tests/unittest/disaggregated/test_kv_transfer_mp.py index 797a582526bb..d58315e1a93a 100644 --- a/tests/unittest/disaggregated/test_kv_transfer_mp.py +++ b/tests/unittest/disaggregated/test_kv_transfer_mp.py @@ -15,7 +15,7 @@ # Limit NIXL busy-polling progress threads; see test_kv_transfer.py for the # full rationale (intermittent 120s timeouts on shared CI nodes, # https://nvbugs/6426834). -os.environ.setdefault("TRTLLM_NIXL_NUM_THREADS", "1") +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" import tensorrt_llm import tensorrt_llm.bindings diff --git a/tests/unittest/disaggregated/test_mamba_transfer.py b/tests/unittest/disaggregated/test_mamba_transfer.py index 0cf0b3209899..f844a31597b9 100644 --- a/tests/unittest/disaggregated/test_mamba_transfer.py +++ b/tests/unittest/disaggregated/test_mamba_transfer.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import threading import uuid from typing import Dict, List @@ -19,6 +20,12 @@ import pytest import torch +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" + import tensorrt_llm import tensorrt_llm.bindings import tensorrt_llm.tensorrt_llm_transfer_agent_binding # noqa: F401 diff --git a/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py b/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py index a81e05026cbb..50d7a06d0012 100644 --- a/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py +++ b/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py @@ -15,8 +15,11 @@ import torch.distributed as dist import torch.multiprocessing as mp -# Exclude IB (no fabric) and gdr_copy (UCX rcache SIGABRT at teardown). -os.environ.setdefault("UCX_TLS", "^ib,gdr_copy") +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" import tensorrt_llm import tensorrt_llm.bindings diff --git a/tests/unittest/others/test_kv_cache_transceiver.py b/tests/unittest/others/test_kv_cache_transceiver.py index 0d6b31e367d1..86dc2ad6dd7c 100644 --- a/tests/unittest/others/test_kv_cache_transceiver.py +++ b/tests/unittest/others/test_kv_cache_transceiver.py @@ -3,6 +3,7 @@ import gc import multiprocessing +import os import sys import time import uuid @@ -11,6 +12,12 @@ import pytest import torch +# Force a deterministic UCX/NIXL config regardless of what the cluster/CI +# injects; see disaggregated/test_kv_transfer.py for the full rationale. +os.environ["UCX_TLS"] = "^ib,gdr_copy" +os.environ.pop("UCX_NET_DEVICES", None) +os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" + import tensorrt_llm import tensorrt_llm.bindings import tensorrt_llm.bindings.executor as trtllm From fd6764de7292d1c5e3e513a5f6d4a2ad3f9c7d7f Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:28:02 +0000 Subject: [PATCH 3/4] [https://nvbugs/6426834][fix] Drop UCX_NET_DEVICES unsetting from transceiver unit tests Keep only UCX_TLS pinning and the NIXL progress-thread cap; removing UCX_NET_DEVICES from the environment is not needed. Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- tests/unittest/disaggregated/test_agent.py | 1 - .../disaggregated/test_agent_multi_backends.py | 1 - .../test_cache_transceiver_single_process.py | 1 - tests/unittest/disaggregated/test_kv_transfer.py | 11 +++-------- tests/unittest/disaggregated/test_kv_transfer_mp.py | 1 - tests/unittest/disaggregated/test_mamba_transfer.py | 1 - .../disaggregated/test_py_cache_transceiver_mp.py | 1 - tests/unittest/others/test_kv_cache_transceiver.py | 1 - 8 files changed, 3 insertions(+), 15 deletions(-) diff --git a/tests/unittest/disaggregated/test_agent.py b/tests/unittest/disaggregated/test_agent.py index ad4688ef2f0d..c8425ab49e18 100644 --- a/tests/unittest/disaggregated/test_agent.py +++ b/tests/unittest/disaggregated/test_agent.py @@ -9,7 +9,6 @@ # Force a deterministic UCX/NIXL config regardless of what the cluster/CI # injects; see test_kv_transfer.py for the full rationale. os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from tensorrt_llm import logger diff --git a/tests/unittest/disaggregated/test_agent_multi_backends.py b/tests/unittest/disaggregated/test_agent_multi_backends.py index c8fa0d0b85d6..d82815c99a79 100644 --- a/tests/unittest/disaggregated/test_agent_multi_backends.py +++ b/tests/unittest/disaggregated/test_agent_multi_backends.py @@ -7,7 +7,6 @@ # injects; see test_kv_transfer.py for the full rationale. The subprocesses # spawned below inherit these via os.environ.copy(). os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" diff --git a/tests/unittest/disaggregated/test_cache_transceiver_single_process.py b/tests/unittest/disaggregated/test_cache_transceiver_single_process.py index 4d3ba3779c0d..26b9e2e9199d 100644 --- a/tests/unittest/disaggregated/test_cache_transceiver_single_process.py +++ b/tests/unittest/disaggregated/test_cache_transceiver_single_process.py @@ -31,7 +31,6 @@ # Force a deterministic UCX/NIXL config regardless of what the cluster/CI # injects; see test_kv_transfer.py for the full rationale. os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" from dataclasses import dataclass from typing import Dict, List, Optional diff --git a/tests/unittest/disaggregated/test_kv_transfer.py b/tests/unittest/disaggregated/test_kv_transfer.py index 821fcc431c03..c2cfc1382e37 100644 --- a/tests/unittest/disaggregated/test_kv_transfer.py +++ b/tests/unittest/disaggregated/test_kv_transfer.py @@ -6,15 +6,10 @@ import uuid # Force a deterministic UCX config regardless of what the cluster/CI injects -# (enroot may inject UCX_TLS=rc_v,sm + UCX_NET_DEVICES=mlx5_*; the CI agent -# bootstrap exports UCX_TLS=tcp,cuda_copy,cuda_ipc): -# - exclude IB (no fabric assumed) and gdr_copy (UCX rcache SIGABRT at -# teardown); -# - drop UCX_NET_DEVICES, otherwise with ^ib the remaining transports cannot -# use the injected mlx5_* device list and UCX fails with "no active -# messages transport". +# (the CI agent bootstrap exports UCX_TLS=tcp,cuda_copy,cuda_ipc before pytest +# starts, which a setdefault would leave in place): exclude IB (no fabric +# assumed) and gdr_copy (UCX rcache SIGABRT at teardown). os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) # Each NIXL agent spawns TRTLLM_NIXL_NUM_THREADS (default 8) busy-polling # progress threads, and a single case builds up to 8 TransferWorkers (one per # rank). On CI nodes shared with other single-GPU jobs the resulting CPU diff --git a/tests/unittest/disaggregated/test_kv_transfer_mp.py b/tests/unittest/disaggregated/test_kv_transfer_mp.py index d58315e1a93a..a52d95bde7a4 100644 --- a/tests/unittest/disaggregated/test_kv_transfer_mp.py +++ b/tests/unittest/disaggregated/test_kv_transfer_mp.py @@ -11,7 +11,6 @@ # Force a deterministic UCX config regardless of what the cluster/CI injects; # see test_kv_transfer.py for the full rationale. os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) # Limit NIXL busy-polling progress threads; see test_kv_transfer.py for the # full rationale (intermittent 120s timeouts on shared CI nodes, # https://nvbugs/6426834). diff --git a/tests/unittest/disaggregated/test_mamba_transfer.py b/tests/unittest/disaggregated/test_mamba_transfer.py index f844a31597b9..8688d75f9a53 100644 --- a/tests/unittest/disaggregated/test_mamba_transfer.py +++ b/tests/unittest/disaggregated/test_mamba_transfer.py @@ -23,7 +23,6 @@ # Force a deterministic UCX/NIXL config regardless of what the cluster/CI # injects; see test_kv_transfer.py for the full rationale. os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" import tensorrt_llm diff --git a/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py b/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py index 50d7a06d0012..d414d18fb48b 100644 --- a/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py +++ b/tests/unittest/disaggregated/test_py_cache_transceiver_mp.py @@ -18,7 +18,6 @@ # Force a deterministic UCX/NIXL config regardless of what the cluster/CI # injects; see test_kv_transfer.py for the full rationale. os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" import tensorrt_llm diff --git a/tests/unittest/others/test_kv_cache_transceiver.py b/tests/unittest/others/test_kv_cache_transceiver.py index 86dc2ad6dd7c..241a4aee0d09 100644 --- a/tests/unittest/others/test_kv_cache_transceiver.py +++ b/tests/unittest/others/test_kv_cache_transceiver.py @@ -15,7 +15,6 @@ # Force a deterministic UCX/NIXL config regardless of what the cluster/CI # injects; see disaggregated/test_kv_transfer.py for the full rationale. os.environ["UCX_TLS"] = "^ib,gdr_copy" -os.environ.pop("UCX_NET_DEVICES", None) os.environ["TRTLLM_NIXL_NUM_THREADS"] = "1" import tensorrt_llm From 127e7f4b0b3e9e79bdb0a112832eb175198487f5 Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Fri, 24 Jul 2026 02:43:35 +0000 Subject: [PATCH 4/4] [https://nvbugs/6426834][chore] Drop stale test_proxy_fast_death waive re-added during rebase (fixed by #16630) Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 066878706ad8..c02464b7ecfb 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -363,7 +363,6 @@ unittest/_torch/visual_gen/multi_gpu/test_ulysses_async.py::test_capture_smoke S unittest/_torch/visual_gen/test_attention_integration.py::test_sage_attention_self_attention[fp8-2-1560] SKIP (https://nvbugs/6198760) unittest/_torch/visual_gen/test_attention_integration.py::test_sage_attention_self_attention[int8-2-1560] SKIP (https://nvbugs/6198760) unittest/auto_deploy/multigpu/custom_ops SKIP (https://nvbugs/6403920) -unittest/executor/test_proxy_fast_death.py::test_pool_session_shutdown_never_blocks_after_release SKIP (https://nvbugs/6480574) unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741)