From 447ab0307596fc042aeea775f8bc566b8b50e992 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Fri, 24 Jul 2026 04:56:43 +0000 Subject: [PATCH] [TRTLLM-14475][chore] Self-sufficient transfer-agent dlopen and drop onnx/modelopt deps Trailing cleanup after the TensorRT backend removal, in two parts. 1) Make the KV-cache transfer-agent wrapper dlopen self-sufficient. The wrapper libraries (libtensorrt_llm_{nixl,ucx,mooncake}_wrapper.so) carry no DT_NEEDED on libtensorrt_llm.so (circular) and resolve its symbols from the process global scope. In the wheel they only worked because tensorrt_llm/_common.py preloaded libtensorrt_llm.so with ctypes RTLD_GLOBAL - a stopgap added when the removed TensorRT plugin's RTLD_GLOBAL load stopped providing this as a side effect. New executor::kv_cache::promoteHostLibraryToGlobalScope() (dladdr-self + RTLD_NOW|RTLD_GLOBAL|RTLD_NOLOAD, call_once, no-op when statically linked) is called at both wrapper-load sites (DynLibLoader::getHandle and the UCX dllOpen), and the _common.py preload is removed. 2) Drop the onnx and nvidia-modelopt dependencies. - onnx / onnx_graphsurgeon: TensorRT ONNX-engine-build graph tools with zero consumers repo-wide. Legacy relics of the removed engine path. - nvidia-modelopt[torch]: removed as a dependency and its AutoDeploy package import sites stripped (all four were try/except-guarded). The main PyTorch backend never imported the package. PRESERVED (package free): loading pre-quantized FP8/NVFP4 checkpoints via models/quant_config_reader.py + _compat.py + the local modelopt_fp4_scale_to_cutlass_fp4_scale, which read the modelopt *checkpoint format* and never imported the package. DISABLED: loading graphs with live modelopt fake-quant ops (is_quantized_graph=True) and the modelopt export context. Guarded test helpers degrade gracefully, so test collection is unaffected; examples/auto_deploy/ build_and_run_flux.py hard-imports the package and will not run without it (out of scope here). Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../tensorrt_llm/executor/transferAgent.h | 8 ++++ .../batch_manager/cacheTransceiver.cpp | 2 + .../cache_transmission/transferAgent.cpp | 29 +++++++++++++++ requirements.txt | 3 -- tensorrt_llm/_common.py | 11 ------ tensorrt_llm/_torch/auto_deploy/__init__.py | 6 --- .../_torch/auto_deploy/export/export.py | 6 +-- .../export/library/modelopt_context.py | 37 ------------------- .../_torch/auto_deploy/utils/node_utils.py | 16 ++------ 9 files changed, 44 insertions(+), 74 deletions(-) delete mode 100644 tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py diff --git a/cpp/include/tensorrt_llm/executor/transferAgent.h b/cpp/include/tensorrt_llm/executor/transferAgent.h index 5f175a33723c..e94f636a8312 100644 --- a/cpp/include/tensorrt_llm/executor/transferAgent.h +++ b/cpp/include/tensorrt_llm/executor/transferAgent.h @@ -444,6 +444,14 @@ class BaseLoopbackAgent virtual void executeLoopbackRequest(MemoryDescs const& memoryDescs, FileDescs const& fileDescs, bool isOffload) = 0; }; +/// @brief Promote the shared library containing this code (libtensorrt_llm.so) to the +/// process's global symbol scope. The KV cache transfer-agent wrapper libraries +/// (libtensorrt_llm_{nixl,ucx,mooncake}_wrapper.so) intentionally carry no DT_NEEDED on +/// libtensorrt_llm.so (the dependency would be circular) and resolve its symbols from the +/// global symbol table, while Python extension modules and their dependencies load with +/// RTLD_LOCAL. Idempotent; a no-op when the code is statically linked (e.g. unit tests). +void promoteHostLibraryToGlobalScope(); + class DynLibLoader final { public: diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp index 89e6bd5d1b97..014e2726a22c 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp @@ -53,6 +53,7 @@ #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/serialization.h" #include "tensorrt_llm/executor/serializeUtils.h" +#include "tensorrt_llm/executor/transferAgent.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" #include @@ -501,6 +502,7 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa if (backendType.value() == executor::CacheTransceiverConfig::BackendType::UCX) { std::lock_guard lock(mDllMutex); + executor::kv_cache::promoteHostLibraryToGlobalScope(); mWrapperLibHandle = dllOpen(UCX_WRAPPER_LIB_NAME); TLLM_CHECK_WITH_INFO( mWrapperLibHandle != nullptr, "UCX wrapper library is not open correctly. error : %s", dlerror()); diff --git a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp index 147c99b37329..9c5aaaac78ae 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,33 @@ namespace tensorrt_llm::executor::kv_cache { +void promoteHostLibraryToGlobalScope() +{ + static std::once_flag once; + std::call_once(once, + []() + { + Dl_info info{}; + if (dladdr(reinterpret_cast(&promoteHostLibraryToGlobalScope), &info) == 0 + || info.dli_fname == nullptr) + { + TLLM_LOG_DEBUG("dladdr failed; skipping global-scope promotion"); + return; + } + // RTLD_NOLOAD promotes the visibility of the already-loaded library without + // loading a second copy. It fails when this code is statically linked into an + // executable (e.g. unit tests) — nothing to promote there. + void* handle = dlopen(info.dli_fname, RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD); + if (handle == nullptr) + { + TLLM_LOG_DEBUG("global-scope promotion skipped for %s: %s", info.dli_fname, dlerror()); + return; + } + TLLM_LOG_DEBUG("promoted %s to the global symbol scope", info.dli_fname); + // Keep the extra reference: the library must stay resident for the wrappers. + }); +} + [[nodiscard]] DynLibLoader& DynLibLoader::getInstance() { static DynLibLoader instance; @@ -37,6 +65,7 @@ namespace tensorrt_llm::executor::kv_cache [[nodiscard]] void* DynLibLoader::getHandle(std::string const& name) { + promoteHostLibraryToGlobalScope(); std::lock_guard lock(mDllMutex); auto it = mHandlers.find(name); if (it != mHandlers.end()) diff --git a/requirements.txt b/requirements.txt index 663b5829012a..e4c6f6be1678 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,6 @@ lark lazy_loader~=0.5 mpi4py numpy>=2.0.0,<2.4 # numba 0.63.1 requires numpy<2.4 -onnx>=1.21.0 -onnx_graphsurgeon>=0.5.2 graphviz openai polygraphy @@ -25,7 +23,6 @@ sentencepiece>=0.1.99 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.12.0a0. torch>=2.11.0,<=2.13.0a0 torchvision -nvidia-modelopt[torch]~=0.37.0 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.30.4 # torch 2.11.0+cu130 depends on nvidia-nccl-cu13==2.28.9 nvidia-nccl-cu13>=2.28.9,<=2.30.4 diff --git a/tensorrt_llm/_common.py b/tensorrt_llm/_common.py index d7feeededb9f..8dc41df25fcb 100644 --- a/tensorrt_llm/_common.py +++ b/tensorrt_llm/_common.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ctypes import os import platform import threading @@ -45,16 +44,6 @@ def _init(log_level: object = None) -> None: project_dir = str(Path(__file__).parent.absolute()) - # Promote libtensorrt_llm.so symbols to the process's global scope. The KV - # cache transfer agent wrappers (libtensorrt_llm_nixl_wrapper.so, - # libtensorrt_llm_ucx_wrapper.so) are dlopen'ed at runtime without linking - # against libtensorrt_llm.so and resolve its symbols from the global symbol - # table, while Python extension modules and their dependencies load with - # RTLD_LOCAL. This promotion was previously a side effect of loading the - # TensorRT plugin library with RTLD_GLOBAL. - if platform.system() != "Windows": - ctypes.CDLL(project_dir + "/libs/libtensorrt_llm.so", mode=ctypes.RTLD_GLOBAL) - # Load FT decoder layer and torch custom ops. if platform.system() == "Windows": ft_decoder_lib = project_dir + "/libs/th_common.dll" diff --git a/tensorrt_llm/_torch/auto_deploy/__init__.py b/tensorrt_llm/_torch/auto_deploy/__init__.py index a8a22a945c05..bbd1d6398854 100644 --- a/tensorrt_llm/_torch/auto_deploy/__init__.py +++ b/tensorrt_llm/_torch/auto_deploy/__init__.py @@ -37,9 +37,3 @@ # import AutoDeploy LLM and LlmArgs (require TRT-LLM base classes) from .llm import * from .llm_args import * - - try: - # This will overwrite the AutoModelForCausalLM.from_config to support modelopt quantization - import modelopt - except ImportError: - pass diff --git a/tensorrt_llm/_torch/auto_deploy/export/export.py b/tensorrt_llm/_torch/auto_deploy/export/export.py index 09a54b3c46e8..c7f0a5de205a 100644 --- a/tensorrt_llm/_torch/auto_deploy/export/export.py +++ b/tensorrt_llm/_torch/auto_deploy/export/export.py @@ -32,10 +32,8 @@ from ..utils.pipeline_cache_hooks import mark_pipeline_cache_hook from .interface import apply_export_patches -try: - from modelopt.torch.quantization.utils import export_torch_mode as torch_export_context -except ImportError: - torch_export_context = nullcontext +# modelopt quantization support has been removed; use a null export context. +torch_export_context = nullcontext # ===================================================================== diff --git a/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py b/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py deleted file mode 100644 index 1a0df9c41119..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Patch for modelopt's torch_export_context.""" - -from contextlib import nullcontext - -from ..interface import ContextManagerPatch, ExportPatchRegistry - - -@ExportPatchRegistry.register("modelopt_context") -class ModeloptContextPatch(ContextManagerPatch): - """Patch to apply modelopt's torch_export_context during export. - - This patch applies the modelopt quantization context manager around - the export process when available, otherwise uses a null context. - """ - - def init_context_manager(self): - """Initialize and return the modelopt context manager or nullcontext if not available.""" - try: - from modelopt.torch.quantization.utils import export_torch_mode as torch_export_context - - return torch_export_context() - except ImportError: - return nullcontext() diff --git a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py index cace2f37e603..198cdab433f3 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py @@ -28,19 +28,9 @@ from .logger import ad_logger -try: - # import modelopt to get quantize_op - from modelopt.torch.quantization import tensor_quant # noqa: F401 - - if hasattr(torch.ops, "tensorrt"): - modelopt_quantize_op = torch.ops.tensorrt.quantize_op - modelopt_dynamic_block_quantize_op = torch.ops.tensorrt.dynamic_block_quantize_op - else: - modelopt_quantize_op = None - modelopt_dynamic_block_quantize_op = None -except ImportError: - modelopt_quantize_op = None - modelopt_dynamic_block_quantize_op = None +# modelopt quantization support has been removed; these ops are no longer available. +modelopt_quantize_op = None +modelopt_dynamic_block_quantize_op = None OpOrOverload = Union[OpOverloadPacket, OpOverload] OperatorLike = Union[OpOrOverload, Callable]