diff --git a/cpp/include/tensorrt_llm/executor/transferAgent.h b/cpp/include/tensorrt_llm/executor/transferAgent.h index 5f175a33723c..e94f636a8312 100644 --- a/cpp/include/tensorrt_llm/executor/transferAgent.h +++ b/cpp/include/tensorrt_llm/executor/transferAgent.h @@ -444,6 +444,14 @@ class BaseLoopbackAgent virtual void executeLoopbackRequest(MemoryDescs const& memoryDescs, FileDescs const& fileDescs, bool isOffload) = 0; }; +/// @brief Promote the shared library containing this code (libtensorrt_llm.so) to the +/// process's global symbol scope. The KV cache transfer-agent wrapper libraries +/// (libtensorrt_llm_{nixl,ucx,mooncake}_wrapper.so) intentionally carry no DT_NEEDED on +/// libtensorrt_llm.so (the dependency would be circular) and resolve its symbols from the +/// global symbol table, while Python extension modules and their dependencies load with +/// RTLD_LOCAL. Idempotent; a no-op when the code is statically linked (e.g. unit tests). +void promoteHostLibraryToGlobalScope(); + class DynLibLoader final { public: diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp index 89e6bd5d1b97..014e2726a22c 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp @@ -53,6 +53,7 @@ #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/serialization.h" #include "tensorrt_llm/executor/serializeUtils.h" +#include "tensorrt_llm/executor/transferAgent.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" #include @@ -501,6 +502,7 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa if (backendType.value() == executor::CacheTransceiverConfig::BackendType::UCX) { std::lock_guard lock(mDllMutex); + executor::kv_cache::promoteHostLibraryToGlobalScope(); mWrapperLibHandle = dllOpen(UCX_WRAPPER_LIB_NAME); TLLM_CHECK_WITH_INFO( mWrapperLibHandle != nullptr, "UCX wrapper library is not open correctly. error : %s", dlerror()); diff --git a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp index 147c99b37329..9c5aaaac78ae 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,33 @@ namespace tensorrt_llm::executor::kv_cache { +void promoteHostLibraryToGlobalScope() +{ + static std::once_flag once; + std::call_once(once, + []() + { + Dl_info info{}; + if (dladdr(reinterpret_cast(&promoteHostLibraryToGlobalScope), &info) == 0 + || info.dli_fname == nullptr) + { + TLLM_LOG_DEBUG("dladdr failed; skipping global-scope promotion"); + return; + } + // RTLD_NOLOAD promotes the visibility of the already-loaded library without + // loading a second copy. It fails when this code is statically linked into an + // executable (e.g. unit tests) — nothing to promote there. + void* handle = dlopen(info.dli_fname, RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD); + if (handle == nullptr) + { + TLLM_LOG_DEBUG("global-scope promotion skipped for %s: %s", info.dli_fname, dlerror()); + return; + } + TLLM_LOG_DEBUG("promoted %s to the global symbol scope", info.dli_fname); + // Keep the extra reference: the library must stay resident for the wrappers. + }); +} + [[nodiscard]] DynLibLoader& DynLibLoader::getInstance() { static DynLibLoader instance; @@ -37,6 +65,7 @@ namespace tensorrt_llm::executor::kv_cache [[nodiscard]] void* DynLibLoader::getHandle(std::string const& name) { + promoteHostLibraryToGlobalScope(); std::lock_guard lock(mDllMutex); auto it = mHandlers.find(name); if (it != mHandlers.end()) diff --git a/requirements.txt b/requirements.txt index 663b5829012a..e4c6f6be1678 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,6 @@ lark lazy_loader~=0.5 mpi4py numpy>=2.0.0,<2.4 # numba 0.63.1 requires numpy<2.4 -onnx>=1.21.0 -onnx_graphsurgeon>=0.5.2 graphviz openai polygraphy @@ -25,7 +23,6 @@ sentencepiece>=0.1.99 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.12.0a0. torch>=2.11.0,<=2.13.0a0 torchvision -nvidia-modelopt[torch]~=0.37.0 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.30.4 # torch 2.11.0+cu130 depends on nvidia-nccl-cu13==2.28.9 nvidia-nccl-cu13>=2.28.9,<=2.30.4 diff --git a/tensorrt_llm/_common.py b/tensorrt_llm/_common.py index d7feeededb9f..8dc41df25fcb 100644 --- a/tensorrt_llm/_common.py +++ b/tensorrt_llm/_common.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ctypes import os import platform import threading @@ -45,16 +44,6 @@ def _init(log_level: object = None) -> None: project_dir = str(Path(__file__).parent.absolute()) - # Promote libtensorrt_llm.so symbols to the process's global scope. The KV - # cache transfer agent wrappers (libtensorrt_llm_nixl_wrapper.so, - # libtensorrt_llm_ucx_wrapper.so) are dlopen'ed at runtime without linking - # against libtensorrt_llm.so and resolve its symbols from the global symbol - # table, while Python extension modules and their dependencies load with - # RTLD_LOCAL. This promotion was previously a side effect of loading the - # TensorRT plugin library with RTLD_GLOBAL. - if platform.system() != "Windows": - ctypes.CDLL(project_dir + "/libs/libtensorrt_llm.so", mode=ctypes.RTLD_GLOBAL) - # Load FT decoder layer and torch custom ops. if platform.system() == "Windows": ft_decoder_lib = project_dir + "/libs/th_common.dll" diff --git a/tensorrt_llm/_torch/auto_deploy/__init__.py b/tensorrt_llm/_torch/auto_deploy/__init__.py index a8a22a945c05..bbd1d6398854 100644 --- a/tensorrt_llm/_torch/auto_deploy/__init__.py +++ b/tensorrt_llm/_torch/auto_deploy/__init__.py @@ -37,9 +37,3 @@ # import AutoDeploy LLM and LlmArgs (require TRT-LLM base classes) from .llm import * from .llm_args import * - - try: - # This will overwrite the AutoModelForCausalLM.from_config to support modelopt quantization - import modelopt - except ImportError: - pass diff --git a/tensorrt_llm/_torch/auto_deploy/export/export.py b/tensorrt_llm/_torch/auto_deploy/export/export.py index 09a54b3c46e8..c7f0a5de205a 100644 --- a/tensorrt_llm/_torch/auto_deploy/export/export.py +++ b/tensorrt_llm/_torch/auto_deploy/export/export.py @@ -32,10 +32,8 @@ from ..utils.pipeline_cache_hooks import mark_pipeline_cache_hook from .interface import apply_export_patches -try: - from modelopt.torch.quantization.utils import export_torch_mode as torch_export_context -except ImportError: - torch_export_context = nullcontext +# modelopt quantization support has been removed; use a null export context. +torch_export_context = nullcontext # ===================================================================== diff --git a/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py b/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py deleted file mode 100644 index 1a0df9c41119..000000000000 --- a/tensorrt_llm/_torch/auto_deploy/export/library/modelopt_context.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Patch for modelopt's torch_export_context.""" - -from contextlib import nullcontext - -from ..interface import ContextManagerPatch, ExportPatchRegistry - - -@ExportPatchRegistry.register("modelopt_context") -class ModeloptContextPatch(ContextManagerPatch): - """Patch to apply modelopt's torch_export_context during export. - - This patch applies the modelopt quantization context manager around - the export process when available, otherwise uses a null context. - """ - - def init_context_manager(self): - """Initialize and return the modelopt context manager or nullcontext if not available.""" - try: - from modelopt.torch.quantization.utils import export_torch_mode as torch_export_context - - return torch_export_context() - except ImportError: - return nullcontext() diff --git a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py index cace2f37e603..198cdab433f3 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py @@ -28,19 +28,9 @@ from .logger import ad_logger -try: - # import modelopt to get quantize_op - from modelopt.torch.quantization import tensor_quant # noqa: F401 - - if hasattr(torch.ops, "tensorrt"): - modelopt_quantize_op = torch.ops.tensorrt.quantize_op - modelopt_dynamic_block_quantize_op = torch.ops.tensorrt.dynamic_block_quantize_op - else: - modelopt_quantize_op = None - modelopt_dynamic_block_quantize_op = None -except ImportError: - modelopt_quantize_op = None - modelopt_dynamic_block_quantize_op = None +# modelopt quantization support has been removed; these ops are no longer available. +modelopt_quantize_op = None +modelopt_dynamic_block_quantize_op = None OpOrOverload = Union[OpOverloadPacket, OpOverload] OperatorLike = Union[OpOrOverload, Callable]