diff --git a/examples/llm-api/quickstart_advanced.py b/examples/llm-api/quickstart_advanced.py index d2acd490d44a..bb06009bdc33 100644 --- a/examples/llm-api/quickstart_advanced.py +++ b/examples/llm-api/quickstart_advanced.py @@ -194,6 +194,11 @@ def add_llm_args(parser): parser.add_argument('--apply_chat_template', default=False, action='store_true') + parser.add_argument('--custom_tokenizer', + type=str, + default=None, + help='Override the tokenizer. Accepts a built-in alias ' + " or a fully-qualified class import path.") # Sampling parser.add_argument("--max_tokens", type=int, default=64) @@ -390,6 +395,7 @@ def setup_llm(args, **kwargs): gather_generation_logits=args.return_generation_logits, max_beam_width=args.max_beam_width, orchestrator_type=args.orchestrator_type, + custom_tokenizer=args.custom_tokenizer, **kwargs) use_beam_search = args.max_beam_width > 1 diff --git a/tensorrt_llm/_torch/attention_backend/interface.py b/tensorrt_llm/_torch/attention_backend/interface.py index fec6ba4e6b92..ea6cfbf2f85e 100644 --- a/tensorrt_llm/_torch/attention_backend/interface.py +++ b/tensorrt_llm/_torch/attention_backend/interface.py @@ -674,6 +674,7 @@ def from_config(config) -> "RopeParams": rope_params = RopeParams() hf_rope_parameters = getattr(config, 'rope_parameters', None) + normalized_rope_parameters = hf_rope_parameters if hf_rope_parameters is not None: if set(hf_rope_parameters.keys()).issubset( ALLOWED_ATTENTION_LAYER_TYPES): @@ -681,17 +682,17 @@ def from_config(config) -> "RopeParams": # Pick "full_attention" as the default; callers override theta # for sliding-window layers independently. if "full_attention" in hf_rope_parameters: - flat = hf_rope_parameters["full_attention"] + normalized_rope_parameters = hf_rope_parameters[ + "full_attention"] else: fallback_key = next(iter(hf_rope_parameters)) logger.warning( f"Per-layer-type rope_parameters has no 'full_attention' entry; " f"falling back to '{fallback_key}'. Available layer types: " f"{list(hf_rope_parameters.keys())}.") - flat = hf_rope_parameters[fallback_key] - config.update(flat) - else: - config.update(hf_rope_parameters) + normalized_rope_parameters = hf_rope_parameters[ + fallback_key] + config.update(normalized_rope_parameters) # get rotary parameters. hidden_size = config.hidden_size @@ -700,6 +701,8 @@ def from_config(config) -> "RopeParams": if not isinstance(head_dim, int): head_dim = hidden_size // num_attention_heads rope_scaling = getattr(config, 'rope_scaling', None) + if rope_scaling is None and normalized_rope_parameters is not None: + rope_scaling = normalized_rope_parameters rope_params.max_positions = config.max_position_embeddings rope_params.theta = get_hf_rope_theta(config, 10000.0) rope_percentage = (getattr(config, 'rotary_pct', None) diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 8f5641920c03..dbab6f07b75a 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -494,12 +494,15 @@ def load_hf_quant_config(hf_quant_config, moe_backend, checkpoint_dir=None): # Read exclude_modules from HF config if present (HF format module names) hf_exclude_modules = hf_quant_config.get('modules_to_not_convert', None) - # DeepSeek V3 FP8 ckpt - if hf_quant_config.get("quant_method") == "fp8" and hf_quant_config.get( - "weight_block_size", []): + # FP8 ckpt: DeepSeek V3 style (weight_block_size) or + # per-tensor static activation scale style (activation_scheme="static", + # e.g. Ministral / Pixtral). + if hf_quant_config.get("quant_method") == "fp8" and ( + hf_quant_config.get("weight_block_size") + or hf_quant_config.get("activation_scheme") == "static"): quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES - block_size = hf_quant_config.get("weight_block_size", []) + block_size = hf_quant_config.get("weight_block_size", [128, 128]) assert tuple(block_size) == ( 128, 128), "FP8_BLOCK_SCALES only supports block_size=(128,128)" quant_config.group_size = block_size[0] diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py index cd73f1b8ead0..d77e4fb76ac4 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py @@ -328,7 +328,7 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig: quant_config = QuantConfig() layer_quant_config = None - hf_quant_config = pretrained_config.quantization_config + hf_quant_config = getattr(pretrained_config, "quantization_config", {}) or {} if hf_quant_config.get("quant_method") == "compressed-tensors": if "NVFP4" in hf_quant_config.get("config_groups"): quant_config.quant_algo = QuantAlgo.NVFP4 @@ -390,7 +390,14 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig: from tensorrt_llm._torch.models.modeling_mistral_large3 import Mistral3Gate model_config.pretrained_config.gate_cls = Mistral3Gate - model_config.pretrained_config.input_processor_type = "mistral_large_3" - model_config.pretrained_config.model_type = "mistral_large_3" + # Native (mistral-format) checkpoints are served through the + # mistral-common tokenizer/processor, which applies its own chat + # template. Tag the config with a dedicated serving model_type so the + # serving layer resolves the PASSTHROUGH placeholder/chat-template + # metadata (registered under "mistral_common" in modeling_mistral.py) + # via the normal config path - i.e. resolve_top_level_model_type() - + # instead of having to inspect the live input processor. + model_config.pretrained_config.input_processor_type = "mistral_common" + model_config.pretrained_config.model_type = "mistral_common" model_config._frozen = True return model_config diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py b/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py index eb7ac6b573d3..426e85592ba4 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py @@ -301,4 +301,4 @@ def hf_decode_incrementally( def apply_chat_template( self, conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]], *args, **kwargs ) -> Union[str, list[int], list[str], list[list[int]]]: - raise NotImplementedError + return self.transformers_tokenizer.apply_chat_template(conversation, *args, **kwargs) diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py index dd6e0332b849..e4d50578fe5a 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py @@ -35,6 +35,9 @@ def __init__(self): "tok_embeddings": "model.embed_tokens", "output": "lm_head", "norm": "model.norm", + # For text-only models: preprocess_weights adds "language_model." prefix + "language_model.tok_embeddings": "model.embed_tokens", + "language_model.output": "lm_head", # For Eagle3 "language_model.eagle_linear": "model.fc", "language_model.layers": "layers", diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py index 2246a73f8fdd..35284a91cad7 100644 --- a/tensorrt_llm/_torch/models/modeling_mistral.py +++ b/tensorrt_llm/_torch/models/modeling_mistral.py @@ -39,7 +39,7 @@ from tensorrt_llm._torch.modules.rms_norm import RMSNorm from tensorrt_llm._torch.speculative import SpecMetadata from tensorrt_llm._utils import nvtx_range -from tensorrt_llm.functional import PositionEmbeddingType +from tensorrt_llm.functional import PositionEmbeddingType, RotaryScalingType from tensorrt_llm.inputs import (BaseMultimodalDummyInputsBuilder, BaseMultimodalInputProcessor, ContentFormat, ExtraProcessedInputs, @@ -47,6 +47,7 @@ MultimodalPlaceholderPlacement, TextPrompt, register_input_processor) from tensorrt_llm.inputs.multimodal import MultimodalParams +from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY from tensorrt_llm.inputs.utils import encode_base64_image from tensorrt_llm.llmapi import SamplingParams from tensorrt_llm.logger import logger @@ -61,10 +62,8 @@ def __init__( ): config = model_config.pretrained_config rope_params = RopeParams.from_config(config) - rope_params_section = getattr(config, "rope_scaling", None) or getattr( - config, "rope_parameters", None) - rope_type = getattr(rope_params_section, "rope_type", None) - if rope_type == "yarn": + + if rope_params.scale_type == RotaryScalingType.yarn: pos_embd_params = PositionalEmbeddingParams( type=PositionEmbeddingType.yarn, rope=rope_params, @@ -256,6 +255,18 @@ def __init__( vocab_size=model_config.pretrained_config.vocab_size, ) + def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs): + if weight_mapper and type(weight_mapper) is MistralWeightMapper: + weight_mapper.permute_qk(weights=weights, config=self.config) + super().load_weights(weights, + weight_mapper=weight_mapper, + params_map=weight_mapper.mistral_llm_mapping) + else: + super().load_weights(weights, + weight_mapper=weight_mapper, + *args, + **kwargs) + class MistralCommonImageProcessor: @@ -311,18 +322,10 @@ def get_num_tokens_per_image(self, image_size): return ncols * nrows + nrows def __call__(self, text, images=None, **kwargs): - if not images: - # Plain-text inputs (e.g. text-only evaluation like MMLU/GSM8K): tokenize - # directly without wrapping in a multi-modal chat conversation, which would - # otherwise inject chat-template tokens and corrupt continuation prompts. - encoded = self.tokenizer.transformers_tokenizer(text, - return_tensors='pt') - return {"input_ids": encoded["input_ids"]} - mm_items = [{ "type": "image", "base64": encode_base64_image(image) - } for image in images] + } for image in (images or [])] conversation = [{ "role": "user", @@ -352,18 +355,16 @@ def __call__(self, text, images=None, **kwargs): return processed -class Mistral3InputProcessor(BaseMultimodalInputProcessor, - BaseMultimodalDummyInputsBuilder): +class MistralHFInputProcessor(BaseMultimodalInputProcessor, + BaseMultimodalDummyInputsBuilder): + """Input processor for Mistral VLM checkpoints in HuggingFace format.""" - def __init__( - self, - model_path: str, - config: PretrainedConfig, - tokenizer: AutoTokenizer | None, - trust_remote_code: bool = False, - model_type: str = "mistral3", - **kwargs, - ): + def __init__(self, + model_path: str, + config: PretrainedConfig, + tokenizer: AutoTokenizer, + trust_remote_code: bool = True, + **kwargs): super().__init__(model_path=model_path, config=config, tokenizer=tokenizer, @@ -371,27 +372,19 @@ def __init__( **kwargs) self._config = config self._dtype = self._config.torch_dtype - self._tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained( - model_path, - config=config, - use_fast=self.use_fast, - trust_remote_code=trust_remote_code) self._model_path = model_path - auto_processor = AutoProcessor.from_pretrained( + self._tokenizer = (tokenizer if tokenizer is not None else + AutoTokenizer.from_pretrained( + model_path, + config=config, + use_fast=True, + trust_remote_code=True)) + self._processor = AutoProcessor.from_pretrained( model_path, use_fast=self.use_fast, trust_remote_code=trust_remote_code) - if model_type == "mistral_large_3": - # For mistral large 3, we add chat template in the model forward, and the - # MistralCommonImageProcessor is used to process the input when both text and images are provided. - # When the input only contains text, we use the text processor to process the input. - self._processor = MistralCommonImageProcessor( - tokenizer=self._tokenizer, dtype=self.dtype) - self.text_processor = auto_processor - else: - # For other mistral models, we use the AutoProcessor to process the input. - self._processor = auto_processor - self.text_processor = self._processor + logger.info(f"[mistral] HF processor={type(self._processor).__name__} " + f"tokenizer={type(self._tokenizer).__name__}") @property def config(self) -> PretrainedConfig: @@ -425,17 +418,10 @@ def call_with_text_prompt( # format is "pt" (pytorch tensors), but not for "pil" (PIL images). do_rescale = False - if images is not None: - processed = self.processor( - text=inputs["prompt"], - images=images, - do_rescale=do_rescale, - ) - else: - processed = self.text_processor( - text=inputs["prompt"], - do_rescale=do_rescale, - ) + prompt = inputs["prompt"] + processed = self.processor(text=prompt, + images=images, + do_rescale=do_rescale) input_ids = processed.pop("input_ids").tolist()[0] # Remaining in `processed`: # * "attention_mask": [B, num_input_tokens] @@ -576,7 +562,6 @@ def get_vocab_size(self) -> int: return self.config.text_config.vocab_size def get_mm_token_ids(self) -> torch.Tensor: - """Get the IDs of all multimodal tokens (placeholders and special tokens alike).""" return torch.tensor([ # This is the `[IMG]` token id inserted into the prompt that should be replaced with image # embeddings. @@ -588,73 +573,120 @@ def get_mm_token_ids(self) -> torch.Tensor: ]) def get_mm_special_token_ids(self) -> torch.Tensor: - """Get the IDs of special multimodal tokens (placeholders not included).""" return torch.tensor([ self.processor.image_break_token_id, self.processor.image_end_token_id, ]) -class MistralCommonInputProcessor(Mistral3InputProcessor): +class MistralNativeInputProcessor(BaseMultimodalInputProcessor, + BaseMultimodalDummyInputsBuilder): + """Input processor for Mistral VLM checkpoints in mistral-native format.""" def __init__( self, model_path: str, config: PretrainedConfig, - tokenizer: AutoTokenizer, + tokenizer: AutoTokenizer | None, trust_remote_code: bool = False, **kwargs, ): - tokenizer = self.load_tokenizer(model_path, - config=config, - tokenizer=tokenizer) super().__init__(model_path=model_path, config=config, tokenizer=tokenizer, trust_remote_code=trust_remote_code, - model_type=getattr(config, "input_processor_type", - "mistral3"), **kwargs) + self._config = config + self._dtype = self._config.torch_dtype + self._model_path = model_path + self._tokenizer = MistralTokenizer.from_pretrained(model_path) + self._processor = MistralCommonImageProcessor(tokenizer=self._tokenizer, + dtype=self.dtype) + logger.info( + f"[mistral] native processor={type(self._processor).__name__} " + f"tokenizer={type(self._tokenizer).__name__}") - @staticmethod - def load_tokenizer(model_path: str, - config: PretrainedConfig, - tokenizer: AutoTokenizer | None = None): - if getattr(config, "input_processor_type", None) == "mistral_large_3": - try: - return MistralTokenizer.from_pretrained(model_path) + @property + def config(self) -> PretrainedConfig: + return self._config - except ValueError: - logger.info( - f"Could not load mistral-common tokenizer from {model_path}, falling back to HuggingFace" - ) + @property + def tokenizer(self) -> AutoTokenizer: + return self._tokenizer - tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained( - model_path, config=config, use_fast=True, trust_remote_code=True) - return tokenizer + @property + def model_path(self) -> str: + return self._model_path + @property + def processor(self) -> AutoProcessor: + return self._processor -@register_auto_model("Mistral3ForConditionalGeneration") -@register_auto_model("PixtralForConditionalGeneration") -@register_input_processor( - MistralCommonInputProcessor, - model_type="mistral_large_3", - placeholder_metadata=MultimodalPlaceholderMetadata( - placeholder_map={ - # NOTE: mistral-common uses the tokenizer to set placeholders, this will be ignored - "image": "[IMG]", - }, + @property + def dtype(self) -> torch.dtype: + return self._dtype + + def get_vocab_size(self) -> int: + return self.config.text_config.vocab_size + + def get_mm_token_ids(self) -> torch.Tensor: + return torch.tensor([ + self.processor.image_token_id, + self.processor.image_break_token_id, + self.processor.image_end_token_id, + ]) + + def get_mm_special_token_ids(self) -> torch.Tensor: + return torch.tensor([ + self.processor.image_break_token_id, + self.processor.image_end_token_id, + ]) + + @torch.inference_mode() + def call_with_text_prompt( + self, inputs: TextPrompt, sampling_params: SamplingParams + ) -> Tuple[List[int], ExtraProcessedInputs | None]: + images = inputs.get("multi_modal_data", {}).get("image") + if not images: + # Text-only: tokenize directly without wrapping in a chat template. + # The chat template is either already applied by the caller (serve + # path) or intentionally absent (e.g. raw few-shot eval like MMLU). + input_ids = self.tokenizer.transformers_tokenizer.encode( + inputs["prompt"]) + return input_ids, None + # Multimodal: MistralCommonImageProcessor builds the full conversation + # and applies the mistral-common chat template with image tokens. + processed = self.processor(text=inputs["prompt"], images=images) + input_ids = processed.pop("input_ids").tolist()[0] + processed.pop("attention_mask", None) + processed["image_sizes"] = processed["image_sizes"].tolist() + return input_ids, {"multimodal_data": {"image": {**processed}}} + + +# Register the native processor's content-format metadata. We do this +# directly rather than via @register_input_processor because that decorator +# also writes to INPUT_PROCESSOR_REGISTRY (keyed by model class), which would +# overwrite the MistralHFInputProcessor entry for Mistral3VLM. +# Mistral is the only supported case where HF preprocessor can be used with +# non-HF checkpoints, so this hack is preferred to changing the registry itself. +MULTIMODAL_PLACEHOLDER_REGISTRY.set_placeholder_metadata( + "mistral_common", + MultimodalPlaceholderMetadata( + placeholder_map={"image": "[IMG]"}, placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, content_format=ContentFormat.PASSTHROUGH, )) +MistralNativeInputProcessor._registered_model_type = "mistral_common" + + +@register_auto_model("Mistral3ForConditionalGeneration") +@register_auto_model("PixtralForConditionalGeneration") @register_input_processor( - MistralCommonInputProcessor, + MistralHFInputProcessor, model_type="mistral3", placeholder_metadata=MultimodalPlaceholderMetadata( - placeholder_map={ - "image": "[IMG]", - }, - # NOTE: for mistral3 multimodal models, it does not strictly have to be before the text. + placeholder_map={"image": "[IMG]"}, + # NOTE: for mistral3 multimodal models, placeholder_placement does not strictly have to be before the text. # Ref: https://github.com/mistralai/mistral-common/blob/039465db2bdc0486df36365c9bdb428188482a18/ # src/mistral_common/tokens/tokenizers/base.py#L326 # However, accuracy tests show that the model generates higher quality output when the image @@ -662,6 +694,15 @@ def load_tokenizer(model_path: str, placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, content_format=ContentFormat.STRING, )) +@register_input_processor( + MistralHFInputProcessor, + model_type="mistral_large_3", + placeholder_metadata=MultimodalPlaceholderMetadata( + # NOTE: mistral-common uses the tokenizer to set placeholders, this will be ignored + placeholder_map={"image": "[IMG]"}, + placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, + content_format=ContentFormat.STRING, + )) class Mistral3VLM(MultimodalModelMixin, PreTrainedModel): """Mistral3VLM implementation for TRTLLM. @@ -743,11 +784,7 @@ def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs): llm_weights = filter_weights(weights=weights, prefix="language_model") logger.debug(f"Loading weights for {type(self.llm)}") if weight_mapper and type(weight_mapper) is MistralWeightMapper: - weight_mapper.permute_qk(weights=llm_weights, - config=self.llm.config) - self.llm.load_weights(llm_weights, - weight_mapper=weight_mapper, - params_map=weight_mapper.mistral_llm_mapping) + self.llm.load_weights(llm_weights, weight_mapper=weight_mapper) else: self.llm.load_weights(llm_weights) logger.debug(f"Successfully loaded weights for {type(self.llm)}") diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py index a52534de4f50..4258b2ed6d1a 100755 --- a/tensorrt_llm/_torch/models/modeling_speculative.py +++ b/tensorrt_llm/_torch/models/modeling_speculative.py @@ -664,6 +664,7 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, spec_metadata: SpecMetadata | None = None, hidden_states: torch.Tensor | None = None, + all_rank_num_tokens: Optional[List[int]] = None, ) -> torch.Tensor: if (input_ids is None) ^ (inputs_embeds is not None): raise ValueError( @@ -676,18 +677,27 @@ def forward( assert hidden_states is not None - # NOTE: If hidden states from the target model have to be concatenated, - # we expect that to happen outside the model definition. This helps us - # avoid data-dependent control flow and gives us better CUDA graph - # coverage. - residual = None - hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1) - hidden_states = self.fc(hidden_states) - hidden_states, residual = self.layers[0](position_ids=position_ids, - hidden_states=hidden_states, - attn_metadata=attn_metadata, - residual=None, - spec_metadata=spec_metadata) + previous_all_rank_num_tokens = attn_metadata.all_rank_num_tokens + if all_rank_num_tokens is not None: + attn_metadata.all_rank_num_tokens = all_rank_num_tokens + + try: + # NOTE: If hidden states from the target model have to be concatenated, + # we expect that to happen outside the model definition. This helps us + # avoid data-dependent control flow and gives us better CUDA graph + # coverage. + residual = None + hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1) + hidden_states = self.fc(hidden_states) + hidden_states, residual = self.layers[0]( + position_ids=position_ids, + hidden_states=hidden_states, + attn_metadata=attn_metadata, + residual=None, + spec_metadata=spec_metadata) + finally: + if all_rank_num_tokens is not None: + attn_metadata.all_rank_num_tokens = previous_all_rank_num_tokens return hidden_states, hidden_states diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index 601ab716dfcb..4c561a7a1489 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -392,6 +392,11 @@ def load_pretrained_config(model_name_or_path: str, trust_remote_code: bool = False, checkpoint_format: Optional[str] = None, **kwargs) -> transformers.PretrainedConfig: + if checkpoint_format in ("mistral", "mistral_large_3"): + from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import \ + MistralConfigLoader + return MistralConfigLoader().load(model_name_or_path).pretrained_config + config_dict, _ = transformers.PretrainedConfig.get_config_dict( model_name_or_path, **kwargs) model_type = config_dict.get("model_type") diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index 6712c40196c4..acd3f67f9f27 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -937,8 +937,15 @@ def create_input_processor( logger.debug(f"Detected checkpoint_format={checkpoint_format}.") from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import \ MistralConfigLoader + from tensorrt_llm._torch.models.modeling_mistral import \ + MistralNativeInputProcessor model_config = MistralConfigLoader().load(model_path_or_dir) config = model_config.pretrained_config + return MistralNativeInputProcessor(model_path_or_dir, + config, + tokenizer, + trust_remote_code=trust_remote_code, + **kwargs) else: logger.debug( f"checkpoint_format={checkpoint_format}; skipping HF config load.") diff --git a/tensorrt_llm/serve/chat_utils.py b/tensorrt_llm/serve/chat_utils.py index 2ab0c8b362e7..d60d93f2ce45 100644 --- a/tensorrt_llm/serve/chat_utils.py +++ b/tensorrt_llm/serve/chat_utils.py @@ -424,6 +424,7 @@ def parse_chat_messages_coroutines( model_config: AutoConfig, multimodal_server_config: Optional[MultimodalServerConfig] = None, request_media_io_kwargs: Optional[Dict[str, Dict[str, Any]]] = None, + model_type_override: Optional[str] = None, ) -> Tuple[List[ConversationMessage], Coroutine[Any, Any, tuple[Optional[Dict[ str, List[Any]]], Optional[Dict[str, List[Any]]]]], list[dict[str, int]]]: @@ -470,7 +471,7 @@ def parse_chat_messages_coroutines( # `content_parts` - overwriting any STRING-style placeholders inserted here. # See also: `_resolve_content_format` (inputs/utils.py) for the full resolution used downstream. registry_format = MULTIMODAL_PLACEHOLDER_REGISTRY.get_content_format( - type(model_config).model_type) + model_type) if registry_format is not None: content_format = registry_format else: @@ -502,11 +503,10 @@ def parse_chat_messages_coroutines( # prepend/append according to placeholder_placement. content_parts = parsed_msg.get("content_parts") interleave = MULTIMODAL_PLACEHOLDER_REGISTRY.get_interleave_placeholders( - type(model_config).model_type) + model_type) if content_parts and interleave: parsed_msg["content"] = interleave_mm_placeholders( - type(model_config).model_type, content_parts, - msg_placeholder_counts, + model_type, content_parts, msg_placeholder_counts, mm_data_tracker.placeholder_modalities()) else: msg_item_order = mm_data_tracker.item_order()[item_order_start:] diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index eedd0be39285..7b0e04d52b6e 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -492,13 +492,21 @@ def _init_llm(self, chat_template: Optional[str] = None): self.tokenizer.tokenizer, "name_or_path", None) or getattr( self.tokenizer, "name_or_path", None) trust_remote_code = self.generator.args.trust_remote_code - try: - self.processor = AutoProcessor.from_pretrained( - hf_tokenizer_path, trust_remote_code=trust_remote_code) - except Exception: - logger.debug("Failed to load AutoProcessor or AutoConfig for %s", - hf_tokenizer_path) + checkpoint_format = getattr(self.generator.args, "checkpoint_format", + None) + if checkpoint_format in ("mistral", "mistral_large_3"): + # Do not load HF processor for mistral native checkpoints + # even if it is available self.processor = None + else: + try: + self.processor = AutoProcessor.from_pretrained( + hf_tokenizer_path, trust_remote_code=trust_remote_code) + except Exception: + logger.debug( + "Failed to load AutoProcessor or AutoConfig for %s", + hf_tokenizer_path) + self.processor = None # load model config try: @@ -1547,7 +1555,8 @@ async def chat_stream_generator( request.messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) except ValidationError: # ValidatorIterator rejects extra fields; fall back to raw JSON. raw_body = await raw_request.json() @@ -1556,7 +1565,8 @@ async def chat_stream_generator( raw_messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) # Decode base64 int32 prompt_token_ids relayed by the orchestrator. if request.prompt_token_ids is None and request.prompt_token_ids_b64: @@ -1735,7 +1745,8 @@ async def create_mm_embedding_response(promise: RequestOutput): request.messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) except ValidationError: # ValidatorIterator rejects extra fields; fall back to raw JSON. raw_body = await raw_request.json() @@ -1744,7 +1755,8 @@ async def create_mm_embedding_response(promise: RequestOutput): raw_messages, self.model_config, self.multimodal_server_config, - request_media_io_kwargs=request.media_io_kwargs) + request_media_io_kwargs=request.media_io_kwargs, + ) if request.prompt_token_ids is not None: prompt = request.prompt_token_ids diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py index 1422cc3804b5..9715865c65a1 100644 --- a/tensorrt_llm/tokenizer/tokenizer.py +++ b/tensorrt_llm/tokenizer/tokenizer.py @@ -40,8 +40,12 @@ # Aliases for built-in custom tokenizers. TOKENIZER_ALIASES = { - "deepseek_v32": "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer", - "deepseek_v4": "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer", + "deepseek_v32": + "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer", + "deepseek_v4": + "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer", + "mistral_common": + "tensorrt_llm._torch.models.checkpoints.mistral.tokenizer.MistralTokenizer", } TLLM_INCREMENTAL_DETOKENIZATION_BACKEND = os.environ.get( diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 91bd03c3df11..3cd3c1c88868 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -492,7 +492,11 @@ def test_nvfp4_4gpus( mocker, ): mocker.patch.dict( - MMMU.EVALUATE_KWARGS, {"model_type": "mistral_large_3", "is_force_single_image": True} + MMMU.EVALUATE_KWARGS, + { + "model_type": "mistral_common", + "is_force_single_image": True, + }, ) pytorch_config = dict( disable_overlap_scheduler=not overlap_scheduler, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6dc4776bb633..e290f7c277ea 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -95,9 +95,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torc accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6211441) accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132) -accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] SKIP (https://nvbugs/6478720) -accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827) -accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892) accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] SKIP (https://nvbugs/6427411) accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tep4] SKIP (https://nvbugs/6255417) diff --git a/tests/unittest/_torch/modeling/test_modeling_mistral.py b/tests/unittest/_torch/modeling/test_modeling_mistral.py index 793b70455057..6cb29057a8bd 100644 --- a/tests/unittest/_torch/modeling/test_modeling_mistral.py +++ b/tests/unittest/_torch/modeling/test_modeling_mistral.py @@ -19,7 +19,7 @@ from tensorrt_llm._torch import model_config as model_config_lib from tensorrt_llm._torch.attention_backend import utils as attention_utils from tensorrt_llm._torch.models import modeling_mistral -from tensorrt_llm._torch.models.modeling_mistral import Mistral3InputProcessor +from tensorrt_llm._torch.models.modeling_mistral import MistralHFInputProcessor from tensorrt_llm._torch.models.modeling_utils import MetaInitMode from tensorrt_llm._torch.pyexecutor import resource_manager from tensorrt_llm.bindings import executor as executor_lib @@ -540,7 +540,7 @@ def test_processor_get_num_tokens_per_image( with mock.patch( "tensorrt_llm._torch.models.modeling_mistral.AutoProcessor" ) as mocked_auto_processor: - input_processor = modeling_mistral.Mistral3InputProcessor( + input_processor = modeling_mistral.MistralHFInputProcessor( model_path=str(tmp_path), config=mistral_3_config, tokenizer=mock.MagicMock(), @@ -637,7 +637,7 @@ def test_mistral_attention_swa_layer_types(): # Deterministic dummy-input sizing (Mistral3 / Pixtral input processor). # # CPU-only unit tests for the encoder-profiling dummy contract: reach into -# Mistral3InputProcessor directly (no model load) and stub the geometry the +# MistralHFInputProcessor directly (no model load) and stub the geometry the # dummy math reads. The ViT token unit is the pre-merge patch count # ``(h//patch)*(w//patch)`` -- deliberately *not* the hashing path's LLM-side # Pixtral count with framing tokens. @@ -649,7 +649,7 @@ def _make_dummy_processor(*, patch_size=14, spatial_merge_size=2, image_size=154 ``_processor`` forces ``_vision_geometry`` to fall back to ``vision_config`` (the HF ``mistral3`` path). """ - instance = Mistral3InputProcessor.__new__(Mistral3InputProcessor) + instance = MistralHFInputProcessor.__new__(MistralHFInputProcessor) instance._config = SimpleNamespace( vision_config=SimpleNamespace( patch_size=patch_size, image_size=image_size, num_channels=num_channels