From 6139ebcf96a1725c127df68f4333b6305c632bd0 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Fri, 5 Jun 2026 14:31:45 -0700
Subject: [PATCH 01/19] add HF preprocessor

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 .../_torch/attention_backend/interface.py     |   2 +
 tensorrt_llm/_torch/model_config.py           |  11 +-
 .../checkpoints/mistral/config_loader.py      |  26 +-
 .../_torch/models/modeling_mistral.py         | 267 ++++++++++--------
 tensorrt_llm/inputs/registry.py               |   9 +
 5 files changed, 190 insertions(+), 125 deletions(-)

diff --git a/tensorrt_llm/_torch/attention_backend/interface.py b/tensorrt_llm/_torch/attention_backend/interface.py
index fec6ba4e6b92..0faad4916123 100644
--- a/tensorrt_llm/_torch/attention_backend/interface.py
+++ b/tensorrt_llm/_torch/attention_backend/interface.py
@@ -700,6 +700,8 @@ def from_config(config) -> "RopeParams":
         if not isinstance(head_dim, int):
             head_dim = hidden_size // num_attention_heads
         rope_scaling = getattr(config, 'rope_scaling', None)
+        if rope_scaling is None and hf_rope_parameters is not None:
+            rope_scaling = hf_rope_parameters
         rope_params.max_positions = config.max_position_embeddings
         rope_params.theta = get_hf_rope_theta(config, 10000.0)
         rope_percentage = (getattr(config, 'rotary_pct', None)
diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py
index 8f5641920c03..25fec2bb1978 100644
--- a/tensorrt_llm/_torch/model_config.py
+++ b/tensorrt_llm/_torch/model_config.py
@@ -494,12 +494,15 @@ def load_hf_quant_config(hf_quant_config, moe_backend, checkpoint_dir=None):
         # Read exclude_modules from HF config if present (HF format module names)
         hf_exclude_modules = hf_quant_config.get('modules_to_not_convert', None)
 
-        # DeepSeek V3 FP8 ckpt
-        if hf_quant_config.get("quant_method") == "fp8" and hf_quant_config.get(
-                "weight_block_size", []):
+        # FP8 ckpt: DeepSeek V3 style (weight_block_size) or
+        # per-tensor static activation scale style (activation_scheme="static",
+        # e.g. Ministral / Pixtral).
+        if hf_quant_config.get("quant_method") == "fp8" and (
+                hf_quant_config.get("weight_block_size")
+                or hf_quant_config.get("activation_scheme") == "static"):
             quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES
 
-            block_size = hf_quant_config.get("weight_block_size", [])
+            block_size = hf_quant_config.get("weight_block_size") or [128, 128]
             assert tuple(block_size) == (
                 128, 128), "FP8_BLOCK_SCALES only supports block_size=(128,128)"
             quant_config.group_size = block_size[0]
diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
index cd73f1b8ead0..7b5909e9d7ab 100644
--- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
+++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
@@ -57,6 +57,18 @@ def __init__(
             self.rope_parameters = self.rope_scaling
 
 
+class _Mistral3VLMPretrainedConfig(_MistralPretrainedConfig):
+    """Pretrained config for mistral-native VLM checkpoints.
+
+    Setting model_type as a class attribute ensures that
+    ``type(config).model_type`` returns ``"mistral3_common"`` so that
+    ``chat_utils.parse_chat_messages_coroutines`` resolves the correct
+    ContentFormat (PASSTHROUGH) without requiring changes to that call-site.
+    """
+
+    model_type = "mistral3_common"
+
+
 def _mistral_pretrained_config_from_dict(config_dict: dict[str, Any]) -> PretrainedConfig:
     return _MistralPretrainedConfig.from_dict(config_dict)
 
@@ -115,7 +127,10 @@ def adapt_config_dict(
     for k, v in defaults.items():
         config_dict.setdefault(k, v)
 
-    config = _mistral_pretrained_config_from_dict(config_dict)
+    if is_vision:
+        config = _Mistral3VLMPretrainedConfig.from_dict(config_dict)
+    else:
+        config = _mistral_pretrained_config_from_dict(config_dict)
 
     return config
 
@@ -328,7 +343,7 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig:
         quant_config = QuantConfig()
         layer_quant_config = None
 
-        hf_quant_config = pretrained_config.quantization_config
+        hf_quant_config = getattr(pretrained_config, "quantization_config", {}) or {}
         if hf_quant_config.get("quant_method") == "compressed-tensors":
             if "NVFP4" in hf_quant_config.get("config_groups"):
                 quant_config.quant_algo = QuantAlgo.NVFP4
@@ -390,7 +405,10 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig:
         from tensorrt_llm._torch.models.modeling_mistral_large3 import Mistral3Gate
 
         model_config.pretrained_config.gate_cls = Mistral3Gate
-        model_config.pretrained_config.input_processor_type = "mistral_large_3"
-        model_config.pretrained_config.model_type = "mistral_large_3"
+        arch = (getattr(pretrained_config, "architectures", None) or [None])[0]
+        if arch == "PixtralForConditionalGeneration":
+            model_config.pretrained_config.input_processor_type = "mistral3_common"
+        else:
+            model_config.pretrained_config.model_type = "mistral3"
         model_config._frozen = True
         return model_config
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 2246a73f8fdd..52eeb63fee35 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -47,6 +47,7 @@
                                  MultimodalPlaceholderPlacement, TextPrompt,
                                  register_input_processor)
 from tensorrt_llm.inputs.multimodal import MultimodalParams
+from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY
 from tensorrt_llm.inputs.utils import encode_base64_image
 from tensorrt_llm.llmapi import SamplingParams
 from tensorrt_llm.logger import logger
@@ -63,7 +64,13 @@ def __init__(
         rope_params = RopeParams.from_config(config)
         rope_params_section = getattr(config, "rope_scaling", None) or getattr(
             config, "rope_parameters", None)
-        rope_type = getattr(rope_params_section, "rope_type", None)
+
+        rope_type = getattr(
+            rope_params_section, "rope_type",
+            None) or (rope_params_section.get("rope_type") if isinstance(
+                rope_params_section, dict) else None) or getattr(
+                    config, "rope_type", None)
+
         if rope_type == "yarn":
             pos_embd_params = PositionalEmbeddingParams(
                 type=PositionEmbeddingType.yarn,
@@ -352,18 +359,67 @@ def __call__(self, text, images=None, **kwargs):
         return processed
 
 
-class Mistral3InputProcessor(BaseMultimodalInputProcessor,
-                             BaseMultimodalDummyInputsBuilder):
+class MistralInputProcessorBase(BaseMultimodalInputProcessor,
+                                BaseMultimodalDummyInputsBuilder):
+    """Shared logic for HF and native Mistral VLM input processors."""
 
-    def __init__(
-        self,
-        model_path: str,
-        config: PretrainedConfig,
-        tokenizer: AutoTokenizer | None,
-        trust_remote_code: bool = False,
-        model_type: str = "mistral3",
-        **kwargs,
-    ):
+    @property
+    def config(self) -> PretrainedConfig:
+        return self._config
+
+    @property
+    def tokenizer(self) -> AutoTokenizer:
+        return self._tokenizer
+
+    @property
+    def model_path(self) -> str:
+        return self._model_path
+
+    @property
+    def processor(self) -> AutoProcessor:
+        return self._processor
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return self._dtype
+
+    def get_vocab_size(self) -> int:
+        return self.config.text_config.vocab_size
+
+    def get_mm_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            self.processor.image_token_id,
+            self.processor.image_break_token_id,
+            self.processor.image_end_token_id,
+        ])
+
+    def get_mm_special_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            self.processor.image_break_token_id,
+            self.processor.image_end_token_id,
+        ])
+
+    @staticmethod
+    def _extract_extra_processed_inputs(
+            processed) -> ExtraProcessedInputs | None:
+        pixel_values = processed.get("pixel_values")
+        if pixel_values is None:
+            return None
+        processed.pop("attention_mask", None)
+        processed["image_sizes"] = processed["image_sizes"].tolist()
+        return {"multimodal_data": {"image": {**processed}}}
+
+
+class MistralHFInputProcessor(BaseMultimodalInputProcessor,
+                              BaseMultimodalDummyInputsBuilder):
+    """Input processor for Mistral VLM checkpoints in HuggingFace format."""
+
+    def __init__(self,
+                 model_path: str,
+                 config: PretrainedConfig,
+                 tokenizer: AutoTokenizer,
+                 trust_remote_code: bool = True,
+                 **kwargs):
         super().__init__(model_path=model_path,
                          config=config,
                          tokenizer=tokenizer,
@@ -371,27 +427,19 @@ def __init__(
                          **kwargs)
         self._config = config
         self._dtype = self._config.torch_dtype
-        self._tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained(
-            model_path,
-            config=config,
-            use_fast=self.use_fast,
-            trust_remote_code=trust_remote_code)
         self._model_path = model_path
-        auto_processor = AutoProcessor.from_pretrained(
+        self._tokenizer = (tokenizer if tokenizer is not None else
+                           AutoTokenizer.from_pretrained(
+                               model_path,
+                               config=config,
+                               use_fast=True,
+                               trust_remote_code=True))
+        self._processor = AutoProcessor.from_pretrained(
             model_path,
             use_fast=self.use_fast,
             trust_remote_code=trust_remote_code)
-        if model_type == "mistral_large_3":
-            # For mistral large 3, we add chat template in the model forward, and the
-            # MistralCommonImageProcessor is used to process the input when both text and images are provided.
-            # When the input only contains text, we use the text processor to process the input.
-            self._processor = MistralCommonImageProcessor(
-                tokenizer=self._tokenizer, dtype=self.dtype)
-            self.text_processor = auto_processor
-        else:
-            # For other mistral models, we use the AutoProcessor to process the input.
-            self._processor = auto_processor
-            self.text_processor = self._processor
+        logger.info(f"[mistral] HF processor={type(self._processor).__name__} "
+                    f"tokenizer={type(self._tokenizer).__name__}")
 
     @property
     def config(self) -> PretrainedConfig:
@@ -413,6 +461,22 @@ def processor(self) -> AutoProcessor:
     def dtype(self) -> torch.dtype:
         return self._dtype
 
+    def get_vocab_size(self) -> int:
+        return self.config.text_config.vocab_size
+
+    def get_mm_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            self.processor.image_token_id,
+            self.processor.image_break_token_id,
+            self.processor.image_end_token_id,
+        ])
+
+    def get_mm_special_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            self.processor.image_break_token_id,
+            self.processor.image_end_token_id,
+        ])
+
     @torch.inference_mode()
     def call_with_text_prompt(
         self, inputs: TextPrompt, sampling_params: SamplingParams
@@ -421,38 +485,22 @@ def call_with_text_prompt(
         do_rescale = getattr(self.processor.image_processor, "do_rescale",
                              False)
         if images is not None and isinstance(images[0], torch.Tensor):
-            # The default multimodal input loader will normalize images to [0, 1] when the requested
-            # format is "pt" (pytorch tensors), but not for "pil" (PIL images).
+            # The default multimodal input loader normalises images to [0, 1]
+            # for "pt" tensors but not for PIL images.
             do_rescale = False
 
-        if images is not None:
-            processed = self.processor(
-                text=inputs["prompt"],
-                images=images,
-                do_rescale=do_rescale,
-            )
-        else:
-            processed = self.text_processor(
-                text=inputs["prompt"],
-                do_rescale=do_rescale,
-            )
+        prompt = inputs["prompt"]
+        processed = self.processor(text=prompt,
+                                   images=images,
+                                   do_rescale=do_rescale)
         input_ids = processed.pop("input_ids").tolist()[0]
-        # Remaining in `processed`:
-        # * "attention_mask": [B, num_input_tokens]
-        # * "pixel_values": [B, C, H, W]
-        # * "image_sizes": [B, 2]
         extra_processed_inputs = None
         pixel_values = processed.get("pixel_values")
         if pixel_values is not None:
-            # We have no use for the `attention_mask`.
             processed.pop("attention_mask")
-            # `image_sizes` is a `[B, 2]` tensor indicating the height and width of each image in the
-            # request. If we keep it as a regular tensor, it would get converted to a CUDA tensor before
-            # reaching the model forward. Since its values are used to infer the amount of padding
-            # + slice the patch embeddings, this would incur a D2H copy. We therefore convert it to a
-            # list here to avoid this.
+            # Keep as list to avoid a D2H copy when the tensor would otherwise
+            # be moved to GPU before the model forward.
             processed["image_sizes"] = processed["image_sizes"].tolist()
-            # NOTE: `processed` is a dict-like object, but not actually a dict.
             extra_processed_inputs = {
                 "multimodal_data": {
                     "image": {
@@ -569,92 +617,69 @@ def get_dummy_mm_data_for_tokens(
                                                num_images=num_images,
                                                dtype=dtype)
 
-    def get_vocab_size(self) -> int:
-        """Return the vocab size of the model."""
-        # Unlike some other VLMs, mistral3's vocab size is stored in its `text_config`, not the top-level
-        # config.
-        return self.config.text_config.vocab_size
-
-    def get_mm_token_ids(self) -> torch.Tensor:
-        """Get the IDs of all multimodal tokens (placeholders and special tokens alike)."""
-        return torch.tensor([
-            # This is the `[IMG]` token id inserted into the prompt that should be replaced with image
-            # embeddings.
-            self.processor.image_token_id,
-            # This is the `[IMG_BREAK]` token id at the end of every "row".
-            self.processor.image_break_token_id,
-            # This is the `[IMG_END]` token id to signify the end of an image.
-            self.processor.image_end_token_id,
-        ])
-
-    def get_mm_special_token_ids(self) -> torch.Tensor:
-        """Get the IDs of special multimodal tokens (placeholders not included)."""
-        return torch.tensor([
-            self.processor.image_break_token_id,
-            self.processor.image_end_token_id,
-        ])
-
 
-class MistralCommonInputProcessor(Mistral3InputProcessor):
+class MistralNativeInputProcessor(MistralInputProcessorBase):
+    """Input processor for Mistral VLM checkpoints in mistral-native format."""
 
     def __init__(
         self,
         model_path: str,
         config: PretrainedConfig,
-        tokenizer: AutoTokenizer,
+        tokenizer: AutoTokenizer | None,
         trust_remote_code: bool = False,
         **kwargs,
     ):
-        tokenizer = self.load_tokenizer(model_path,
-                                        config=config,
-                                        tokenizer=tokenizer)
         super().__init__(model_path=model_path,
                          config=config,
                          tokenizer=tokenizer,
                          trust_remote_code=trust_remote_code,
-                         model_type=getattr(config, "input_processor_type",
-                                            "mistral3"),
                          **kwargs)
+        self._config = config
+        self._dtype = self._config.torch_dtype
+        self._model_path = model_path
+        self._tokenizer = MistralTokenizer.from_pretrained(model_path)
+        self._processor = MistralCommonImageProcessor(tokenizer=self._tokenizer,
+                                                      dtype=self.dtype)
+        logger.info(
+            f"[mistral] native processor={type(self._processor).__name__} "
+            f"tokenizer={type(self._tokenizer).__name__}")
 
-    @staticmethod
-    def load_tokenizer(model_path: str,
-                       config: PretrainedConfig,
-                       tokenizer: AutoTokenizer | None = None):
-        if getattr(config, "input_processor_type", None) == "mistral_large_3":
-            try:
-                return MistralTokenizer.from_pretrained(model_path)
-
-            except ValueError:
-                logger.info(
-                    f"Could not load mistral-common tokenizer from {model_path}, falling back to HuggingFace"
-                )
-
-        tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained(
-            model_path, config=config, use_fast=True, trust_remote_code=True)
-        return tokenizer
+    @torch.inference_mode()
+    def call_with_text_prompt(
+        self, inputs: TextPrompt, sampling_params: SamplingParams
+    ) -> Tuple[List[int], ExtraProcessedInputs | None]:
+        images = inputs.get("multi_modal_data", {}).get("image")
+        # MistralCommonImageProcessor builds the full conversation and applies
+        # the mistral-common chat template internally.
+        processed = self.processor(text=inputs["prompt"], images=images)
+        input_ids = processed.pop("input_ids").tolist()[0]
+        return input_ids, self._extract_extra_processed_inputs(processed)
+
+
+# Register the native processor's content-format metadata.  We do this
+# directly rather than via @register_input_processor because that decorator
+# also writes to INPUT_PROCESSOR_REGISTRY (keyed by model class), which would
+# overwrite the MistralHFInputProcessor entry for Mistral3VLM.  The class
+# dispatch for native checkpoints is handled in create_input_processor via
+# input_processor_type, so only the placeholder side-effect is needed here.
+MULTIMODAL_PLACEHOLDER_REGISTRY.set_placeholder_metadata(
+    "mistral3_common",
+    MultimodalPlaceholderMetadata(
+        placeholder_map={"image": "[IMG]"},
+        placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT,
+        content_format=ContentFormat.PASSTHROUGH,
+    ))
+MistralNativeInputProcessor._registered_model_type = "mistral3_common"
 
 
 @register_auto_model("Mistral3ForConditionalGeneration")
 @register_auto_model("PixtralForConditionalGeneration")
 @register_input_processor(
-    MistralCommonInputProcessor,
-    model_type="mistral_large_3",
-    placeholder_metadata=MultimodalPlaceholderMetadata(
-        placeholder_map={
-            # NOTE: mistral-common uses the tokenizer to set placeholders, this will be ignored
-            "image": "[IMG]",
-        },
-        placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT,
-        content_format=ContentFormat.PASSTHROUGH,
-    ))
-@register_input_processor(
-    MistralCommonInputProcessor,
+    MistralHFInputProcessor,
     model_type="mistral3",
     placeholder_metadata=MultimodalPlaceholderMetadata(
-        placeholder_map={
-            "image": "[IMG]",
-        },
-        # NOTE: for mistral3 multimodal models, it does not strictly have to be before the text.
+        placeholder_map={"image": "[IMG]"},
+        # NOTE: for mistral3 multimodal models, placeholder_placement does not strictly have to be before the text.
         # Ref: https://github.com/mistralai/mistral-common/blob/039465db2bdc0486df36365c9bdb428188482a18/
         #      src/mistral_common/tokens/tokenizers/base.py#L326
         # However, accuracy tests show that the model generates higher quality output when the image
@@ -662,6 +687,14 @@ def load_tokenizer(model_path: str,
         placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT,
         content_format=ContentFormat.STRING,
     ))
+@register_input_processor(
+    MistralHFInputProcessor,
+    model_type="mistral_large_3",
+    placeholder_metadata=MultimodalPlaceholderMetadata(
+        placeholder_map={"image": "[IMG]"},
+        placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT,
+        content_format=ContentFormat.STRING,
+    ))
 class Mistral3VLM(MultimodalModelMixin, PreTrainedModel):
     """Mistral3VLM implementation for TRTLLM.
 
diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py
index 6712c40196c4..f00295dea5b4 100644
--- a/tensorrt_llm/inputs/registry.py
+++ b/tensorrt_llm/inputs/registry.py
@@ -939,6 +939,15 @@ def create_input_processor(
             MistralConfigLoader
         model_config = MistralConfigLoader().load(model_path_or_dir)
         config = model_config.pretrained_config
+        if getattr(config, "input_processor_type", None) == "mistral3_common":
+            from tensorrt_llm._torch.models.modeling_mistral import \
+                MistralNativeInputProcessor
+            return MistralNativeInputProcessor(
+                model_path_or_dir,
+                config,
+                tokenizer,
+                trust_remote_code=trust_remote_code,
+                **kwargs)
     else:
         logger.debug(
             f"checkpoint_format={checkpoint_format}; skipping HF config load.")

From 3e2ca5de7fc91458d9b97f46b57bb5e4a84a7f7d Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Sat, 6 Jun 2026 16:51:21 -0700
Subject: [PATCH 02/19] allow HF processor for Mistral checkpoints

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/llm-api/quickstart_multimodal.py     |  9 +++++++
 .../checkpoints/mistral/config_loader.py      | 26 +++++--------------
 .../_torch/models/modeling_mistral.py         |  6 ++---
 .../_torch/pyexecutor/config_utils.py         |  5 ++++
 tensorrt_llm/inputs/registry.py               |  9 +++++--
 tensorrt_llm/serve/chat_utils.py              | 11 ++++----
 tensorrt_llm/serve/openai_server.py           | 21 ++++++++++++---
 7 files changed, 53 insertions(+), 34 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index 471986294304..1c0db1a000da 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -138,6 +138,13 @@ def add_multimodal_args(parser):
                         default=None,
                         help="Pruning rate for video frames (EVS). "
                         "None disables EVS, values in [0, 1) enable pruning.")
+    parser.add_argument(
+        "--input_processor",
+        type=str,
+        default=None,
+        help="Override the automatically selected input processor. "
+        "Must be a registered model_type string (e.g. 'mistral3_common' for the "
+        "native Mistral VLM processor). None (default) selects automatically.")
     return parser
 
 
@@ -199,6 +206,8 @@ def main():
     image_format = args.image_format
     if args.model_type is not None:
         model_type = args.model_type
+    elif hasattr(llm.input_processor, '_registered_model_type'):
+        model_type = llm.input_processor._registered_model_type
     else:
         model_type = json.load(
             open(os.path.join(str(llm._hf_model_dir),
diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
index 7b5909e9d7ab..b94a684a5fc7 100644
--- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
+++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
@@ -57,18 +57,6 @@ def __init__(
             self.rope_parameters = self.rope_scaling
 
 
-class _Mistral3VLMPretrainedConfig(_MistralPretrainedConfig):
-    """Pretrained config for mistral-native VLM checkpoints.
-
-    Setting model_type as a class attribute ensures that
-    ``type(config).model_type`` returns ``"mistral3_common"`` so that
-    ``chat_utils.parse_chat_messages_coroutines`` resolves the correct
-    ContentFormat (PASSTHROUGH) without requiring changes to that call-site.
-    """
-
-    model_type = "mistral3_common"
-
-
 def _mistral_pretrained_config_from_dict(config_dict: dict[str, Any]) -> PretrainedConfig:
     return _MistralPretrainedConfig.from_dict(config_dict)
 
@@ -127,10 +115,10 @@ def adapt_config_dict(
     for k, v in defaults.items():
         config_dict.setdefault(k, v)
 
-    if is_vision:
-        config = _Mistral3VLMPretrainedConfig.from_dict(config_dict)
-    else:
-        config = _mistral_pretrained_config_from_dict(config_dict)
+    # if is_vision:
+    #     config = _Mistral3VLMPretrainedConfig.from_dict(config_dict)
+    # else:
+    config = _mistral_pretrained_config_from_dict(config_dict)
 
     return config
 
@@ -406,9 +394,7 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig:
 
         model_config.pretrained_config.gate_cls = Mistral3Gate
         arch = (getattr(pretrained_config, "architectures", None) or [None])[0]
-        if arch == "PixtralForConditionalGeneration":
-            model_config.pretrained_config.input_processor_type = "mistral3_common"
-        else:
-            model_config.pretrained_config.model_type = "mistral3"
+        model_config.pretrained_config.input_processor_type = "mistral3"
+        model_config.pretrained_config.model_type = "mistral3"
         model_config._frozen = True
         return model_config
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 52eeb63fee35..124281a5a511 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -659,9 +659,9 @@ def call_with_text_prompt(
 # Register the native processor's content-format metadata.  We do this
 # directly rather than via @register_input_processor because that decorator
 # also writes to INPUT_PROCESSOR_REGISTRY (keyed by model class), which would
-# overwrite the MistralHFInputProcessor entry for Mistral3VLM.  The class
-# dispatch for native checkpoints is handled in create_input_processor via
-# input_processor_type, so only the placeholder side-effect is needed here.
+# overwrite the MistralHFInputProcessor entry for Mistral3VLM.
+# Mistral is the only supported case where HF preprocessor can be used with
+# non-HF checkpoints, so this hack is preferred to changing the registry itself.
 MULTIMODAL_PLACEHOLDER_REGISTRY.set_placeholder_metadata(
     "mistral3_common",
     MultimodalPlaceholderMetadata(
diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py
index 601ab716dfcb..4c561a7a1489 100644
--- a/tensorrt_llm/_torch/pyexecutor/config_utils.py
+++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py
@@ -392,6 +392,11 @@ def load_pretrained_config(model_name_or_path: str,
                            trust_remote_code: bool = False,
                            checkpoint_format: Optional[str] = None,
                            **kwargs) -> transformers.PretrainedConfig:
+    if checkpoint_format in ("mistral", "mistral_large_3"):
+        from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import \
+            MistralConfigLoader
+        return MistralConfigLoader().load(model_name_or_path).pretrained_config
+
     config_dict, _ = transformers.PretrainedConfig.get_config_dict(
         model_name_or_path, **kwargs)
     model_type = config_dict.get("model_type")
diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py
index f00295dea5b4..5d96b8632265 100644
--- a/tensorrt_llm/inputs/registry.py
+++ b/tensorrt_llm/inputs/registry.py
@@ -914,7 +914,9 @@ def create_input_processor(
         trust_remote_code: Whether Hugging Face config/processor loading may run
             model-provided Python code.
         **kwargs: Additional arguments passed to input processor constructors
-            (e.g., video_pruning_rate for multimodal models).
+            (e.g., video_pruning_rate for multimodal models).  A special key
+            ``input_processor`` (str) overrides automatic processor selection with
+            the named registered model_type.
 
     Returns:
         An InputProcessor implementation (model-specific if registered; otherwise DefaultInputProcessor).
@@ -922,6 +924,9 @@ def create_input_processor(
     from tensorrt_llm._torch.model_config import ModelConfig
     from tensorrt_llm._torch.models import get_model_architecture
 
+    # Pop the override before passing kwargs to constructors.
+    input_processor_override = kwargs.pop("input_processor", None)
+
     config = None
 
     if checkpoint_format == "HF":
@@ -939,7 +944,7 @@ def create_input_processor(
             MistralConfigLoader
         model_config = MistralConfigLoader().load(model_path_or_dir)
         config = model_config.pretrained_config
-        if getattr(config, "input_processor_type", None) == "mistral3_common":
+        if input_processor_override == "mistral3_common":
             from tensorrt_llm._torch.models.modeling_mistral import \
                 MistralNativeInputProcessor
             return MistralNativeInputProcessor(
diff --git a/tensorrt_llm/serve/chat_utils.py b/tensorrt_llm/serve/chat_utils.py
index 2ab0c8b362e7..9132f7fce822 100644
--- a/tensorrt_llm/serve/chat_utils.py
+++ b/tensorrt_llm/serve/chat_utils.py
@@ -424,6 +424,7 @@ def parse_chat_messages_coroutines(
     model_config: AutoConfig,
     multimodal_server_config: Optional[MultimodalServerConfig] = None,
     request_media_io_kwargs: Optional[Dict[str, Dict[str, Any]]] = None,
+    model_type_override: Optional[str] = None,
 ) -> Tuple[List[ConversationMessage], Coroutine[Any, Any, tuple[Optional[Dict[
         str, List[Any]]], Optional[Dict[str, List[Any]]]]], list[dict[str,
                                                                       int]]]:
@@ -450,7 +451,8 @@ def parse_chat_messages_coroutines(
     """
     conversation = []
     mm_placeholder_counts = []
-    model_type = resolve_top_level_model_type(model_config)
+    model_type = model_type_override or resolve_top_level_model_type(
+        model_config)
     mm_data_tracker = MultimodalDataTracker(
         model_type,
         multimodal_server_config,
@@ -470,7 +472,7 @@ def parse_chat_messages_coroutines(
     #    `content_parts` - overwriting any STRING-style placeholders inserted here.
     # See also: `_resolve_content_format` (inputs/utils.py) for the full resolution used downstream.
     registry_format = MULTIMODAL_PLACEHOLDER_REGISTRY.get_content_format(
-        type(model_config).model_type)
+        model_type)
     if registry_format is not None:
         content_format = registry_format
     else:
@@ -502,11 +504,10 @@ def parse_chat_messages_coroutines(
             # prepend/append according to placeholder_placement.
             content_parts = parsed_msg.get("content_parts")
             interleave = MULTIMODAL_PLACEHOLDER_REGISTRY.get_interleave_placeholders(
-                type(model_config).model_type)
+                model_type)
             if content_parts and interleave:
                 parsed_msg["content"] = interleave_mm_placeholders(
-                    type(model_config).model_type, content_parts,
-                    msg_placeholder_counts,
+                    model_type, content_parts, msg_placeholder_counts,
                     mm_data_tracker.placeholder_modalities())
             else:
                 msg_item_order = mm_data_tracker.item_order()[item_order_start:]
diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py
index eedd0be39285..b05ad04b2446 100644
--- a/tensorrt_llm/serve/openai_server.py
+++ b/tensorrt_llm/serve/openai_server.py
@@ -515,6 +515,15 @@ def _init_llm(self, chat_template: Optional[str] = None):
 
         self.chat_template = load_chat_template(chat_template)
 
+        # When an explicit input_processor override is active,
+        # use its registered model_type for all chat-template
+        # and placeholder-registry lookups
+        _input_proc = getattr(self.generator, 'input_processor', None)
+        _registered = getattr(_input_proc, '_registered_model_type', None)
+        self.effective_model_type = _registered or resolve_top_level_model_type(
+            self.model_config) if self.model_config is not None else (
+                _registered or "")
+
         # Enable response storage for Responses API
         self.enable_store = (len(
             os.getenv("TRTLLM_RESPONSES_API_DISABLE_STORE", ""))
@@ -1547,7 +1556,8 @@ async def chat_stream_generator(
                     request.messages,
                     self.model_config,
                     self.multimodal_server_config,
-                    request_media_io_kwargs=request.media_io_kwargs)
+                    request_media_io_kwargs=request.media_io_kwargs,
+                    model_type_override=self.effective_model_type or None)
             except ValidationError:
                 # ValidatorIterator rejects extra fields; fall back to raw JSON.
                 raw_body = await raw_request.json()
@@ -1556,7 +1566,8 @@ async def chat_stream_generator(
                     raw_messages,
                     self.model_config,
                     self.multimodal_server_config,
-                    request_media_io_kwargs=request.media_io_kwargs)
+                    request_media_io_kwargs=request.media_io_kwargs,
+                    model_type_override=self.effective_model_type or None)
 
             # Decode base64 int32 prompt_token_ids relayed by the orchestrator.
             if request.prompt_token_ids is None and request.prompt_token_ids_b64:
@@ -1735,7 +1746,8 @@ async def create_mm_embedding_response(promise: RequestOutput):
                     request.messages,
                     self.model_config,
                     self.multimodal_server_config,
-                    request_media_io_kwargs=request.media_io_kwargs)
+                    request_media_io_kwargs=request.media_io_kwargs,
+                    model_type_override=self.effective_model_type or None)
             except ValidationError:
                 # ValidatorIterator rejects extra fields; fall back to raw JSON.
                 raw_body = await raw_request.json()
@@ -1744,7 +1756,8 @@ async def create_mm_embedding_response(promise: RequestOutput):
                     raw_messages,
                     self.model_config,
                     self.multimodal_server_config,
-                    request_media_io_kwargs=request.media_io_kwargs)
+                    request_media_io_kwargs=request.media_io_kwargs,
+                    model_type_override=self.effective_model_type or None)
 
             if request.prompt_token_ids is not None:
                 prompt = request.prompt_token_ids

From 92be6c283c929e2d31b28386b6ac429ff0055064 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Mon, 8 Jun 2026 04:53:43 -0700
Subject: [PATCH 03/19] small fixes

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/llm-api/quickstart_multimodal.py                     | 4 ++--
 .../_torch/models/checkpoints/mistral/config_loader.py        | 3 ---
 tensorrt_llm/inputs/registry.py                               | 2 ++
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index 1c0db1a000da..86f2c879330b 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -143,8 +143,8 @@ def add_multimodal_args(parser):
         type=str,
         default=None,
         help="Override the automatically selected input processor. "
-        "Must be a registered model_type string (e.g. 'mistral3_common' for the "
-        "native Mistral VLM processor). None (default) selects automatically.")
+        "Must be a registered model_type. None (default) selects automatically."
+    )
     return parser
 
 
diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
index b94a684a5fc7..b424a11da036 100644
--- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
+++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
@@ -115,9 +115,6 @@ def adapt_config_dict(
     for k, v in defaults.items():
         config_dict.setdefault(k, v)
 
-    # if is_vision:
-    #     config = _Mistral3VLMPretrainedConfig.from_dict(config_dict)
-    # else:
     config = _mistral_pretrained_config_from_dict(config_dict)
 
     return config
diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py
index 5d96b8632265..064b661574b0 100644
--- a/tensorrt_llm/inputs/registry.py
+++ b/tensorrt_llm/inputs/registry.py
@@ -944,6 +944,8 @@ def create_input_processor(
             MistralConfigLoader
         model_config = MistralConfigLoader().load(model_path_or_dir)
         config = model_config.pretrained_config
+        # The only case where a model class may use two processors.
+        # Will need a more generic approach if more such models appear.
         if input_processor_override == "mistral3_common":
             from tensorrt_llm._torch.models.modeling_mistral import \
                 MistralNativeInputProcessor

From 790bd0ab155f502105feec1e95e9589199c4f34c Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Mon, 8 Jun 2026 11:52:48 -0700
Subject: [PATCH 04/19] text-only path fix

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tensorrt_llm/_torch/models/modeling_mistral.py | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 124281a5a511..e30d9cf880f8 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -318,18 +318,10 @@ def get_num_tokens_per_image(self, image_size):
         return ncols * nrows + nrows
 
     def __call__(self, text, images=None, **kwargs):
-        if not images:
-            # Plain-text inputs (e.g. text-only evaluation like MMLU/GSM8K): tokenize
-            # directly without wrapping in a multi-modal chat conversation, which would
-            # otherwise inject chat-template tokens and corrupt continuation prompts.
-            encoded = self.tokenizer.transformers_tokenizer(text,
-                                                            return_tensors='pt')
-            return {"input_ids": encoded["input_ids"]}
-
         mm_items = [{
             "type": "image",
             "base64": encode_base64_image(image)
-        } for image in images]
+        } for image in (images or [])]
 
         conversation = [{
             "role": "user",
@@ -342,11 +334,8 @@ def __call__(self, text, images=None, **kwargs):
         encoded = self.tokenizer.transformers_tokenizer.apply_chat_template(
             conversation, tokenize=True, return_dict=True, return_tensors='pt')
 
-        processed = {
-            "input_ids": encoded.input_ids,
-        }
+        processed = {"input_ids": encoded.input_ids}
 
-        # text-only mode for VLM
         if "pixel_values" in encoded:
             processed.update({
                 "pixel_values":

From ba1895bcd69cfa0f399e4e8582cf2ad07510f69d Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Mon, 8 Jun 2026 15:12:31 -0700
Subject: [PATCH 05/19] text-only Mistral models fix

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/llm-api/quickstart_advanced.py        |  7 +++++++
 examples/models/core/mistral_large_3/README.md | 13 +++++++++++--
 .../models/checkpoints/mistral/tokenizer.py    |  2 +-
 .../checkpoints/mistral/weight_mapper.py       |  3 +++
 tensorrt_llm/_torch/models/modeling_mistral.py | 18 +++++++++++++-----
 tensorrt_llm/tokenizer/tokenizer.py            |  5 ++++-
 6 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/examples/llm-api/quickstart_advanced.py b/examples/llm-api/quickstart_advanced.py
index d2acd490d44a..5e07f364b3a7 100644
--- a/examples/llm-api/quickstart_advanced.py
+++ b/examples/llm-api/quickstart_advanced.py
@@ -194,6 +194,12 @@ def add_llm_args(parser):
     parser.add_argument('--apply_chat_template',
                         default=False,
                         action='store_true')
+    parser.add_argument(
+        '--custom_tokenizer',
+        type=str,
+        default=None,
+        help='Override the tokenizer. Accepts a built-in alias '
+        "(e.g. 'mistral') or a fully-qualified class import path.")
 
     # Sampling
     parser.add_argument("--max_tokens", type=int, default=64)
@@ -390,6 +396,7 @@ def setup_llm(args, **kwargs):
         gather_generation_logits=args.return_generation_logits,
         max_beam_width=args.max_beam_width,
         orchestrator_type=args.orchestrator_type,
+        custom_tokenizer=args.custom_tokenizer,
         **kwargs)
 
     use_beam_search = args.max_beam_width > 1
diff --git a/examples/models/core/mistral_large_3/README.md b/examples/models/core/mistral_large_3/README.md
index da219bf7b0a8..c203b553779c 100644
--- a/examples/models/core/mistral_large_3/README.md
+++ b/examples/models/core/mistral_large_3/README.md
@@ -9,6 +9,8 @@ export mistral_large_3_eagle_model_path=<mistral_large_3_eagle_model_path>
 
 ## Multimodal run
 
+Native Mistral checkpoints are using HF tokenizer and multimodal processor by default. Pass `--input_processor mistral3_common` to select the native processor explicitly.
+
 * Run the Mistral Large V3 by `quickstart_multimodal.py`
 
 ```bash
@@ -20,11 +22,14 @@ mpirun -n 1 --allow-run-as-root --oversubscribe python3 examples/llm-api/quickst
     --checkpoint_format mistral \
     --model_type mistral_large_3 \
     --moe_backend TRTLLM \
-    --image_format pil
+    --image_format pil \
+    --input_processor mistral3_common # optional
 ```
 
 ## LLM-only run
 
+To use the mistral-common tokenizer instead of the HuggingFace tokenizer, pass `--custom_tokenizer mistral`. This is optional; the HuggingFace tokenizer is used by default.
+
 * Run the Mistral Large V3 by `quickstart_advanced.py`
 
 ```bash
@@ -34,7 +39,8 @@ mpirun -n 1 --allow-run-as-root --oversubscribe python3 examples/llm-api/quickst
     --moe_ep_size 4 \
     --max_tokens 100 \
     --checkpoint_format mistral \
-    --moe_backend TRTLLM
+    --moe_backend TRTLLM \
+    --custom_tokenizer mistral # optional
 ```
 
 ```bash
@@ -61,6 +67,9 @@ backend: pytorch
 tensor_parallel_size: 4
 moe_expert_parallel_size: 4
 checkpoint_format: mistral
+# Optional fields to use native preprocessing
+input_processor: mistral3_common 
+custom_tokenizer: mistral
 " > serve.yml
 mpirun -n 1 --allow-run-as-root --oversubscribe python3 -m tensorrt_llm.commands.serve serve \
     ${mistral_large_3_model_path} \
diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py b/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py
index eb7ac6b573d3..426e85592ba4 100644
--- a/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py
+++ b/tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py
@@ -301,4 +301,4 @@ def hf_decode_incrementally(
     def apply_chat_template(
         self, conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]], *args, **kwargs
     ) -> Union[str, list[int], list[str], list[list[int]]]:
-        raise NotImplementedError
+        return self.transformers_tokenizer.apply_chat_template(conversation, *args, **kwargs)
diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py
index dd6e0332b849..e4d50578fe5a 100644
--- a/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py
+++ b/tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py
@@ -35,6 +35,9 @@ def __init__(self):
             "tok_embeddings": "model.embed_tokens",
             "output": "lm_head",
             "norm": "model.norm",
+            # For text-only models: preprocess_weights adds "language_model." prefix
+            "language_model.tok_embeddings": "model.embed_tokens",
+            "language_model.output": "lm_head",
             # For Eagle3
             "language_model.eagle_linear": "model.fc",
             "language_model.layers": "layers",
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index e30d9cf880f8..96a700ff4815 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -263,6 +263,18 @@ def __init__(
             vocab_size=model_config.pretrained_config.vocab_size,
         )
 
+    def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs):
+        if weight_mapper and type(weight_mapper) is MistralWeightMapper:
+            weight_mapper.permute_qk(weights=weights, config=self.config)
+            super().load_weights(weights,
+                                 weight_mapper=weight_mapper,
+                                 params_map=weight_mapper.mistral_llm_mapping)
+        else:
+            super().load_weights(weights,
+                                 weight_mapper=weight_mapper,
+                                 *args,
+                                 **kwargs)
+
 
 class MistralCommonImageProcessor:
 
@@ -765,11 +777,7 @@ def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs):
         llm_weights = filter_weights(weights=weights, prefix="language_model")
         logger.debug(f"Loading weights for {type(self.llm)}")
         if weight_mapper and type(weight_mapper) is MistralWeightMapper:
-            weight_mapper.permute_qk(weights=llm_weights,
-                                     config=self.llm.config)
-            self.llm.load_weights(llm_weights,
-                                  weight_mapper=weight_mapper,
-                                  params_map=weight_mapper.mistral_llm_mapping)
+            self.llm.load_weights(llm_weights, weight_mapper=weight_mapper)
         else:
             self.llm.load_weights(llm_weights)
         logger.debug(f"Successfully loaded weights for {type(self.llm)}")
diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py
index 1422cc3804b5..9b14218bc051 100644
--- a/tensorrt_llm/tokenizer/tokenizer.py
+++ b/tensorrt_llm/tokenizer/tokenizer.py
@@ -40,8 +40,11 @@
 
 # Aliases for built-in custom tokenizers.
 TOKENIZER_ALIASES = {
-    "deepseek_v32": "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer",
+    "deepseek_v32":
+    "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer",
     "deepseek_v4": "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer",
+    "mistral":
+    "tensorrt_llm._torch.models.checkpoints.mistral.tokenizer.MistralTokenizer",
 }
 
 TLLM_INCREMENTAL_DETOKENIZATION_BACKEND = os.environ.get(

From 8fb00d13f03708e1d0b15f323aa6fd5f96eb85f3 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Mon, 8 Jun 2026 15:23:56 -0700
Subject: [PATCH 06/19] unify naming

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/models/core/mistral_large_3/README.md         | 10 +++++-----
 .../_torch/models/checkpoints/mistral/config_loader.py |  1 -
 tensorrt_llm/_torch/models/modeling_mistral.py         |  4 ++--
 tensorrt_llm/inputs/registry.py                        |  2 +-
 tensorrt_llm/tokenizer/tokenizer.py                    |  2 +-
 5 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/examples/models/core/mistral_large_3/README.md b/examples/models/core/mistral_large_3/README.md
index c203b553779c..47e374338ce1 100644
--- a/examples/models/core/mistral_large_3/README.md
+++ b/examples/models/core/mistral_large_3/README.md
@@ -9,7 +9,7 @@ export mistral_large_3_eagle_model_path=<mistral_large_3_eagle_model_path>
 
 ## Multimodal run
 
-Native Mistral checkpoints are using HF tokenizer and multimodal processor by default. Pass `--input_processor mistral3_common` to select the native processor explicitly.
+Native Mistral checkpoints are using HF tokenizer and multimodal processor by default. Pass `--input_processor mistral_common` to select the native processor explicitly.
 
 * Run the Mistral Large V3 by `quickstart_multimodal.py`
 
@@ -23,12 +23,12 @@ mpirun -n 1 --allow-run-as-root --oversubscribe python3 examples/llm-api/quickst
     --model_type mistral_large_3 \
     --moe_backend TRTLLM \
     --image_format pil \
-    --input_processor mistral3_common # optional
+    --input_processor mistral_common # optional
 ```
 
 ## LLM-only run
 
-To use the mistral-common tokenizer instead of the HuggingFace tokenizer, pass `--custom_tokenizer mistral`. This is optional; the HuggingFace tokenizer is used by default.
+To use the mistral-common tokenizer instead of the HuggingFace tokenizer, pass `--custom_tokenizer mistral_common`. This is optional; the HuggingFace tokenizer is used by default.
 
 * Run the Mistral Large V3 by `quickstart_advanced.py`
 
@@ -40,7 +40,7 @@ mpirun -n 1 --allow-run-as-root --oversubscribe python3 examples/llm-api/quickst
     --max_tokens 100 \
     --checkpoint_format mistral \
     --moe_backend TRTLLM \
-    --custom_tokenizer mistral # optional
+    --custom_tokenizer mistral_common # optional
 ```
 
 ```bash
@@ -68,7 +68,7 @@ tensor_parallel_size: 4
 moe_expert_parallel_size: 4
 checkpoint_format: mistral
 # Optional fields to use native preprocessing
-input_processor: mistral3_common 
+input_processor: mistral_common 
 custom_tokenizer: mistral
 " > serve.yml
 mpirun -n 1 --allow-run-as-root --oversubscribe python3 -m tensorrt_llm.commands.serve serve \
diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
index b424a11da036..d611f705c815 100644
--- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
+++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
@@ -390,7 +390,6 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig:
         from tensorrt_llm._torch.models.modeling_mistral_large3 import Mistral3Gate
 
         model_config.pretrained_config.gate_cls = Mistral3Gate
-        arch = (getattr(pretrained_config, "architectures", None) or [None])[0]
         model_config.pretrained_config.input_processor_type = "mistral3"
         model_config.pretrained_config.model_type = "mistral3"
         model_config._frozen = True
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 96a700ff4815..2552774d881f 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -664,13 +664,13 @@ def call_with_text_prompt(
 # Mistral is the only supported case where HF preprocessor can be used with
 # non-HF checkpoints, so this hack is preferred to changing the registry itself.
 MULTIMODAL_PLACEHOLDER_REGISTRY.set_placeholder_metadata(
-    "mistral3_common",
+    "mistral_common",
     MultimodalPlaceholderMetadata(
         placeholder_map={"image": "[IMG]"},
         placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT,
         content_format=ContentFormat.PASSTHROUGH,
     ))
-MistralNativeInputProcessor._registered_model_type = "mistral3_common"
+MistralNativeInputProcessor._registered_model_type = "mistral_common"
 
 
 @register_auto_model("Mistral3ForConditionalGeneration")
diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py
index 064b661574b0..f0c9b696231d 100644
--- a/tensorrt_llm/inputs/registry.py
+++ b/tensorrt_llm/inputs/registry.py
@@ -946,7 +946,7 @@ def create_input_processor(
         config = model_config.pretrained_config
         # The only case where a model class may use two processors.
         # Will need a more generic approach if more such models appear.
-        if input_processor_override == "mistral3_common":
+        if input_processor_override == "mistral_common":
             from tensorrt_llm._torch.models.modeling_mistral import \
                 MistralNativeInputProcessor
             return MistralNativeInputProcessor(
diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py
index 9b14218bc051..fe06292e9699 100644
--- a/tensorrt_llm/tokenizer/tokenizer.py
+++ b/tensorrt_llm/tokenizer/tokenizer.py
@@ -43,7 +43,7 @@
     "deepseek_v32":
     "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer",
     "deepseek_v4": "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer",
-    "mistral":
+    "mistral_common":
     "tensorrt_llm._torch.models.checkpoints.mistral.tokenizer.MistralTokenizer",
 }
 

From 1aad308ff84dc07afa61b3777ed24c6405c86b14 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Tue, 9 Jun 2026 11:36:22 -0700
Subject: [PATCH 07/19] Mistral processor disambiguation in tests

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tensorrt_llm/inputs/utils.py                         |  6 ++++++
 .../defs/accuracy/test_llm_api_pytorch_multimodal.py | 12 +++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/tensorrt_llm/inputs/utils.py b/tensorrt_llm/inputs/utils.py
index 70ddbd8f055d..e769e22cb7d3 100644
--- a/tensorrt_llm/inputs/utils.py
+++ b/tensorrt_llm/inputs/utils.py
@@ -18,6 +18,8 @@
 from transformers import AutoProcessor, ProcessorMixin
 from transformers.utils import logging
 
+from tensorrt_llm._torch.models.modeling_mistral import \
+    MistralCommonImageProcessor
 from tensorrt_llm.inputs.content_format import (ContentFormat,
                                                 detect_content_format)
 from tensorrt_llm.inputs.media_io import (_get_aiohttp_session,
@@ -706,6 +708,10 @@ def apply_chat_template(
             return tokenizer.encode(prompt)
         return prompt
 
+    # Special handling of Mistral native processor
+    if isinstance(processor, MistralCommonImageProcessor):
+        model_type = "mistral_common"
+
     # Check for PASSTHROUGH early — before we need tokenizer/processor/template.
     # The registry may already know this model skips chat templates entirely.
     registry_format = MULTIMODAL_PLACEHOLDER_REGISTRY.get_content_format(
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
index 91bd03c3df11..f2e841410ef1 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
@@ -472,9 +472,9 @@ class TestMistralLarge3_675B(LlmapiAccuracyTestHarness):
     @pytest.mark.skip_less_mpi_world_size(4)
     @pytest.mark.skip_less_device_memory(183000)
     @pytest.mark.parametrize(
-        "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler,moe_backend",
+        "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler,moe_backend,input_processor",
         [
-            (4, 1, 4, False, True, True, "TRTLLM"),
+            (4, 1, 4, False, True, True, "TRTLLM", "mistral3"),
         ],
         ids=[
             "latency_moe_trtllm",
@@ -489,10 +489,15 @@ def test_nvfp4_4gpus(
         cuda_graph,
         overlap_scheduler,
         moe_backend,
+        input_processor,
         mocker,
     ):
         mocker.patch.dict(
-            MMMU.EVALUATE_KWARGS, {"model_type": "mistral_large_3", "is_force_single_image": True}
+            MMMU.EVALUATE_KWARGS,
+            {
+                "model_type": "mistral_large_3",
+                "is_force_single_image": input_processor == "mistral_common",
+            },
         )
         pytorch_config = dict(
             disable_overlap_scheduler=not overlap_scheduler,
@@ -506,6 +511,7 @@ def test_nvfp4_4gpus(
             self.MODEL_PATH,
             max_num_tokens=self.MAX_NUM_TOKENS,
             checkpoint_format="mistral_large_3",
+            input_processor=input_processor,
             tensor_parallel_size=tp_size,
             pipeline_parallel_size=pp_size,
             moe_expert_parallel_size=ep_size,

From 429f3e429b1f792889bd7484c36993751aaea3cd Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Tue, 9 Jun 2026 13:53:20 -0700
Subject: [PATCH 08/19] remove circular import

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tensorrt_llm/_torch/models/modeling_mistral.py | 1 +
 tensorrt_llm/inputs/utils.py                   | 6 ++----
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 2552774d881f..ac5ae356aaad 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -277,6 +277,7 @@ def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs):
 
 
 class MistralCommonImageProcessor:
+    is_mistral_native_processor = True
 
     def __init__(self, tokenizer: MistralTokenizer, dtype) -> None:
         super().__init__()
diff --git a/tensorrt_llm/inputs/utils.py b/tensorrt_llm/inputs/utils.py
index e769e22cb7d3..3b763e4832eb 100644
--- a/tensorrt_llm/inputs/utils.py
+++ b/tensorrt_llm/inputs/utils.py
@@ -18,8 +18,6 @@
 from transformers import AutoProcessor, ProcessorMixin
 from transformers.utils import logging
 
-from tensorrt_llm._torch.models.modeling_mistral import \
-    MistralCommonImageProcessor
 from tensorrt_llm.inputs.content_format import (ContentFormat,
                                                 detect_content_format)
 from tensorrt_llm.inputs.media_io import (_get_aiohttp_session,
@@ -708,8 +706,8 @@ def apply_chat_template(
             return tokenizer.encode(prompt)
         return prompt
 
-    # Special handling of Mistral native processor
-    if isinstance(processor, MistralCommonImageProcessor):
+    # Special handling of Mistral native processor (duck-typed to avoid circular import)
+    if getattr(processor, "is_mistral_native_processor", False):
         model_type = "mistral_common"
 
     # Check for PASSTHROUGH early — before we need tokenizer/processor/template.

From 8807d8a2c94f614728da7390cce79174eb0af77d Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Wed, 10 Jun 2026 07:31:32 -0700
Subject: [PATCH 09/19] revert arbitrary processor selection

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/llm-api/quickstart_multimodal.py     |  9 -------
 .../models/core/mistral_large_3/README.md     | 13 ++--------
 .../_torch/models/modeling_mistral.py         |  1 -
 tensorrt_llm/inputs/registry.py               | 25 ++++++-------------
 tensorrt_llm/inputs/utils.py                  |  4 ---
 .../test_llm_api_pytorch_multimodal.py        |  8 +++---
 .../_torch/modeling/test_modeling_mistral.py  |  2 +-
 7 files changed, 14 insertions(+), 48 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index 86f2c879330b..471986294304 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -138,13 +138,6 @@ def add_multimodal_args(parser):
                         default=None,
                         help="Pruning rate for video frames (EVS). "
                         "None disables EVS, values in [0, 1) enable pruning.")
-    parser.add_argument(
-        "--input_processor",
-        type=str,
-        default=None,
-        help="Override the automatically selected input processor. "
-        "Must be a registered model_type. None (default) selects automatically."
-    )
     return parser
 
 
@@ -206,8 +199,6 @@ def main():
     image_format = args.image_format
     if args.model_type is not None:
         model_type = args.model_type
-    elif hasattr(llm.input_processor, '_registered_model_type'):
-        model_type = llm.input_processor._registered_model_type
     else:
         model_type = json.load(
             open(os.path.join(str(llm._hf_model_dir),
diff --git a/examples/models/core/mistral_large_3/README.md b/examples/models/core/mistral_large_3/README.md
index 47e374338ce1..da219bf7b0a8 100644
--- a/examples/models/core/mistral_large_3/README.md
+++ b/examples/models/core/mistral_large_3/README.md
@@ -9,8 +9,6 @@ export mistral_large_3_eagle_model_path=<mistral_large_3_eagle_model_path>
 
 ## Multimodal run
 
-Native Mistral checkpoints are using HF tokenizer and multimodal processor by default. Pass `--input_processor mistral_common` to select the native processor explicitly.
-
 * Run the Mistral Large V3 by `quickstart_multimodal.py`
 
 ```bash
@@ -22,14 +20,11 @@ mpirun -n 1 --allow-run-as-root --oversubscribe python3 examples/llm-api/quickst
     --checkpoint_format mistral \
     --model_type mistral_large_3 \
     --moe_backend TRTLLM \
-    --image_format pil \
-    --input_processor mistral_common # optional
+    --image_format pil
 ```
 
 ## LLM-only run
 
-To use the mistral-common tokenizer instead of the HuggingFace tokenizer, pass `--custom_tokenizer mistral_common`. This is optional; the HuggingFace tokenizer is used by default.
-
 * Run the Mistral Large V3 by `quickstart_advanced.py`
 
 ```bash
@@ -39,8 +34,7 @@ mpirun -n 1 --allow-run-as-root --oversubscribe python3 examples/llm-api/quickst
     --moe_ep_size 4 \
     --max_tokens 100 \
     --checkpoint_format mistral \
-    --moe_backend TRTLLM \
-    --custom_tokenizer mistral_common # optional
+    --moe_backend TRTLLM
 ```
 
 ```bash
@@ -67,9 +61,6 @@ backend: pytorch
 tensor_parallel_size: 4
 moe_expert_parallel_size: 4
 checkpoint_format: mistral
-# Optional fields to use native preprocessing
-input_processor: mistral_common 
-custom_tokenizer: mistral
 " > serve.yml
 mpirun -n 1 --allow-run-as-root --oversubscribe python3 -m tensorrt_llm.commands.serve serve \
     ${mistral_large_3_model_path} \
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index ac5ae356aaad..2552774d881f 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -277,7 +277,6 @@ def load_weights(self, weights: Dict, weight_mapper=None, *args, **kwargs):
 
 
 class MistralCommonImageProcessor:
-    is_mistral_native_processor = True
 
     def __init__(self, tokenizer: MistralTokenizer, dtype) -> None:
         super().__init__()
diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py
index f0c9b696231d..acd3f67f9f27 100644
--- a/tensorrt_llm/inputs/registry.py
+++ b/tensorrt_llm/inputs/registry.py
@@ -914,9 +914,7 @@ def create_input_processor(
         trust_remote_code: Whether Hugging Face config/processor loading may run
             model-provided Python code.
         **kwargs: Additional arguments passed to input processor constructors
-            (e.g., video_pruning_rate for multimodal models).  A special key
-            ``input_processor`` (str) overrides automatic processor selection with
-            the named registered model_type.
+            (e.g., video_pruning_rate for multimodal models).
 
     Returns:
         An InputProcessor implementation (model-specific if registered; otherwise DefaultInputProcessor).
@@ -924,9 +922,6 @@ def create_input_processor(
     from tensorrt_llm._torch.model_config import ModelConfig
     from tensorrt_llm._torch.models import get_model_architecture
 
-    # Pop the override before passing kwargs to constructors.
-    input_processor_override = kwargs.pop("input_processor", None)
-
     config = None
 
     if checkpoint_format == "HF":
@@ -942,19 +937,15 @@ def create_input_processor(
         logger.debug(f"Detected checkpoint_format={checkpoint_format}.")
         from tensorrt_llm._torch.models.checkpoints.mistral.config_loader import \
             MistralConfigLoader
+        from tensorrt_llm._torch.models.modeling_mistral import \
+            MistralNativeInputProcessor
         model_config = MistralConfigLoader().load(model_path_or_dir)
         config = model_config.pretrained_config
-        # The only case where a model class may use two processors.
-        # Will need a more generic approach if more such models appear.
-        if input_processor_override == "mistral_common":
-            from tensorrt_llm._torch.models.modeling_mistral import \
-                MistralNativeInputProcessor
-            return MistralNativeInputProcessor(
-                model_path_or_dir,
-                config,
-                tokenizer,
-                trust_remote_code=trust_remote_code,
-                **kwargs)
+        return MistralNativeInputProcessor(model_path_or_dir,
+                                           config,
+                                           tokenizer,
+                                           trust_remote_code=trust_remote_code,
+                                           **kwargs)
     else:
         logger.debug(
             f"checkpoint_format={checkpoint_format}; skipping HF config load.")
diff --git a/tensorrt_llm/inputs/utils.py b/tensorrt_llm/inputs/utils.py
index 3b763e4832eb..70ddbd8f055d 100644
--- a/tensorrt_llm/inputs/utils.py
+++ b/tensorrt_llm/inputs/utils.py
@@ -706,10 +706,6 @@ def apply_chat_template(
             return tokenizer.encode(prompt)
         return prompt
 
-    # Special handling of Mistral native processor (duck-typed to avoid circular import)
-    if getattr(processor, "is_mistral_native_processor", False):
-        model_type = "mistral_common"
-
     # Check for PASSTHROUGH early — before we need tokenizer/processor/template.
     # The registry may already know this model skips chat templates entirely.
     registry_format = MULTIMODAL_PLACEHOLDER_REGISTRY.get_content_format(
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
index f2e841410ef1..0b3c09bbc77c 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
@@ -472,9 +472,9 @@ class TestMistralLarge3_675B(LlmapiAccuracyTestHarness):
     @pytest.mark.skip_less_mpi_world_size(4)
     @pytest.mark.skip_less_device_memory(183000)
     @pytest.mark.parametrize(
-        "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler,moe_backend,input_processor",
+        "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler,moe_backend",
         [
-            (4, 1, 4, False, True, True, "TRTLLM", "mistral3"),
+            (4, 1, 4, False, True, True, "TRTLLM"),
         ],
         ids=[
             "latency_moe_trtllm",
@@ -489,14 +489,13 @@ def test_nvfp4_4gpus(
         cuda_graph,
         overlap_scheduler,
         moe_backend,
-        input_processor,
         mocker,
     ):
         mocker.patch.dict(
             MMMU.EVALUATE_KWARGS,
             {
                 "model_type": "mistral_large_3",
-                "is_force_single_image": input_processor == "mistral_common",
+                "is_force_single_image": True,
             },
         )
         pytorch_config = dict(
@@ -511,7 +510,6 @@ def test_nvfp4_4gpus(
             self.MODEL_PATH,
             max_num_tokens=self.MAX_NUM_TOKENS,
             checkpoint_format="mistral_large_3",
-            input_processor=input_processor,
             tensor_parallel_size=tp_size,
             pipeline_parallel_size=pp_size,
             moe_expert_parallel_size=ep_size,
diff --git a/tests/unittest/_torch/modeling/test_modeling_mistral.py b/tests/unittest/_torch/modeling/test_modeling_mistral.py
index 793b70455057..259b53d4eb44 100644
--- a/tests/unittest/_torch/modeling/test_modeling_mistral.py
+++ b/tests/unittest/_torch/modeling/test_modeling_mistral.py
@@ -540,7 +540,7 @@ def test_processor_get_num_tokens_per_image(
     with mock.patch(
         "tensorrt_llm._torch.models.modeling_mistral.AutoProcessor"
     ) as mocked_auto_processor:
-        input_processor = modeling_mistral.Mistral3InputProcessor(
+        input_processor = modeling_mistral.MistralHFInputProcessor(
             model_path=str(tmp_path),
             config=mistral_3_config,
             tokenizer=mock.MagicMock(),

From 7e89e28c2b12b3e56ae0471fc964ee67b74ac291 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Thu, 11 Jun 2026 01:53:41 -0700
Subject: [PATCH 10/19] model_type fix in test

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 .../defs/accuracy/test_llm_api_pytorch_multimodal.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
index 0b3c09bbc77c..3cd3c1c88868 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py
@@ -494,7 +494,7 @@ def test_nvfp4_4gpus(
         mocker.patch.dict(
             MMMU.EVALUATE_KWARGS,
             {
-                "model_type": "mistral_large_3",
+                "model_type": "mistral_common",
                 "is_force_single_image": True,
             },
         )

From a91743cd7a6e3223a6cb399af5a000f8f926cc52 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Thu, 11 Jun 2026 02:31:26 -0700
Subject: [PATCH 11/19] remove stale base processor

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/llm-api/quickstart_multimodal.py     |  2 +
 .../_torch/models/modeling_mistral.py         | 97 +++++++++----------
 2 files changed, 46 insertions(+), 53 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index 471986294304..6aaa76055e4a 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -199,6 +199,8 @@ def main():
     image_format = args.image_format
     if args.model_type is not None:
         model_type = args.model_type
+    elif hasattr(llm.input_processor, '_registered_model_type'):
+        model_type = llm.input_processor._registered_model_type
     else:
         model_type = json.load(
             open(os.path.join(str(llm._hf_model_dir),
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 2552774d881f..525060089aba 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -360,57 +360,6 @@ def __call__(self, text, images=None, **kwargs):
         return processed
 
 
-class MistralInputProcessorBase(BaseMultimodalInputProcessor,
-                                BaseMultimodalDummyInputsBuilder):
-    """Shared logic for HF and native Mistral VLM input processors."""
-
-    @property
-    def config(self) -> PretrainedConfig:
-        return self._config
-
-    @property
-    def tokenizer(self) -> AutoTokenizer:
-        return self._tokenizer
-
-    @property
-    def model_path(self) -> str:
-        return self._model_path
-
-    @property
-    def processor(self) -> AutoProcessor:
-        return self._processor
-
-    @property
-    def dtype(self) -> torch.dtype:
-        return self._dtype
-
-    def get_vocab_size(self) -> int:
-        return self.config.text_config.vocab_size
-
-    def get_mm_token_ids(self) -> torch.Tensor:
-        return torch.tensor([
-            self.processor.image_token_id,
-            self.processor.image_break_token_id,
-            self.processor.image_end_token_id,
-        ])
-
-    def get_mm_special_token_ids(self) -> torch.Tensor:
-        return torch.tensor([
-            self.processor.image_break_token_id,
-            self.processor.image_end_token_id,
-        ])
-
-    @staticmethod
-    def _extract_extra_processed_inputs(
-            processed) -> ExtraProcessedInputs | None:
-        pixel_values = processed.get("pixel_values")
-        if pixel_values is None:
-            return None
-        processed.pop("attention_mask", None)
-        processed["image_sizes"] = processed["image_sizes"].tolist()
-        return {"multimodal_data": {"image": {**processed}}}
-
-
 class MistralHFInputProcessor(BaseMultimodalInputProcessor,
                               BaseMultimodalDummyInputsBuilder):
     """Input processor for Mistral VLM checkpoints in HuggingFace format."""
@@ -619,7 +568,8 @@ def get_dummy_mm_data_for_tokens(
                                                dtype=dtype)
 
 
-class MistralNativeInputProcessor(MistralInputProcessorBase):
+class MistralNativeInputProcessor(BaseMultimodalInputProcessor,
+                                  BaseMultimodalDummyInputsBuilder):
     """Input processor for Mistral VLM checkpoints in mistral-native format."""
 
     def __init__(
@@ -645,6 +595,42 @@ def __init__(
             f"[mistral] native processor={type(self._processor).__name__} "
             f"tokenizer={type(self._tokenizer).__name__}")
 
+    @property
+    def config(self) -> PretrainedConfig:
+        return self._config
+
+    @property
+    def tokenizer(self) -> AutoTokenizer:
+        return self._tokenizer
+
+    @property
+    def model_path(self) -> str:
+        return self._model_path
+
+    @property
+    def processor(self) -> AutoProcessor:
+        return self._processor
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return self._dtype
+
+    def get_vocab_size(self) -> int:
+        return self.config.text_config.vocab_size
+
+    def get_mm_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            self.processor.image_token_id,
+            self.processor.image_break_token_id,
+            self.processor.image_end_token_id,
+        ])
+
+    def get_mm_special_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            self.processor.image_break_token_id,
+            self.processor.image_end_token_id,
+        ])
+
     @torch.inference_mode()
     def call_with_text_prompt(
         self, inputs: TextPrompt, sampling_params: SamplingParams
@@ -654,7 +640,12 @@ def call_with_text_prompt(
         # the mistral-common chat template internally.
         processed = self.processor(text=inputs["prompt"], images=images)
         input_ids = processed.pop("input_ids").tolist()[0]
-        return input_ids, self._extract_extra_processed_inputs(processed)
+        pixel_values = processed.get("pixel_values")
+        if pixel_values is None:
+            return input_ids, None
+        processed.pop("attention_mask", None)
+        processed["image_sizes"] = processed["image_sizes"].tolist()
+        return input_ids, {"multimodal_data": {"image": {**processed}}}
 
 
 # Register the native processor's content-format metadata.  We do this

From 6cd35b851cc573fcc706f1aa9603a46e8e5e2234 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Thu, 11 Jun 2026 03:06:57 -0700
Subject: [PATCH 12/19] review fixes

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/llm-api/quickstart_advanced.py           | 11 +++++------
 .../_torch/attention_backend/interface.py         | 15 +++++++++------
 tensorrt_llm/_torch/models/modeling_mistral.py    | 12 ++----------
 3 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/examples/llm-api/quickstart_advanced.py b/examples/llm-api/quickstart_advanced.py
index 5e07f364b3a7..bb06009bdc33 100644
--- a/examples/llm-api/quickstart_advanced.py
+++ b/examples/llm-api/quickstart_advanced.py
@@ -194,12 +194,11 @@ def add_llm_args(parser):
     parser.add_argument('--apply_chat_template',
                         default=False,
                         action='store_true')
-    parser.add_argument(
-        '--custom_tokenizer',
-        type=str,
-        default=None,
-        help='Override the tokenizer. Accepts a built-in alias '
-        "(e.g. 'mistral') or a fully-qualified class import path.")
+    parser.add_argument('--custom_tokenizer',
+                        type=str,
+                        default=None,
+                        help='Override the tokenizer. Accepts a built-in alias '
+                        " or a fully-qualified class import path.")
 
     # Sampling
     parser.add_argument("--max_tokens", type=int, default=64)
diff --git a/tensorrt_llm/_torch/attention_backend/interface.py b/tensorrt_llm/_torch/attention_backend/interface.py
index 0faad4916123..bb571227ce5c 100644
--- a/tensorrt_llm/_torch/attention_backend/interface.py
+++ b/tensorrt_llm/_torch/attention_backend/interface.py
@@ -674,6 +674,7 @@ def from_config(config) -> "RopeParams":
         rope_params = RopeParams()
 
         hf_rope_parameters = getattr(config, 'rope_parameters', None)
+        normalized_rope_parameters = hf_rope_parameters
         if hf_rope_parameters is not None:
             if set(hf_rope_parameters.keys()).issubset(
                     ALLOWED_ATTENTION_LAYER_TYPES):
@@ -681,17 +682,19 @@ def from_config(config) -> "RopeParams":
                 # Pick "full_attention" as the default; callers override theta
                 # for sliding-window layers independently.
                 if "full_attention" in hf_rope_parameters:
-                    flat = hf_rope_parameters["full_attention"]
+                    normalized_rope_parameters = hf_rope_parameters[
+                        "full_attention"]
                 else:
                     fallback_key = next(iter(hf_rope_parameters))
                     logger.warning(
                         f"Per-layer-type rope_parameters has no 'full_attention' entry; "
                         f"falling back to '{fallback_key}'. Available layer types: "
                         f"{list(hf_rope_parameters.keys())}.")
-                    flat = hf_rope_parameters[fallback_key]
-                config.update(flat)
+                    normalized_rope_parameters = hf_rope_parameters[
+                        fallback_key]
+                config.update(normalized_rope_parameters)
             else:
-                config.update(hf_rope_parameters)
+                config.update(normalized_rope_parameters)
 
         # get rotary parameters.
         hidden_size = config.hidden_size
@@ -700,8 +703,8 @@ def from_config(config) -> "RopeParams":
         if not isinstance(head_dim, int):
             head_dim = hidden_size // num_attention_heads
         rope_scaling = getattr(config, 'rope_scaling', None)
-        if rope_scaling is None and hf_rope_parameters is not None:
-            rope_scaling = hf_rope_parameters
+        if rope_scaling is None and normalized_rope_parameters is not None:
+            rope_scaling = normalized_rope_parameters
         rope_params.max_positions = config.max_position_embeddings
         rope_params.theta = get_hf_rope_theta(config, 10000.0)
         rope_percentage = (getattr(config, 'rotary_pct', None)
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 525060089aba..bfe1c9a49f88 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -39,7 +39,7 @@
 from tensorrt_llm._torch.modules.rms_norm import RMSNorm
 from tensorrt_llm._torch.speculative import SpecMetadata
 from tensorrt_llm._utils import nvtx_range
-from tensorrt_llm.functional import PositionEmbeddingType
+from tensorrt_llm.functional import PositionEmbeddingType, RotaryScalingType
 from tensorrt_llm.inputs import (BaseMultimodalDummyInputsBuilder,
                                  BaseMultimodalInputProcessor, ContentFormat,
                                  ExtraProcessedInputs,
@@ -62,16 +62,8 @@ def __init__(
     ):
         config = model_config.pretrained_config
         rope_params = RopeParams.from_config(config)
-        rope_params_section = getattr(config, "rope_scaling", None) or getattr(
-            config, "rope_parameters", None)
 
-        rope_type = getattr(
-            rope_params_section, "rope_type",
-            None) or (rope_params_section.get("rope_type") if isinstance(
-                rope_params_section, dict) else None) or getattr(
-                    config, "rope_type", None)
-
-        if rope_type == "yarn":
+        if rope_params.scale_type == RotaryScalingType.yarn:
             pos_embd_params = PositionalEmbeddingParams(
                 type=PositionEmbeddingType.yarn,
                 rope=rope_params,

From 2bc73bd1d8b489d3ab9538e9a95d3ca4943bf4e4 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Fri, 12 Jun 2026 01:21:11 -0700
Subject: [PATCH 13/19] unwaive

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tests/integration/test_lists/waives.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt
index 6dc4776bb633..e290f7c277ea 100644
--- a/tests/integration/test_lists/waives.txt
+++ b/tests/integration/test_lists/waives.txt
@@ -95,9 +95,6 @@ accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torc
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=False] SKIP (https://nvbugs/6427411)
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp4_tp2pp2[torch_compile=True-enable_gemm_allreduce_fusion=True] SKIP (https://nvbugs/6211441)
 accuracy/test_llm_api_pytorch.py::TestMiniMaxM2::test_4gpus[attention_dp=False-cuda_graph=True-overlap_scheduler=True-tp_size=4-ep_size=4] SKIP (https://nvbugs/6159132)
-accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] SKIP (https://nvbugs/6478720)
-accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6248827)
-accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] SKIP (https://nvbugs/6157892)
 accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] SKIP (https://nvbugs/6427411)
 accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] SKIP (https://nvbugs/6427411)
 accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tep4] SKIP (https://nvbugs/6255417)

From cf9217e935f1346acaf601d053b519e1ee16857e Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Tue, 16 Jun 2026 09:30:09 -0700
Subject: [PATCH 14/19] hardcode mistral_common for native checkpoints +
 comments

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 examples/llm-api/quickstart_multimodal.py     |  2 -
 .../checkpoints/mistral/config_loader.py      | 11 +++-
 .../_torch/models/modeling_mistral.py         | 62 ++++++++++++-------
 tensorrt_llm/serve/chat_utils.py              |  3 +-
 tensorrt_llm/serve/openai_server.py           | 34 +++++-----
 5 files changed, 66 insertions(+), 46 deletions(-)

diff --git a/examples/llm-api/quickstart_multimodal.py b/examples/llm-api/quickstart_multimodal.py
index 6aaa76055e4a..471986294304 100644
--- a/examples/llm-api/quickstart_multimodal.py
+++ b/examples/llm-api/quickstart_multimodal.py
@@ -199,8 +199,6 @@ def main():
     image_format = args.image_format
     if args.model_type is not None:
         model_type = args.model_type
-    elif hasattr(llm.input_processor, '_registered_model_type'):
-        model_type = llm.input_processor._registered_model_type
     else:
         model_type = json.load(
             open(os.path.join(str(llm._hf_model_dir),
diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
index d611f705c815..d77e4fb76ac4 100644
--- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
+++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py
@@ -390,7 +390,14 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig:
         from tensorrt_llm._torch.models.modeling_mistral_large3 import Mistral3Gate
 
         model_config.pretrained_config.gate_cls = Mistral3Gate
-        model_config.pretrained_config.input_processor_type = "mistral3"
-        model_config.pretrained_config.model_type = "mistral3"
+        # Native (mistral-format) checkpoints are served through the
+        # mistral-common tokenizer/processor, which applies its own chat
+        # template. Tag the config with a dedicated serving model_type so the
+        # serving layer resolves the PASSTHROUGH placeholder/chat-template
+        # metadata (registered under "mistral_common" in modeling_mistral.py)
+        # via the normal config path - i.e. resolve_top_level_model_type() -
+        # instead of having to inspect the live input processor.
+        model_config.pretrained_config.input_processor_type = "mistral_common"
+        model_config.pretrained_config.model_type = "mistral_common"
         model_config._frozen = True
         return model_config
diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index bfe1c9a49f88..16d963095fbb 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -338,8 +338,11 @@ def __call__(self, text, images=None, **kwargs):
         encoded = self.tokenizer.transformers_tokenizer.apply_chat_template(
             conversation, tokenize=True, return_dict=True, return_tensors='pt')
 
-        processed = {"input_ids": encoded.input_ids}
+        processed = {
+            "input_ids": encoded.input_ids,
+        }
 
+        # text-only mode for VLM
         if "pixel_values" in encoded:
             processed.update({
                 "pixel_values":
@@ -403,22 +406,6 @@ def processor(self) -> AutoProcessor:
     def dtype(self) -> torch.dtype:
         return self._dtype
 
-    def get_vocab_size(self) -> int:
-        return self.config.text_config.vocab_size
-
-    def get_mm_token_ids(self) -> torch.Tensor:
-        return torch.tensor([
-            self.processor.image_token_id,
-            self.processor.image_break_token_id,
-            self.processor.image_end_token_id,
-        ])
-
-    def get_mm_special_token_ids(self) -> torch.Tensor:
-        return torch.tensor([
-            self.processor.image_break_token_id,
-            self.processor.image_end_token_id,
-        ])
-
     @torch.inference_mode()
     def call_with_text_prompt(
         self, inputs: TextPrompt, sampling_params: SamplingParams
@@ -427,8 +414,8 @@ def call_with_text_prompt(
         do_rescale = getattr(self.processor.image_processor, "do_rescale",
                              False)
         if images is not None and isinstance(images[0], torch.Tensor):
-            # The default multimodal input loader normalises images to [0, 1]
-            # for "pt" tensors but not for PIL images.
+            # The default multimodal input loader will normalize images to [0, 1] when the requested
+            # format is "pt" (pytorch tensors), but not for "pil" (PIL images).
             do_rescale = False
 
         prompt = inputs["prompt"]
@@ -436,13 +423,22 @@ def call_with_text_prompt(
                                    images=images,
                                    do_rescale=do_rescale)
         input_ids = processed.pop("input_ids").tolist()[0]
+        # Remaining in `processed`:
+        # * "attention_mask": [B, num_input_tokens]
+        # * "pixel_values": [B, C, H, W]
+        # * "image_sizes": [B, 2]
         extra_processed_inputs = None
         pixel_values = processed.get("pixel_values")
         if pixel_values is not None:
+            # We have no use for the `attention_mask`.
             processed.pop("attention_mask")
-            # Keep as list to avoid a D2H copy when the tensor would otherwise
-            # be moved to GPU before the model forward.
+            # `image_sizes` is a `[B, 2]` tensor indicating the height and width of each image in the
+            # request. If we keep it as a regular tensor, it would get converted to a CUDA tensor before
+            # reaching the model forward. Since its values are used to infer the amount of padding
+            # + slice the patch embeddings, this would incur a D2H copy. We therefore convert it to a
+            # list here to avoid this.
             processed["image_sizes"] = processed["image_sizes"].tolist()
+            # NOTE: `processed` is a dict-like object, but not actually a dict.
             extra_processed_inputs = {
                 "multimodal_data": {
                     "image": {
@@ -559,6 +555,29 @@ def get_dummy_mm_data_for_tokens(
                                                num_images=num_images,
                                                dtype=dtype)
 
+    def get_vocab_size(self) -> int:
+        """Return the vocab size of the model."""
+        # Unlike some other VLMs, mistral3's vocab size is stored in its `text_config`, not the top-level
+        # config.
+        return self.config.text_config.vocab_size
+
+    def get_mm_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            # This is the `[IMG]` token id inserted into the prompt that should be replaced with image
+            # embeddings.
+            self.processor.image_token_id,
+            # This is the `[IMG_BREAK]` token id at the end of every "row".
+            self.processor.image_break_token_id,
+            # This is the `[IMG_END]` token id to signify the end of an image.
+            self.processor.image_end_token_id,
+        ])
+
+    def get_mm_special_token_ids(self) -> torch.Tensor:
+        return torch.tensor([
+            self.processor.image_break_token_id,
+            self.processor.image_end_token_id,
+        ])
+
 
 class MistralNativeInputProcessor(BaseMultimodalInputProcessor,
                                   BaseMultimodalDummyInputsBuilder):
@@ -675,6 +694,7 @@ def call_with_text_prompt(
     MistralHFInputProcessor,
     model_type="mistral_large_3",
     placeholder_metadata=MultimodalPlaceholderMetadata(
+        # NOTE: mistral-common uses the tokenizer to set placeholders, this will be ignored
         placeholder_map={"image": "[IMG]"},
         placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT,
         content_format=ContentFormat.STRING,
diff --git a/tensorrt_llm/serve/chat_utils.py b/tensorrt_llm/serve/chat_utils.py
index 9132f7fce822..d60d93f2ce45 100644
--- a/tensorrt_llm/serve/chat_utils.py
+++ b/tensorrt_llm/serve/chat_utils.py
@@ -451,8 +451,7 @@ def parse_chat_messages_coroutines(
     """
     conversation = []
     mm_placeholder_counts = []
-    model_type = model_type_override or resolve_top_level_model_type(
-        model_config)
+    model_type = resolve_top_level_model_type(model_config)
     mm_data_tracker = MultimodalDataTracker(
         model_type,
         multimodal_server_config,
diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py
index b05ad04b2446..1d941697c31e 100644
--- a/tensorrt_llm/serve/openai_server.py
+++ b/tensorrt_llm/serve/openai_server.py
@@ -492,13 +492,18 @@ def _init_llm(self, chat_template: Optional[str] = None):
                 self.tokenizer.tokenizer, "name_or_path", None) or getattr(
                     self.tokenizer, "name_or_path", None)
         trust_remote_code = self.generator.args.trust_remote_code
-        try:
-            self.processor = AutoProcessor.from_pretrained(
-                hf_tokenizer_path, trust_remote_code=trust_remote_code)
-        except Exception:
-            logger.debug("Failed to load AutoProcessor or AutoConfig for %s",
-                         hf_tokenizer_path)
+        checkpoint_format = self.generator.args.checkpoint_format
+        if checkpoint_format in ("mistral", "mistral_large_3"):
             self.processor = None
+        else:
+            try:
+                self.processor = AutoProcessor.from_pretrained(
+                    hf_tokenizer_path, trust_remote_code=trust_remote_code)
+            except Exception:
+                logger.debug(
+                    "Failed to load AutoProcessor or AutoConfig for %s",
+                    hf_tokenizer_path)
+                self.processor = None
 
         # load model config
         try:
@@ -515,15 +520,6 @@ def _init_llm(self, chat_template: Optional[str] = None):
 
         self.chat_template = load_chat_template(chat_template)
 
-        # When an explicit input_processor override is active,
-        # use its registered model_type for all chat-template
-        # and placeholder-registry lookups
-        _input_proc = getattr(self.generator, 'input_processor', None)
-        _registered = getattr(_input_proc, '_registered_model_type', None)
-        self.effective_model_type = _registered or resolve_top_level_model_type(
-            self.model_config) if self.model_config is not None else (
-                _registered or "")
-
         # Enable response storage for Responses API
         self.enable_store = (len(
             os.getenv("TRTLLM_RESPONSES_API_DISABLE_STORE", ""))
@@ -1557,7 +1553,7 @@ async def chat_stream_generator(
                     self.model_config,
                     self.multimodal_server_config,
                     request_media_io_kwargs=request.media_io_kwargs,
-                    model_type_override=self.effective_model_type or None)
+                )
             except ValidationError:
                 # ValidatorIterator rejects extra fields; fall back to raw JSON.
                 raw_body = await raw_request.json()
@@ -1567,7 +1563,7 @@ async def chat_stream_generator(
                     self.model_config,
                     self.multimodal_server_config,
                     request_media_io_kwargs=request.media_io_kwargs,
-                    model_type_override=self.effective_model_type or None)
+                )
 
             # Decode base64 int32 prompt_token_ids relayed by the orchestrator.
             if request.prompt_token_ids is None and request.prompt_token_ids_b64:
@@ -1747,7 +1743,7 @@ async def create_mm_embedding_response(promise: RequestOutput):
                     self.model_config,
                     self.multimodal_server_config,
                     request_media_io_kwargs=request.media_io_kwargs,
-                    model_type_override=self.effective_model_type or None)
+                )
             except ValidationError:
                 # ValidatorIterator rejects extra fields; fall back to raw JSON.
                 raw_body = await raw_request.json()
@@ -1757,7 +1753,7 @@ async def create_mm_embedding_response(promise: RequestOutput):
                     self.model_config,
                     self.multimodal_server_config,
                     request_media_io_kwargs=request.media_io_kwargs,
-                    model_type_override=self.effective_model_type or None)
+                )
 
             if request.prompt_token_ids is not None:
                 prompt = request.prompt_token_ids

From 3d282e8787b63a26b78c31959f1435f8e761ae5b Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Thu, 2 Jul 2026 05:13:12 -0700
Subject: [PATCH 15/19] review

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tensorrt_llm/_torch/attention_backend/interface.py | 4 +---
 tensorrt_llm/_torch/model_config.py                | 2 +-
 tensorrt_llm/serve/openai_server.py                | 2 ++
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorrt_llm/_torch/attention_backend/interface.py b/tensorrt_llm/_torch/attention_backend/interface.py
index bb571227ce5c..ea6cfbf2f85e 100644
--- a/tensorrt_llm/_torch/attention_backend/interface.py
+++ b/tensorrt_llm/_torch/attention_backend/interface.py
@@ -692,9 +692,7 @@ def from_config(config) -> "RopeParams":
                         f"{list(hf_rope_parameters.keys())}.")
                     normalized_rope_parameters = hf_rope_parameters[
                         fallback_key]
-                config.update(normalized_rope_parameters)
-            else:
-                config.update(normalized_rope_parameters)
+            config.update(normalized_rope_parameters)
 
         # get rotary parameters.
         hidden_size = config.hidden_size
diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py
index 25fec2bb1978..dbab6f07b75a 100644
--- a/tensorrt_llm/_torch/model_config.py
+++ b/tensorrt_llm/_torch/model_config.py
@@ -502,7 +502,7 @@ def load_hf_quant_config(hf_quant_config, moe_backend, checkpoint_dir=None):
                 or hf_quant_config.get("activation_scheme") == "static"):
             quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES
 
-            block_size = hf_quant_config.get("weight_block_size") or [128, 128]
+            block_size = hf_quant_config.get("weight_block_size", [128, 128])
             assert tuple(block_size) == (
                 128, 128), "FP8_BLOCK_SCALES only supports block_size=(128,128)"
             quant_config.group_size = block_size[0]
diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py
index 1d941697c31e..028145341c84 100644
--- a/tensorrt_llm/serve/openai_server.py
+++ b/tensorrt_llm/serve/openai_server.py
@@ -494,6 +494,8 @@ def _init_llm(self, chat_template: Optional[str] = None):
         trust_remote_code = self.generator.args.trust_remote_code
         checkpoint_format = self.generator.args.checkpoint_format
         if checkpoint_format in ("mistral", "mistral_large_3"):
+            # Do not load HF processor for mistral native checkpoints
+            # even if it is available
             self.processor = None
         else:
             try:

From 2a302b095e4cffbafd1254e43b33edcb8c222d58 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Thu, 2 Jul 2026 11:00:49 -0700
Subject: [PATCH 16/19] review

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tensorrt_llm/tokenizer/tokenizer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py
index fe06292e9699..9715865c65a1 100644
--- a/tensorrt_llm/tokenizer/tokenizer.py
+++ b/tensorrt_llm/tokenizer/tokenizer.py
@@ -42,7 +42,8 @@
 TOKENIZER_ALIASES = {
     "deepseek_v32":
     "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer",
-    "deepseek_v4": "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer",
+    "deepseek_v4":
+    "tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer",
     "mistral_common":
     "tensorrt_llm._torch.models.checkpoints.mistral.tokenizer.MistralTokenizer",
 }

From 483eaf8e26ad3437c3bf112d5663aacfb67a1196 Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Mon, 6 Jul 2026 04:09:35 -0700
Subject: [PATCH 17/19] rebase issue

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tests/unittest/_torch/modeling/test_modeling_mistral.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unittest/_torch/modeling/test_modeling_mistral.py b/tests/unittest/_torch/modeling/test_modeling_mistral.py
index 259b53d4eb44..6cb29057a8bd 100644
--- a/tests/unittest/_torch/modeling/test_modeling_mistral.py
+++ b/tests/unittest/_torch/modeling/test_modeling_mistral.py
@@ -19,7 +19,7 @@
 from tensorrt_llm._torch import model_config as model_config_lib
 from tensorrt_llm._torch.attention_backend import utils as attention_utils
 from tensorrt_llm._torch.models import modeling_mistral
-from tensorrt_llm._torch.models.modeling_mistral import Mistral3InputProcessor
+from tensorrt_llm._torch.models.modeling_mistral import MistralHFInputProcessor
 from tensorrt_llm._torch.models.modeling_utils import MetaInitMode
 from tensorrt_llm._torch.pyexecutor import resource_manager
 from tensorrt_llm.bindings import executor as executor_lib
@@ -637,7 +637,7 @@ def test_mistral_attention_swa_layer_types():
 # Deterministic dummy-input sizing (Mistral3 / Pixtral input processor).
 #
 # CPU-only unit tests for the encoder-profiling dummy contract: reach into
-# Mistral3InputProcessor directly (no model load) and stub the geometry the
+# MistralHFInputProcessor directly (no model load) and stub the geometry the
 # dummy math reads. The ViT token unit is the pre-merge patch count
 # ``(h//patch)*(w//patch)`` -- deliberately *not* the hashing path's LLM-side
 # Pixtral count with framing tokens.
@@ -649,7 +649,7 @@ def _make_dummy_processor(*, patch_size=14, spatial_merge_size=2, image_size=154
     ``_processor`` forces ``_vision_geometry`` to fall back to ``vision_config``
     (the HF ``mistral3`` path).
     """
-    instance = Mistral3InputProcessor.__new__(Mistral3InputProcessor)
+    instance = MistralHFInputProcessor.__new__(MistralHFInputProcessor)
     instance._config = SimpleNamespace(
         vision_config=SimpleNamespace(
             patch_size=patch_size, image_size=image_size, num_channels=num_channels

From 017bbb371a05f6ab68c0824a9f9eb3182f9de3af Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Mon, 6 Jul 2026 08:11:56 -0700
Subject: [PATCH 18/19] argparse fix

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 tensorrt_llm/serve/openai_server.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py
index 028145341c84..7b0e04d52b6e 100644
--- a/tensorrt_llm/serve/openai_server.py
+++ b/tensorrt_llm/serve/openai_server.py
@@ -492,7 +492,8 @@ def _init_llm(self, chat_template: Optional[str] = None):
                 self.tokenizer.tokenizer, "name_or_path", None) or getattr(
                     self.tokenizer, "name_or_path", None)
         trust_remote_code = self.generator.args.trust_remote_code
-        checkpoint_format = self.generator.args.checkpoint_format
+        checkpoint_format = getattr(self.generator.args, "checkpoint_format",
+                                    None)
         if checkpoint_format in ("mistral", "mistral_large_3"):
             # Do not load HF processor for mistral native checkpoints
             # even if it is available

From 98508b448aed817489cc6660a76ad104db20ae4b Mon Sep 17 00:00:00 2001
From: Olya Kozlova <okozlova@nvidia.com>
Date: Tue, 14 Jul 2026 08:56:20 -0700
Subject: [PATCH 19/19] eagle fix

Signed-off-by: Olya Kozlova <okozlova@nvidia.com>
---
 .../_torch/models/modeling_mistral.py         | 14 +++++---
 .../_torch/models/modeling_speculative.py     | 34 ++++++++++++-------
 2 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py
index 16d963095fbb..35284a91cad7 100644
--- a/tensorrt_llm/_torch/models/modeling_mistral.py
+++ b/tensorrt_llm/_torch/models/modeling_mistral.py
@@ -647,13 +647,17 @@ def call_with_text_prompt(
         self, inputs: TextPrompt, sampling_params: SamplingParams
     ) -> Tuple[List[int], ExtraProcessedInputs | None]:
         images = inputs.get("multi_modal_data", {}).get("image")
-        # MistralCommonImageProcessor builds the full conversation and applies
-        # the mistral-common chat template internally.
+        if not images:
+            # Text-only: tokenize directly without wrapping in a chat template.
+            # The chat template is either already applied by the caller (serve
+            # path) or intentionally absent (e.g. raw few-shot eval like MMLU).
+            input_ids = self.tokenizer.transformers_tokenizer.encode(
+                inputs["prompt"])
+            return input_ids, None
+        # Multimodal: MistralCommonImageProcessor builds the full conversation
+        # and applies the mistral-common chat template with image tokens.
         processed = self.processor(text=inputs["prompt"], images=images)
         input_ids = processed.pop("input_ids").tolist()[0]
-        pixel_values = processed.get("pixel_values")
-        if pixel_values is None:
-            return input_ids, None
         processed.pop("attention_mask", None)
         processed["image_sizes"] = processed["image_sizes"].tolist()
         return input_ids, {"multimodal_data": {"image": {**processed}}}
diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py
index a52534de4f50..4258b2ed6d1a 100755
--- a/tensorrt_llm/_torch/models/modeling_speculative.py
+++ b/tensorrt_llm/_torch/models/modeling_speculative.py
@@ -664,6 +664,7 @@ def forward(
         inputs_embeds: torch.FloatTensor | None = None,
         spec_metadata: SpecMetadata | None = None,
         hidden_states: torch.Tensor | None = None,
+        all_rank_num_tokens: Optional[List[int]] = None,
     ) -> torch.Tensor:
         if (input_ids is None) ^ (inputs_embeds is not None):
             raise ValueError(
@@ -676,18 +677,27 @@ def forward(
 
         assert hidden_states is not None
 
-        # NOTE: If hidden states from the target model have to be concatenated,
-        # we expect that to happen outside the model definition. This helps us
-        # avoid data-dependent control flow and gives us better CUDA graph
-        # coverage.
-        residual = None
-        hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1)
-        hidden_states = self.fc(hidden_states)
-        hidden_states, residual = self.layers[0](position_ids=position_ids,
-                                                 hidden_states=hidden_states,
-                                                 attn_metadata=attn_metadata,
-                                                 residual=None,
-                                                 spec_metadata=spec_metadata)
+        previous_all_rank_num_tokens = attn_metadata.all_rank_num_tokens
+        if all_rank_num_tokens is not None:
+            attn_metadata.all_rank_num_tokens = all_rank_num_tokens
+
+        try:
+            # NOTE: If hidden states from the target model have to be concatenated,
+            # we expect that to happen outside the model definition. This helps us
+            # avoid data-dependent control flow and gives us better CUDA graph
+            # coverage.
+            residual = None
+            hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1)
+            hidden_states = self.fc(hidden_states)
+            hidden_states, residual = self.layers[0](
+                position_ids=position_ids,
+                hidden_states=hidden_states,
+                attn_metadata=attn_metadata,
+                residual=None,
+                spec_metadata=spec_metadata)
+        finally:
+            if all_rank_num_tokens is not None:
+                attn_metadata.all_rank_num_tokens = previous_all_rank_num_tokens
 
         return hidden_states, hidden_states