[https://nvbugs/6157892][fix] Mistral format refactor

examples/llm-api/quickstart_advanced.py

@@ -194,6 +194,11 @@ def add_llm_args(parser):
     parser.add_argument('--apply_chat_template',
                         default=False,
                         action='store_true')
+    parser.add_argument('--custom_tokenizer',
+                        type=str,
+                        default=None,
+                        help='Override the tokenizer. Accepts a built-in alias '
+                        " or a fully-qualified class import path.")
 
     # Sampling
     parser.add_argument("--max_tokens", type=int, default=64)
@@ -390,6 +395,7 @@ def setup_llm(args, **kwargs):
         gather_generation_logits=args.return_generation_logits,
         max_beam_width=args.max_beam_width,
         orchestrator_type=args.orchestrator_type,
+        custom_tokenizer=args.custom_tokenizer,
         **kwargs)
 
     use_beam_search = args.max_beam_width > 1

tensorrt_llm/_torch/attention_backend/interface.py

@@ -674,24 +674,25 @@ def from_config(config) -> "RopeParams":
         rope_params = RopeParams()
 
         hf_rope_parameters = getattr(config, 'rope_parameters', None)
+        normalized_rope_parameters = hf_rope_parameters
         if hf_rope_parameters is not None:
             if set(hf_rope_parameters.keys()).issubset(
                     ALLOWED_ATTENTION_LAYER_TYPES):
                 # Per-layer-type RoPE config (e.g. Gemma3 in transformers 5.x).
                 # Pick "full_attention" as the default; callers override theta
                 # for sliding-window layers independently.
                 if "full_attention" in hf_rope_parameters:
-                    flat = hf_rope_parameters["full_attention"]
+                    normalized_rope_parameters = hf_rope_parameters[
+                        "full_attention"]
                 else:
                     fallback_key = next(iter(hf_rope_parameters))
                     logger.warning(
                         f"Per-layer-type rope_parameters has no 'full_attention' entry; "
                         f"falling back to '{fallback_key}'. Available layer types: "
                         f"{list(hf_rope_parameters.keys())}.")
-                    flat = hf_rope_parameters[fallback_key]
-                config.update(flat)
-            else:
-                config.update(hf_rope_parameters)
+                    normalized_rope_parameters = hf_rope_parameters[
+                        fallback_key]
+            config.update(normalized_rope_parameters)
 
         # get rotary parameters.
         hidden_size = config.hidden_size
@@ -700,6 +701,8 @@ def from_config(config) -> "RopeParams":
         if not isinstance(head_dim, int):
             head_dim = hidden_size // num_attention_heads
         rope_scaling = getattr(config, 'rope_scaling', None)
+        if rope_scaling is None and normalized_rope_parameters is not None:
+            rope_scaling = normalized_rope_parameters
         rope_params.max_positions = config.max_position_embeddings
         rope_params.theta = get_hf_rope_theta(config, 10000.0)
         rope_percentage = (getattr(config, 'rotary_pct', None)

tensorrt_llm/_torch/model_config.py

@@ -494,12 +494,15 @@ def load_hf_quant_config(hf_quant_config, moe_backend, checkpoint_dir=None):
         # Read exclude_modules from HF config if present (HF format module names)
         hf_exclude_modules = hf_quant_config.get('modules_to_not_convert', None)
 
-        # DeepSeek V3 FP8 ckpt
-        if hf_quant_config.get("quant_method") == "fp8" and hf_quant_config.get(
-                "weight_block_size", []):
+        # FP8 ckpt: DeepSeek V3 style (weight_block_size) or
+        # per-tensor static activation scale style (activation_scheme="static",
+        # e.g. Ministral / Pixtral).
+        if hf_quant_config.get("quant_method") == "fp8" and (
+                hf_quant_config.get("weight_block_size")
+                or hf_quant_config.get("activation_scheme") == "static"):
             quant_config.quant_algo = QuantAlgo.FP8_BLOCK_SCALES
 
-            block_size = hf_quant_config.get("weight_block_size", [])
+            block_size = hf_quant_config.get("weight_block_size", [128, 128])
             assert tuple(block_size) == (
                 128, 128), "FP8_BLOCK_SCALES only supports block_size=(128,128)"
             quant_config.group_size = block_size[0]

tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py

@@ -328,7 +328,7 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig:
         quant_config = QuantConfig()
         layer_quant_config = None
 
-        hf_quant_config = pretrained_config.quantization_config
+        hf_quant_config = getattr(pretrained_config, "quantization_config", {}) or {}
         if hf_quant_config.get("quant_method") == "compressed-tensors":
             if "NVFP4" in hf_quant_config.get("config_groups"):
                 quant_config.quant_algo = QuantAlgo.NVFP4
@@ -390,7 +390,14 @@ def load(self, checkpoint_dir: str, **kwargs) -> ModelConfig:
         from tensorrt_llm._torch.models.modeling_mistral_large3 import Mistral3Gate
 
         model_config.pretrained_config.gate_cls = Mistral3Gate
-        model_config.pretrained_config.input_processor_type = "mistral_large_3"
-        model_config.pretrained_config.model_type = "mistral_large_3"
+        # Native (mistral-format) checkpoints are served through the
+        # mistral-common tokenizer/processor, which applies its own chat
+        # template. Tag the config with a dedicated serving model_type so the
+        # serving layer resolves the PASSTHROUGH placeholder/chat-template
+        # metadata (registered under "mistral_common" in modeling_mistral.py)
+        # via the normal config path - i.e. resolve_top_level_model_type() -
+        # instead of having to inspect the live input processor.
+        model_config.pretrained_config.input_processor_type = "mistral_common"
+        model_config.pretrained_config.model_type = "mistral_common"
         model_config._frozen = True
         return model_config

tensorrt_llm/_torch/models/checkpoints/mistral/tokenizer.py

@@ -301,4 +301,4 @@ def hf_decode_incrementally(
     def apply_chat_template(
         self, conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]], *args, **kwargs
     ) -> Union[str, list[int], list[str], list[list[int]]]:
-        raise NotImplementedError
+        return self.transformers_tokenizer.apply_chat_template(conversation, *args, **kwargs)

tensorrt_llm/_torch/models/checkpoints/mistral/weight_mapper.py

@@ -35,6 +35,9 @@ def __init__(self):
             "tok_embeddings": "model.embed_tokens",
             "output": "lm_head",
             "norm": "model.norm",
+            # For text-only models: preprocess_weights adds "language_model." prefix
+            "language_model.tok_embeddings": "model.embed_tokens",
+            "language_model.output": "lm_head",
             # For Eagle3
             "language_model.eagle_linear": "model.fc",
             "language_model.layers": "layers",