From 007b790e375b653e1247225e8eeab9933f98cc5f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 13 Aug 2026 00:52:37 +0000 Subject: [PATCH 1/5] fixture(MODEL-NEMOTRON-H W3): the released Nemotron-3.5-Lightning config + 18487-tensor index, headers only (#517) FOLLOWING_AGENTS_PROTOCOL W3 of .agents/specs/nemotron-h-model.md needs a config the parse gate can read and an index the enumeration gate can be checked against, WITHOUT the 20.1 GiB checkpoint. Both are projected from the live NAS copy of nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 at the pinned revision 29f2d1746d8f41e316523194b19018707749b1b1, reading JSON and safetensors HEADERS only -- no tensor bytes. config.json is the released file MINUS exactly two keys, quantization_config.{config_groups,quantized_layers}: 865 KB of the 1.34 MB file, 5981 entries each, and the surface W1 (#517 W1) owns. `ignore` (72 entries, 2.4 KB) is DELIBERATELY KEPT -- its `mtp*` wildcard is what makes the MTP tower unquantized, which the enumeration has to know and would otherwise have to guess. index.json projects all 18487 tensors to one entry per (layer index, tensor suffix) family with dtype and shape; only the ROUTED-EXPERT index is collapsed to {E}, with `count` recording how many experts share the entry. Layer indices are PRESERVED on purpose, so a wrong layer-type mapping cannot survive the fixture. 707 families, sum(count) == 18487, dtypes BF16 485 / F32 6086 / F8_E4M3 5981 / U8 5935. Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [Claude Code] --- .../nemotron_h_35_lightning/config.json | 208 + .../nemotron_h_35_lightning/index.json | 5014 +++++++++++++++++ 2 files changed, 5222 insertions(+) create mode 100644 tests/vllm/models/fixtures/nemotron_h_35_lightning/config.json create mode 100644 tests/vllm/models/fixtures/nemotron_h_35_lightning/index.json diff --git a/tests/vllm/models/fixtures/nemotron_h_35_lightning/config.json b/tests/vllm/models/fixtures/nemotron_h_35_lightning/config.json new file mode 100644 index 000000000..7bd70ad88 --- /dev/null +++ b/tests/vllm/models/fixtures/nemotron_h_35_lightning/config.json @@ -0,0 +1,208 @@ +{ + "architectures": [ + "NemotronHForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "chunk_size": 128, + "conv_kernel": 4, + "dtype": "bfloat16", + "eos_token_id": 2, + "expand": 2, + "head_dim": 128, + "hidden_dropout": 0.0, + "hidden_size": 2688, + "initializer_range": 0.02, + "intermediate_size": 1856, + "layer_norm_epsilon": 1e-05, + "layers_block_type": [ + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "moe" + ], + "mamba_head_dim": 64, + "mamba_hidden_act": "silu", + "mamba_num_heads": 64, + "mamba_proj_bias": false, + "mamba_ssm_cache_dtype": "float32", + "max_position_embeddings": 1048576, + "mlp_bias": false, + "mlp_hidden_act": "relu2", + "model_type": "nemotron_h", + "moe_intermediate_size": 1856, + "moe_latent_size": null, + "moe_shared_expert_intermediate_size": 3712, + "moe_shared_expert_overlap": true, + "mtp_layers_block_type": [ + "attention", + "moe" + ], + "n_group": 1, + "n_groups": 8, + "n_routed_experts": 128, + "n_shared_experts": 1, + "norm_eps": 1e-05, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_experts_per_tok": 6, + "num_hidden_layers": 52, + "num_key_value_heads": 2, + "num_logits_to_keep": 1, + "num_nextn_predict_layers": 1, + "pad_token_id": 0, + "partial_rotary_factor": 1.0, + "quantization_config": { + "ignore": [ + "backbone.embeddings", + "backbone.layers.0.mixer.conv1d", + "backbone.layers.1.mixer.gate", + "backbone.layers.10.mixer.gate", + "backbone.layers.11.mixer.conv1d", + "backbone.layers.12.mixer.k_proj", + "backbone.layers.12.mixer.o_proj", + "backbone.layers.12.mixer.q_proj", + "backbone.layers.12.mixer.v_proj", + "backbone.layers.13.mixer.gate", + "backbone.layers.14.mixer.conv1d", + "backbone.layers.15.mixer.gate", + "backbone.layers.16.mixer.conv1d", + "backbone.layers.17.mixer.gate", + "backbone.layers.18.mixer.conv1d", + "backbone.layers.19.mixer.k_proj", + "backbone.layers.19.mixer.o_proj", + "backbone.layers.19.mixer.q_proj", + "backbone.layers.19.mixer.v_proj", + "backbone.layers.2.mixer.conv1d", + "backbone.layers.20.mixer.gate", + "backbone.layers.21.mixer.conv1d", + "backbone.layers.22.mixer.gate", + "backbone.layers.23.mixer.conv1d", + "backbone.layers.24.mixer.gate", + "backbone.layers.25.mixer.conv1d", + "backbone.layers.26.mixer.k_proj", + "backbone.layers.26.mixer.o_proj", + "backbone.layers.26.mixer.q_proj", + "backbone.layers.26.mixer.v_proj", + "backbone.layers.27.mixer.gate", + "backbone.layers.28.mixer.conv1d", + "backbone.layers.29.mixer.gate", + "backbone.layers.3.mixer.gate", + "backbone.layers.30.mixer.conv1d", + "backbone.layers.31.mixer.gate", + "backbone.layers.32.mixer.conv1d", + "backbone.layers.33.mixer.k_proj", + "backbone.layers.33.mixer.o_proj", + "backbone.layers.33.mixer.q_proj", + "backbone.layers.33.mixer.v_proj", + "backbone.layers.34.mixer.gate", + "backbone.layers.35.mixer.conv1d", + "backbone.layers.36.mixer.gate", + "backbone.layers.37.mixer.conv1d", + "backbone.layers.38.mixer.gate", + "backbone.layers.39.mixer.conv1d", + "backbone.layers.4.mixer.conv1d", + "backbone.layers.40.mixer.gate", + "backbone.layers.41.mixer.conv1d", + "backbone.layers.42.mixer.k_proj", + "backbone.layers.42.mixer.o_proj", + "backbone.layers.42.mixer.q_proj", + "backbone.layers.42.mixer.v_proj", + "backbone.layers.43.mixer.gate", + "backbone.layers.44.mixer.conv1d", + "backbone.layers.45.mixer.gate", + "backbone.layers.46.mixer.conv1d", + "backbone.layers.47.mixer.gate", + "backbone.layers.48.mixer.conv1d", + "backbone.layers.49.mixer.gate", + "backbone.layers.5.mixer.k_proj", + "backbone.layers.5.mixer.o_proj", + "backbone.layers.5.mixer.q_proj", + "backbone.layers.5.mixer.v_proj", + "backbone.layers.50.mixer.conv1d", + "backbone.layers.51.mixer.gate", + "backbone.layers.6.mixer.gate", + "backbone.layers.7.mixer.conv1d", + "backbone.layers.8.mixer.gate", + "backbone.layers.9.mixer.conv1d", + "mtp*" + ], + "kv_cache_scheme": { + "dynamic": false, + "num_bits": 8, + "type": "float" + }, + "producer": { + "name": "modelopt", + "version": "0.44.0rc5" + }, + "quant_algo": "MIXED_PRECISION", + "quant_method": "modelopt" + }, + "rescale_prenorm_residual": true, + "residual_in_fp32": false, + "rope_theta": 10000, + "routed_scaling_factor": 2.5, + "sliding_window": null, + "ssm_state_size": 128, + "tie_word_embeddings": false, + "time_step_floor": 0.0001, + "time_step_max": 0.1, + "time_step_min": 0.001, + "topk_group": 1, + "transformers_version": "5.8.1", + "use_bias": false, + "use_cache": true, + "use_conv_bias": true, + "use_mamba_kernels": true, + "vocab_size": 131072 +} diff --git a/tests/vllm/models/fixtures/nemotron_h_35_lightning/index.json b/tests/vllm/models/fixtures/nemotron_h_35_lightning/index.json new file mode 100644 index 000000000..b2402867d --- /dev/null +++ b/tests/vllm/models/fixtures/nemotron_h_35_lightning/index.json @@ -0,0 +1,5014 @@ +{ + "_provenance": "Projected from nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 @ 29f2d1746d8f41e316523194b19018707749b1b1 (model.safetensors.index.json + the 52 safetensors headers; HEADERS ONLY -- no tensor bytes were read). One entry per (layer index, tensor suffix) family; only the ROUTED-EXPERT index is collapsed to {E} and `count` is how many experts share it. Layer indices are preserved.", + "revision": "29f2d1746d8f41e316523194b19018707749b1b1", + "tensors": { + "backbone.embeddings.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 131072, + 2688 + ] + }, + "backbone.layers.0.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.0.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.0.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.0.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.0.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.0.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.0.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.0.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.0.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.0.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.0.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.0.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.0.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.1.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.1.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.1.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.1.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.1.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.1.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.1.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.1.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.1.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.1.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.1.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.1.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.1.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.1.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.1.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.10.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.10.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.10.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.10.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.10.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.10.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.10.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.10.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.10.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.10.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.10.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.10.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.10.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.10.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.10.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.11.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.11.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.11.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.11.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.11.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.11.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.11.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.11.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.11.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.11.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.11.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.11.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.11.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.12.mixer.k_proj.k_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.12.mixer.k_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.12.mixer.o_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.12.mixer.q_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096, + 2688 + ] + }, + "backbone.layers.12.mixer.v_proj.v_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.12.mixer.v_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.12.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.13.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.13.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.13.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.13.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.13.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.13.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.13.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.13.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.13.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.13.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.13.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.13.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.13.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.13.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.13.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.14.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.14.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.14.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.14.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.14.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.14.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.14.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.14.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.14.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.14.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.14.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.14.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.14.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.15.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.15.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.15.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.15.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.15.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.15.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.15.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.15.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.15.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.15.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.15.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.15.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.15.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.15.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.15.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.16.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.16.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.16.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.16.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.16.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.16.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.16.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.16.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.16.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.16.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.16.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.16.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.16.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.17.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.17.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.17.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.17.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.17.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.17.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.17.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.17.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.17.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.17.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.17.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.17.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.17.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.17.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.17.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.18.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.18.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.18.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.18.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.18.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.18.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.18.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.18.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.18.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.18.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.18.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.18.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.18.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.19.mixer.k_proj.k_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.19.mixer.k_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.19.mixer.o_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.19.mixer.q_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096, + 2688 + ] + }, + "backbone.layers.19.mixer.v_proj.v_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.19.mixer.v_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.19.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.2.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.2.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.2.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.2.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.2.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.2.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.2.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.2.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.2.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.2.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.2.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.2.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.2.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.20.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.20.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.20.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.20.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.20.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.20.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.20.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.20.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.20.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.20.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.20.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.20.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.20.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.20.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.20.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.21.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.21.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.21.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.21.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.21.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.21.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.21.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.21.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.21.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.21.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.21.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.21.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.21.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.22.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.22.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.22.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.22.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.22.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.22.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.22.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.22.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.22.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.22.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.22.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.22.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.22.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.22.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.22.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.23.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.23.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.23.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.23.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.23.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.23.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.23.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.23.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.23.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.23.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.23.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.23.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.23.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.24.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.24.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.24.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.24.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.24.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.24.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.24.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.24.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.24.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.24.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.24.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.24.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.24.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.24.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.24.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.25.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.25.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.25.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.25.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.25.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.25.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.25.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.25.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.25.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.25.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.25.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.25.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.25.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.26.mixer.k_proj.k_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.26.mixer.k_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.26.mixer.o_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.26.mixer.q_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096, + 2688 + ] + }, + "backbone.layers.26.mixer.v_proj.v_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.26.mixer.v_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.26.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.27.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.27.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.27.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.27.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.27.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.27.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.27.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.27.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.27.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.27.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.27.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.27.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.27.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.27.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.27.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.28.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.28.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.28.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.28.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.28.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.28.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.28.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.28.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.28.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.28.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.28.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.28.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.28.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.29.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.29.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.29.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.29.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.29.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.29.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.29.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.29.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.29.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.29.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.29.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.29.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.29.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.29.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.29.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.3.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.3.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.3.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.3.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.3.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.3.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.3.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.3.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.3.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.3.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.3.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.3.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.3.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.3.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.3.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.30.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.30.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.30.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.30.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.30.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.30.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.30.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.30.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.30.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.30.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.30.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.30.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.30.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.31.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.31.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.31.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.31.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.31.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.31.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.31.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.31.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.31.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.31.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.31.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.31.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.31.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.31.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.31.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.32.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.32.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.32.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.32.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.32.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.32.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.32.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.32.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.32.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.32.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.32.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.32.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.32.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.33.mixer.k_proj.k_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.33.mixer.k_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.33.mixer.o_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.33.mixer.q_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096, + 2688 + ] + }, + "backbone.layers.33.mixer.v_proj.v_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.33.mixer.v_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.33.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.34.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.34.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.34.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.34.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.34.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.34.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.34.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.34.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.34.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.34.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.34.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.34.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.34.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.34.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.34.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.35.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.35.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.35.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.35.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.35.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.35.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.35.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.35.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.35.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.35.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.35.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.35.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.35.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.36.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.36.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.36.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.36.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.36.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.36.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.36.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.36.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.36.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.36.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.36.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.36.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.36.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.36.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.36.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.37.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.37.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.37.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.37.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.37.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.37.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.37.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.37.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.37.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.37.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.37.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.37.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.37.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.38.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.38.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.38.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.38.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.38.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.38.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.38.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.38.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.38.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.38.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.38.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.38.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.38.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.38.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.38.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.39.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.39.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.39.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.39.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.39.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.39.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.39.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.39.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.39.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.39.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.39.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.39.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.39.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.4.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.4.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.4.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.4.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.4.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.4.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.4.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.4.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.4.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.4.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.4.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.4.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.4.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.40.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.40.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.40.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.40.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.40.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.40.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.40.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.40.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.40.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.40.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.40.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.40.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.40.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.40.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.40.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.41.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.41.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.41.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.41.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.41.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.41.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.41.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.41.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.41.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.41.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.41.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.41.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.41.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.42.mixer.k_proj.k_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.42.mixer.k_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.42.mixer.o_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.42.mixer.q_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096, + 2688 + ] + }, + "backbone.layers.42.mixer.v_proj.v_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.42.mixer.v_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.42.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.43.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.43.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.43.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.43.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.43.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.43.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.43.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.43.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.43.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.43.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.43.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.43.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.43.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.43.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.43.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.44.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.44.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.44.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.44.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.44.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.44.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.44.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.44.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.44.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.44.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.44.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.44.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.44.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.45.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.45.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.45.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.45.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.45.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.45.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.45.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.45.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.45.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.45.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.45.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.45.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.45.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.45.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.45.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.46.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.46.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.46.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.46.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.46.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.46.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.46.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.46.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.46.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.46.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.46.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.46.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.46.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.47.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.47.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.47.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.47.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.47.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.47.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.47.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.47.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.47.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.47.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.47.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.47.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.47.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.47.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.47.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.48.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.48.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.48.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.48.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.48.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.48.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.48.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.48.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.48.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.48.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.48.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.48.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.48.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.49.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.49.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.49.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.49.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.49.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.49.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.49.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.49.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.49.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.49.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.49.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.49.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.49.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.49.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.49.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.5.mixer.k_proj.k_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.5.mixer.k_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.5.mixer.o_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.5.mixer.q_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096, + 2688 + ] + }, + "backbone.layers.5.mixer.v_proj.v_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.5.mixer.v_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "backbone.layers.5.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.50.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.50.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.50.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.50.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.50.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.50.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.50.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.50.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.50.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.50.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.50.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.50.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.50.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.51.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.51.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.51.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.51.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.51.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.51.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.51.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.51.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.51.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.51.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.51.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.51.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.51.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.51.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.51.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.6.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.6.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.6.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.6.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.6.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.6.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.6.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.6.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.6.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.6.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.6.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.6.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.6.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.6.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.6.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.7.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.7.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.7.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.7.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.7.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.7.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.7.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.7.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.7.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.7.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.7.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.7.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.7.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.8.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 2688, + 928 + ] + }, + "backbone.layers.8.mixer.experts.{E}.down_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 116 + ] + }, + "backbone.layers.8.mixer.experts.{E}.down_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.8.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "U8", + "shape": [ + 1856, + 1344 + ] + }, + "backbone.layers.8.mixer.experts.{E}.up_proj.weight_scale": { + "count": 128, + "dtype": "F8_E4M3", + "shape": [ + 1856, + 168 + ] + }, + "backbone.layers.8.mixer.experts.{E}.up_proj.weight_scale_2": { + "count": 128, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.8.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "backbone.layers.8.mixer.gate.weight": { + "count": 1, + "dtype": "F32", + "shape": [ + 128, + 2688 + ] + }, + "backbone.layers.8.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 2688, + 1856 + ] + }, + "backbone.layers.8.mixer.shared_experts.down_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 232 + ] + }, + "backbone.layers.8.mixer.shared_experts.down_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.8.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 3712, + 1344 + ] + }, + "backbone.layers.8.mixer.shared_experts.up_proj.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 3712, + 168 + ] + }, + "backbone.layers.8.mixer.shared_experts.up_proj.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.8.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.layers.9.mixer.A_log": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.9.mixer.D": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.9.mixer.conv1d.bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144 + ] + }, + "backbone.layers.9.mixer.conv1d.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 6144, + 1, + 4 + ] + }, + "backbone.layers.9.mixer.dt_bias": { + "count": 1, + "dtype": "BF16", + "shape": [ + 64 + ] + }, + "backbone.layers.9.mixer.in_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.9.mixer.in_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 10304, + 2688 + ] + }, + "backbone.layers.9.mixer.in_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.9.mixer.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096 + ] + }, + "backbone.layers.9.mixer.out_proj.input_scale": { + "count": 1, + "dtype": "F32", + "shape": [ + 1 + ] + }, + "backbone.layers.9.mixer.out_proj.weight": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 2688, + 4096 + ] + }, + "backbone.layers.9.mixer.out_proj.weight_scale": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "backbone.layers.9.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "backbone.norm_f.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "lm_head.weight": { + "count": 1, + "dtype": "U8", + "shape": [ + 131072, + 1344 + ] + }, + "lm_head.weight_scale": { + "count": 1, + "dtype": "F8_E4M3", + "shape": [ + 131072, + 168 + ] + }, + "lm_head.weight_scale_2": { + "count": 1, + "dtype": "F32", + "shape": [] + }, + "mtp.layers.0.eh_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 5376 + ] + }, + "mtp.layers.0.enorm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "mtp.layers.0.hnorm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "mtp.layers.0.mixer.k_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "mtp.layers.0.mixer.o_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 4096 + ] + }, + "mtp.layers.0.mixer.q_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 4096, + 2688 + ] + }, + "mtp.layers.0.mixer.v_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 256, + 2688 + ] + }, + "mtp.layers.0.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "mtp.layers.1.final_layernorm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + }, + "mtp.layers.1.mixer.experts.{E}.down_proj.weight": { + "count": 128, + "dtype": "BF16", + "shape": [ + 2688, + 1856 + ] + }, + "mtp.layers.1.mixer.experts.{E}.up_proj.weight": { + "count": 128, + "dtype": "BF16", + "shape": [ + 1856, + 2688 + ] + }, + "mtp.layers.1.mixer.gate.e_score_correction_bias": { + "count": 1, + "dtype": "F32", + "shape": [ + 128 + ] + }, + "mtp.layers.1.mixer.gate.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 128, + 2688 + ] + }, + "mtp.layers.1.mixer.shared_experts.down_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688, + 3712 + ] + }, + "mtp.layers.1.mixer.shared_experts.up_proj.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 3712, + 2688 + ] + }, + "mtp.layers.1.norm.weight": { + "count": 1, + "dtype": "BF16", + "shape": [ + 2688 + ] + } + }, + "total_tensors": 18487 +} From 3295d0c1a230b235a7c01758eaf1adfc10e1ffa1 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 13 Aug 2026 00:53:21 +0000 Subject: [PATCH 2/5] feat(MODEL-NEMOTRON-H W3): NemotronH is KNOWN -- registered, parsed, enumerated, KV-shaped (#517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FOLLOWING_AGENTS_PROTOCOL W3 of .agents/specs/nemotron-h-model.md. It makes the architecture known; it runs nothing. The forward REFUSES BY NAME (W4, itself blocked on the Mamba2 SSD CUDA arm #496 W2) and the GGUF arm REFUSES BY NAME (W7, owed per §5b). RED FIRST. The gate was written against declared-but-empty implementations and run before any of it worked, Release -Werror, doctest 2.5.2: [doctest] test cases: 10 | 2 passed | 8 failed | 0 skipped [doctest] assertions: 1258 | 1186 passed | 72 failed | [doctest] Status: FAILURE! failing in exactly the intended shape -- CHECK( 0 == 52 ) for the schedule depth, CHECK( {} == {5, 12, 19, 26, 33, 42} ) for the attention indices, REQUIRE( 0 == 2 ) for the KV groups, "did NOT throw at all!" for the GGUF refusal, 18487 unclaimed tensors for the enumeration. ENUMERATION, the hard number: 18487 of 18487 released tensors CLAIMED by a named consumer. 0 unaccounted, 0 invented, 0 refused. Gated both directions against the committed headers-only projection of the released index, plus a live case that re-verifies that projection against the checkpoint itself when VT_NEMOTRON35_SNAPSHOT names it. Four things the plan had wrong or unstated, settled against source and disk: 1. layers_block_type is the source of truth -- but vLLM VENDORS a NemotronHConfig with the OPPOSITE polarity (hybrid_override_pattern as the ctor arg, layers_block_type derived). That class is imported for TYPE ANNOTATION only; the runtime object comes from transformers AutoConfig, where num_hidden_layers is a property over layers_block_type whose setter discards the checkpoint's value. Both spellings parse, the modern one wins. 2. moe_latent_size does NOT need three states: upstream's predicate is `getattr(config, "moe_latent_size", None) is not None`, so ABSENT and `null` are the same state. std::optional covers both; a real value refuses (§0). 3. The shared detail::ResolveMambaSsmCacheDType is the WRONG reader here and silently HALVES the recurrent state. It is keyed on `mamba_ssm_dtype` (Qwen3.5's spelling); NemotronH ships `mamba_ssm_cache_dtype`, so it returned the CONVOLUTION dtype. Caught by the KV gate as page_size_bytes() 1085440 vs an expected 2134016 -- the SSM state at bf16 instead of f32. Not a Qwen bug; two families, two config keys. Resolved locally, reason recorded at the call site. 4. Upstream's default conv layout is "SD" = (state_len, dim) while ours is (dim, state_len). Same bytes; the local convention is kept so the shared runner sees one orientation, and the divergence is commented rather than left for W4 to rediscover. KV: two groups carrying REAL per-layer names -- 6 attention layers on FullAttentionSpec(2 kv heads, head_size 128), 23 mamba layers on MambaSpec {{6144,3},{64,64,128}} / {bf16,f32}. conv_dim == 6144 is falsified straight off disk by conv1d.weight BF16 [6144,1,4]. The names are load-bearing: kv_cache_utils.cpp:979 and kv_cache_interface.cpp:151-158 both multiply a group's page by layer_names.size(), so a one-element tag under-counts by 23x and 6x. SCOPE: no per-module quant algorithm is resolved -- that is W1, not on main. W3 reads four coarse falsifiable keys (quant_method, quant_algo, kv_cache_scheme, the `mtp*` entry in `ignore`) and derives scale companions STRUCTURALLY; the 18487-tensor gate is what proves the derivation. A non-ModelOpt producer refuses by name. One piece of debt is recorded in the code: a dense `mlp` block's quantized companion layout is DERIVED, because no in-scope checkpoint ships one. nemotron_h.h lives under src/, not include/vllm/: W3 ships nothing on the public ABI and include/vllm/ is a USER_USAGE_PREFIXES surface (#515). RECORDS. docs/FEATURES.md gains an honest scaffold row (registering an architecture IS a feature-surface change, and check-supported-models.py binds the two), the stale "Inventoried but blocked" entry is removed so the page does not say two things, and the three registered counts move 37->38 / 31->32. docs/USAGE.md gains one row in "Architectures that resolve but refuse to run": check-doc-checkpoint classifies the root CMakeLists.txt as a USER_USAGE file, and the obligation is real rather than an artifact -- pointing the CLI at a NemotronH checkpoint now resolves and refuses by name instead of reporting an unsupported architecture. That is one row past the FEATURES.md-only scope I was given, on the operator's own reasoning applied to the surface the checker names. test_model_registry.cpp PINS the registry contents four ways; all four are reconciled, never weakened: the two size assertions 37->38, both byte-order arch sequences, kExampleConfigArchitectures, both pinned "Supported architectures:" refusal strings, and a new explicit NemotronH branch asserting is_hybrid TRUE / supports_multimodal FALSE instead of letting it fall into the pure-text-full-attention else-branch. test_model_loader_gguf.cpp's canonical arch string gains the entry too. GATES, Release -Werror CPU: test_nemotron_h_scaffold 10/10 cases, 38245/38245 assertions, SUCCESS with VT_NEMOTRON35_SNAPSHOT 10/10 cases, 39113/39113 assertions, SUCCESS Debug arm (asserts unmasked) identical full ctest 100% tests passed, 0 tests failed out of 401 (test_voxtral_e2e skipped, no asset) MUTATION-PROVEN (each applied alone, rebuilt, run, reverted, tree verified clean): moe -> attention layer mapping 4 cases / 9 assertions FAILURE attention index set shifted by +1 2 cases / 2 assertions FAILURE conv_dim without 2*n_groups*state_size 4 cases / 8 assertions FAILURE SSM dtype collapsed to the activation 1 case / 2 assertions FAILURE mamba dt_bias left unclaimed 1 case / 2 assertions FAILURE Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [Claude Code] --- .agents/specs/nemotron-h-model.md | 108 ++++ CMakeLists.txt | 5 + docs/FEATURES.md | 10 +- docs/USAGE.md | 1 + src/vllm/model_executor/models/nemotron_h.h | 282 +++++++++ .../models/nemotron_h_registry.cpp | 181 ++++++ .../models/nemotron_h_weights.cpp | 553 +++++++++++++++++ tests/CMakeLists.txt | 22 + tests/vllm/models/test_model_registry.cpp | 22 +- .../vllm/models/test_nemotron_h_scaffold.cpp | 582 ++++++++++++++++++ tests/vllm/test_model_loader_gguf.cpp | 1 + 11 files changed, 1758 insertions(+), 9 deletions(-) create mode 100644 src/vllm/model_executor/models/nemotron_h.h create mode 100644 src/vllm/model_executor/models/nemotron_h_registry.cpp create mode 100644 src/vllm/model_executor/models/nemotron_h_weights.cpp create mode 100644 tests/vllm/models/test_nemotron_h_scaffold.cpp diff --git a/.agents/specs/nemotron-h-model.md b/.agents/specs/nemotron-h-model.md index 4b13b56d4..51b12c7fe 100644 --- a/.agents/specs/nemotron-h-model.md +++ b/.agents/specs/nemotron-h-model.md @@ -221,6 +221,114 @@ Until the GGUF arm lands it is **refused by name** at load, naming the missing piece, never silently dequantized to a supported path — a silent fallback is exactly what a token gate cannot see. Tracked as W7. +## 5c. W3 result — registered, parsed, enumerated, KV-shaped (2026-08-13) + +W3 landed on `row/MODEL-NEMOTRON-H-W3B` (base `fafa16f0`). It makes the +architecture KNOWN; it runs nothing. The forward and GGUF arms REFUSE BY NAME. + +**Enumeration, the hard numbers.** `EnumerateNemotronHTensors` claims **18487 of +18487** released tensors; **0 unaccounted, 0 invented, 0 refused** — every tensor +has a named consumer, so no refusal was needed. Composition: 5 root (embeddings, +`norm_f`, the NVFP4 `lm_head` triple), 23 mamba layers x 13, 6 attention x 7, +23 MoE x 777 (2 router + 128x6 expert + 6 shared + 1 norm), and 270 MTP +(`mtp.layers.0` 8, `mtp.layers.1` 262). Both directions are gated against a +committed headers-only projection of the index, and a second case re-verifies +that projection against the LIVE checkpoint when `VT_NEMOTRON35_SNAPSHOT` names +it. + +**Four things the reconnaissance had wrong or unstated, settled against source +and disk:** + +1. **`layers_block_type` really is the source of truth, but for a subtler + reason than "the config says so".** vLLM VENDORS its own `NemotronHConfig` + (`transformers_utils/configs/nemotron_h.py:277-287`) in which the polarity is + REVERSED — `hybrid_override_pattern` is the ctor argument and + `layers_block_type` the derived property. That class is imported by + `nemotron_h.py:83` for TYPE ANNOTATION only; the object that reaches the + model comes from transformers `AutoConfig`, where `num_hidden_layers` is a + property over `layers_block_type` whose setter discards the checkpoint's + value (`configuration_nemotron_h.py:225-238`). §5a's live oracle run is what + settles it on the real checkpoint. Both spellings are accepted here, the + modern one winning. +2. **`moe_latent_size`: absent and `null` are the SAME state.** Upstream's + predicate is `getattr(config, "moe_latent_size", None) is not None` + (`nemotron_h.py:143`), so a missing key and an explicit `null` both mean "no + latent MoE". A three-state representation would have been inventing a + distinction upstream does not make. `std::optional` covers both; a real value + REFUSES (§0). +3. **The shared `detail::ResolveMambaSsmCacheDType` is the WRONG reader here, + and using it silently halves the recurrent state.** It reads + `HfConfig::mamba_ssm_dtype`, which `hf_config.cpp:439` parses from the key + **`mamba_ssm_dtype`** — Qwen3.5/3.6's spelling. NemotronH ships + **`mamba_ssm_cache_dtype`** (`configuration_nemotron_h.py:121`), so the + shared helper saw an empty string and returned the CONVOLUTION dtype. Caught + by the KV gate as `page_size_bytes() == 1085440` against an expected + `2134016` — the SSM state at bf16 instead of f32. This is not a Qwen bug; + the two families genuinely use different config keys. Resolved locally by + `NemotronHSsmCacheDType`, with the reason recorded at the call site. +4. **The conv-state layout discrepancy is real and deliberate.** Upstream's + default is `"SD"` = `(state_len, dim)` (`mamba_utils.py:27-48`, + `VLLM_SSM_CONV_STATE_LAYOUT` unset); our local convention across + `qwen3_5_common.cpp:85` and `kimi_linear_registry.cpp:156` is + `(dim, state_len)`. Same bytes, same page size; the local convention is kept + so the shared runner sees one orientation, and the divergence is commented + rather than left for W4. + +**KV topology.** Two groups carrying their REAL per-layer names — 6 +`backbone.layers.{5,12,19,26,33,42}.mixer` on a `FullAttentionSpec(2 kv heads, +head_size 128)`, and 23 mamba layers on a `MambaSpec` with shapes +`{{6144, 3}, {64, 64, 128}}` and dtypes `{bf16, f32}`. `conv_dim == 6144` is +falsified straight off disk by `mixer.conv1d.weight` BF16 `[6144, 1, 4]`, and +`in_proj` `[10304, 2688]` confirms `z + xBC + dt`. The names are load-bearing: +`kv_cache_utils.cpp:979` multiplies a mamba group's page by +`layer_names.size()`, and `kv_cache_interface.cpp:151-158` does the same for an +attention group, so a one-element tag would under-count both by 23x and 6x. + +**Scope boundary held.** No per-module quant algorithm is resolved — that is W1, +which is not on `main`. W3 reads four coarse, individually falsifiable keys +(`quant_method`, `quant_algo`, `kv_cache_scheme`, and the `mtp*` entry in +`ignore`) and derives the scale companions STRUCTURALLY; the enumeration gate is +what proves that derivation against all 18487 tensors. A non-ModelOpt producer +refuses by name. One piece of honest debt is recorded in the code: the quantized +companion layout of a dense `mlp` block is DERIVED from the shared linear +layout, because no in-scope released NemotronH checkpoint ships one. + +**Fixture.** `tests/vllm/models/fixtures/nemotron_h_35_lightning/` holds the +released `config.json` minus exactly `quantization_config.{config_groups, +quantized_layers}` (865 KB of 1.34 MB, the 5981-entry maps W1 owns) and a +707-family projection of the index. `ignore` is KEPT, unlike the original plan: +at 2.4 KB it is small, and its `mtp*` wildcard is what makes the MTP tower +unquantized — eliding it would have forced a guess about 270 tensors. + +**Mutation proof (IMP-MUTATE).** Each defect applied alone to the restored tree, +rebuilt, the gate run, then `git checkout` and `git status --porcelain` verified +empty. All five turn it RED: + +| Mutation | Result | +|---|---| +| `layers_block_type` `"moe"` mapped to `kAttention` | 4 cases / 9 assertions FAILURE | +| `LayerIndices` shifts the FIRST attention index by +1 | 2 cases / 2 assertions FAILURE | +| `conv_dim` drops the `2*n_groups*state_size` term | 4 cases / 8 assertions FAILURE | +| SSM cache dtype collapsed to the conv/activation dtype | 1 case / 2 assertions FAILURE | +| mamba `dt_bias` left UNCLAIMED (23 tensors) | 1 case / 2 assertions FAILURE | + +**Gate evidence.** Release `-Werror` CPU: `test_nemotron_h_scaffold` 10/10 cases, +38245/38245 assertions, `Status: SUCCESS!`; with `VT_NEMOTRON35_SNAPSHOT` set, +10/10 and 39113/39113. Debug arm (asserts unmasked): identical. Full `ctest`: +`100% tests passed, 0 tests failed out of 401` (`test_voxtral_e2e` skipped, no +asset). `test_model_registry` 24/24 and `test_model_loader_gguf` 3/3 after their +pinned 37-architecture ledgers were reconciled to 38. + +One defect was found this way rather than by inspection: the live re-verification +first died with `[json.exception.type_error.304] cannot use at() with null` from +inside a loop nowhere near its cause. `nlohmann::json::parse(x).items()` binds a +range to a TEMPORARY that is destroyed before the body runs. It reads as a clean +one-liner and it is undefined behaviour; the materialized form is what the +muse-glimmer precedent already used. + +**Not done here:** the forward (W4), the MTP head (W5), the e2e token gate (W6), +the GGUF arm (W7). The row stays `INVENTORIED`. + ## 6. Risks / decisions - **Non-gated MoE must not become a parallel path.** If diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f5b2e0dd..3168f138f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -787,6 +787,11 @@ add_library(vllm STATIC src/vllm/model_executor/models/kimi_linear_forward.cpp src/vllm/model_executor/models/kimi_linear.cpp src/vllm/model_executor/models/kimi_linear_device.cpp + # MODEL-NEMOTRON-H W3 (#517): the additive NemotronH structural bring-up — + # config descent, the on-disk name map, the registry entry and the het-KV + # topology. No forward: nemotron_h.cpp is W4. + src/vllm/model_executor/models/nemotron_h_registry.cpp + src/vllm/model_executor/models/nemotron_h_weights.cpp src/vllm/model_executor/models/glm4_registry.cpp src/vllm/model_executor/models/glm4_weights.cpp src/vllm/model_executor/models/glm4.cpp diff --git a/docs/FEATURES.md b/docs/FEATURES.md index c368654a2..0b29c3b1e 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -25,7 +25,7 @@ are our reading of their documented behavior, not measurements. | Embeddable behind a C ABI | ✅ | ☐ | ☐ | ✅ | | Weight formats | Safetensors + GGUF | Safetensors | Safetensors | GGUF | | Correctness gate | token-exact vs vLLM | reference | own | own | -| Architectures | 37 registered, 27 gated | 130+ | 100+ | 100+ | +| Architectures | 38 registered, 27 gated | 130+ | 100+ | 100+ | | Downloadable server binaries | ✅ v0.0.2: eight indexed archives with checksums, provenance, manifests, and SBOMs. Windows ZIP downloads do not exist; native CPU/Vulkan lanes await hosted runtime, dry-run, prerelease, and authenticated audit gates | ✅ wheels/containers | ✅ wheels/containers | ✅ host-specific binaries | | Native Windows builds | ◐ CPU/Vulkan: `/MT /W4 /WX`, central `NOMINMAX`, UTF-8, aligned allocation, runtime ISA dispatch. Local closure includes the float-domain DeepSeek probe; hosted compile/runtime/release pending | ✅ | ✅ | ✅ | @@ -87,7 +87,7 @@ are our reading of their documented behavior, not measurements. The supported set is exactly what the C++ registry registers: every architecture self-registers via `REGISTER_VLLM_MODEL`, and `scripts/check-supported-models.py` gates this list against the source so it -cannot drift. Today that is **37 registered architectures**. Each row names the +cannot drift. Today that is **38 registered architectures**. Each row names the checkpoint it was gated against and the verdict; caveats are in [STATUS.md](STATUS.md), agent detail in `.agents/model-matrix.md`. A mergeable gate/up MLP routes through one shared merged-GEMM method, so a tuned arm added @@ -136,6 +136,7 @@ speed-pending, which [BENCHMARKS.md](BENCHMARKS.md) tracks. | `LagunaForCausalLM` | poolside/Laguna-S-2.1-NVFP4, GGUF-Q4_K, Laguna-XS | byte-exact near-tie (distributional vs vLLM) | vLLM parity+ 1.03x, default on, via the `laguna-gen` CLI; the registered engine forward VT_CHECKs non-bf16 (`ARCH-ONE-SURFACE` fold) | | `KimiLinearForCausalLM` | Kimi-Linear-48B-A3B (KDA + NoPE-MLA + MoE) | **Folded onto the shared paged runner (ROW 7 §21, #122): engine==CLI 128/128 byte-identical; vs golden 122/128 (the intrinsic near-tie profile); FA2 paged MLA default-ON; SACRED post-fold green** | Served via `vllm_engine_load` + `vllm_complete_tokens` (ABI v13); server 19.0 tok/s wall vs vLLM ~21 (~0.90×), speed residual open | | `KimiK3ForConditionalGeneration` | Kimi-K3 (2.8T MoE) | scaffold: registry+config+enumeration gated, forward refuses | HW-infeasible (~1.56 TB); no run | +| `NemotronHForCausalLM` | Nemotron-3.5-Lightning-30B-A3B-NVFP4 (`nvidia` @`29f2d174`) | scaffold: registry+config+enumeration+KV-shape gated, forward refuses. 18487/18487 released tensors claimed, 0 unaccounted; het-KV shapes match `mamba2_state_shape`. Nothing runs yet (spec #517, blocked on #496) | no run; GGUF k-quants refuse by name and are owed | | `MuseGlimmerForCausalLM` | real tensors, **bf16 depth 4/52 only**: 5 prefill argmax positions match a torch transcription of vllm#51655 and HF. GGUF full depth generates coherently (#347, #359) but is **NOT token-exact** | text forward + loader vs an fp32 reference, per-mechanism property tests, scaffold 11/11, GGUF gate 17/17. An ABSENT config key now takes the architecture's constant (#412): GGUF post-norms ran at 1e-5, not 1e-8 | no vLLM denominator (pin cannot load it); SECONDARY llama.cpp, same GGUF, GB10 CPU: prefill tie **0.997x**, decode 0.232x, RSS 1.92x (#333) | | `MuseGlimmerForConditionalGeneration` | vision: **no reference run of any kind**; enumeration gated vs the released 30B index (1436/1436). Image/video need bf16 safetensors: `mmproj-kquant.gguf` is refused by name | perception encoder loaded and wired, so an image or video prompt runs; `perception_emb_norm` now armed by default (#405). Reachability plus placeholder scatter only, no image or video correctness | not measurable; anchored to open vllm#51655 | | `LlamaModel` | landed tiny synthetic embedding fixture (engine path == direct pooler path, identical vectors; f64 LAST+normalize reference); real checkpoint (e5-mistral class) is a NAMED residual | pooling/embed only, text paths refuse by task; `vllm_embed` + `/v1/embeddings` | n/a (CPU correctness-grade embeddings) | @@ -166,11 +167,10 @@ Enumerated in `.agents/model-matrix.md`, not registered, no runnable GB10 gate: | `DeepseekV3ForCausalLM`, `DeepseekV32ForCausalLM` | DeepSeek-V3 / V3.2 | 671B, ~642 GiB fp8 vs 119 GiB unified; V3.2 also DSA-indexer dep-blocked | | `GlmMoeDsaForCausalLM` | GLM-5 (DSA) | ~1404 GiB bf16; dep-blocked (GLM-5.x is DeepSeek-V3.2 verbatim) | | `MiniMaxM2ForCausalLM` | MiniMax-M2 | ~230B, ~428 GiB bf16, ~4x over the unified pool | -| `NemotronHForCausalLM` | Nemotron-H / Nemotron-3.5-Lightning-30B-A3B | capability-blocked, not HW-blocked (20.1 GiB fits the pool): Mamba2 SSD unported ([#496](https://github.com/mudler/vllm.cpp/issues/496)), MoE and loader owed; spec [#517](https://github.com/mudler/vllm.cpp/issues/517) | -27 of the 31 registered text-generation architectures carry a passing +27 of the 32 registered text-generation architectures carry a passing correctness gate today; the rest are honestly marked scaffold or blocked above. -(The 37 registered total also covers 3 Parakeet ASR entry points and the +(The 38 registered total also covers 3 Parakeet ASR entry points and the `LlamaModel` embedding arch, which are not text generation.) vLLM registers 130+ text architectures, so this is a curated, gated subset, not a breadth claim. The first EMBEDDING architecture is registered and live diff --git a/docs/USAGE.md b/docs/USAGE.md index 8ff0590df..1c1f7dd29 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -342,6 +342,7 @@ tokens quietly. | Architecture | Why it refuses | |---|---| | `KimiK3ForConditionalGeneration` | Needs ~1.56 TB (MXFP4); no host here can run it | +| `NemotronHForCausalLM` | The Mamba2 forward is not ported yet (#517 W4, blocked on #496). Safetensors resolve and parse; a GGUF file is refused by name, since no GGUF arm exists for it | This is a deliberate state, not a bug: registering the architecture is what lets the config parse and weight-name mapping be tested before the forward exists. diff --git a/src/vllm/model_executor/models/nemotron_h.h b/src/vllm/model_executor/models/nemotron_h.h new file mode 100644 index 000000000..dac7c35a7 --- /dev/null +++ b/src/vllm/model_executor/models/nemotron_h.h @@ -0,0 +1,282 @@ +// Nemotron-H (`NemotronHForCausalLM`) — the W3 STRUCTURAL bring-up of the row +// `MODEL-TEXT-nemotron-h-nemotron-hfor-causal-lm` +// ([spec](../../../../.agents/specs/nemotron-h-model.md) §4 W3, issue #517). +// +// W3 SCOPE, exactly: make the architecture KNOWN, PARSED, ENUMERATED and +// KV-SHAPED. Config descent over the released `config.json`, the on-disk weight +// name map, the registry entry, and the heterogeneous KV-cache topology (one +// full-attention group over the 6 attention layers + one Mamba2 recurrent-state +// group over the 23 mamba layers). There is **no forward path here** — that is +// W4, and `ForwardNemotronHForCausalLM` REFUSES BY NAME rather than returning a +// silent wrong answer. The row stays `INVENTORIED`; this file changes no +// lifecycle state. +// +// This header lives in `src/`, not `include/vllm/`, deliberately: `include/vllm/` +// is a `USER_USAGE_PREFIXES` surface (#515) and W3 ships nothing on the public +// ABI. Same call as the W1 resolver header. +// +// ─── WHAT THIS IS A PORT OF (file:line on BOTH sides) ──────────────────────── +// OURS <- UPSTREAM +// NemotronHParams <- transformers `NemotronHConfig` +// (models/nemotron_h/configuration_nemotron_h.py +// @ transformers 7d06b1a5 / 5.10.0.dev0): +// field defaults :86-140, the legacy-alias +// normalization + pattern fallbacks in +// `__post_init__` :142-190, `validate_layer_type` +// :195-223, `num_hidden_layers` as a PROPERTY +// over `layers_block_type` :225-238. +// vLLM vendors its OWN NemotronHConfig +// (transformers_utils/configs/nemotron_h.py +// @ 555967922) in which the polarity is +// REVERSED — `hybrid_override_pattern` is the +// ctor arg and `layers_block_type` the derived +// property (:277-287) — but that class is +// imported by `nemotron_h.py:83` for TYPE +// ANNOTATION only; the object that actually +// reaches the model at runtime comes from +// transformers `AutoConfig`. The spec's live +// oracle run (§5a) settles it on the real +// checkpoint: `CONFIG pattern = +// MEMEM*EMEMEM*EM...` was DERIVED from +// `layers_block_type`, which is what the +// released config.json ships. So +// `layers_block_type` is the source of truth +// here and `hybrid_override_pattern` is the +// legacy alias, not the other way round. +// EnumerateNemotronHTensors <- nemotron_h.py:86-123 (NemotronHMLP up/down), +// :126-256 (NemotronHMoE gate + +// e_score_correction_bias :158, shared_experts +// :181, FusedMoE ckpt_names +// ("up_proj","down_proj","") :220), +// :373-389 (MambaMixer2 fed +// mamba_num_heads*mamba_head_dim), :503 +// (NemotronHAttention), the `.mixer` prefix on +// every block (:290, :335, :373, :503), and the +// on-disk mapper `hf_to_vllm_mapper` +// (:716-724: prefix `backbone.` -> `model.`, +// substr `A_log` -> `A`, `embeddings` -> +// `embed_tokens`, and q/k/v STACKED into +// qkv_proj at load — so the CHECKPOINT ships +// them SEPARATE, which is what we enumerate). +// MTP: nemotron_h_mtp.py:40-90 +// (enorm/hnorm/eh_proj under +// has_start_projections, final_layernorm under +// has_end_norm) + :241-275 (total_layers = +// num_nextn_predict_layers * pattern_len, char +// '*' -> attention, 'E' -> moe). +// MakeNemotronHKVCache <- nemotron_h.py:761-798 +// (get_mamba_state_shape_from_config: +// intermediate_size = mamba_num_heads * +// mamba_head_dim) -> +// mamba_utils.py:173-198 (mamba2_state_shape) +// and :743-758 (get_mamba_state_dtype_from_config) +// -> mamba_utils.py:93-106 (_mamba_state_dtype). +// NemotronHModel::Forward <- nemotron_h.py:604-660 — REFUSE-by-name (W4). +#pragma once + +#include +#include +#include +#include + +#include "vllm/model_executor/models/model_registry.h" +#include "vllm/transformers_utils/hf_config.h" +#include "vllm/v1/kv_cache_interface.h" +#include "vt/dtype.h" + +namespace vllm { + +// The four decoder block kinds. Mirror of `ALL_DECODER_LAYER_TYPES` +// (nemotron_h.py:531-536) and of the pattern mapping +// {"M": mamba, "E": moe, "*": attention, "-": mlp} +// (configuration_nemotron_h.py:259-268). +enum class NemotronHBlock { kMamba, kAttention, kMoe, kMlp }; + +// The `layers_block_type` spelling of a block, as it appears in config.json. +std::string_view NemotronHBlockName(NemotronHBlock block); + +// The COARSE quantization surface — only the signals W3 needs to know WHICH +// tensors the checkpoint ships, never which kernel consumes them. +// +// Scope boundary, deliberate: resolving a per-module `quant_algo` out of +// `quantization_config.{config_groups,quantized_layers}` (5981 entries) is W1 +// of this row (#517 W1) and is NOT on `main` at this commit. W3 therefore reads +// four small, unambiguous, individually falsifiable keys and derives the scale +// companions STRUCTURALLY (mamba projections -> FP8 pair, expert/lm_head -> +// NVFP4 pair, attention k/v -> fp8-KV scale, `mtp.*` -> unquantized). The +// enumeration gate then proves that structural derivation against the released +// 18487-tensor index, so a wrong rule fails loudly instead of silently. +struct NemotronHQuantSurface { + bool present = false; // a `quantization_config` object exists at all + std::string quant_method; // "modelopt" + std::string quant_algo; // "MIXED_PRECISION" + // `kv_cache_scheme` = {"dynamic": false, "num_bits": 8, "type": "float"} — + // the fp8 KV scheme, which is what puts `k_proj.k_scale` / `v_proj.v_scale` + // on a quantized attention layer. + bool fp8_kv_cache = false; + // The `ignore` list carries the `mtp*` wildcard, so the whole MTP tower ships + // UNQUANTIZED (bf16, no scale companions) even though its backbone twin does + // not. Enumerating scales for it would over-claim 270 tensors. + bool mtp_ignored = false; +}; + +// Every NemotronH config field the enumeration / KV-cache builder / (W4) +// forward consume, resolved ONCE from the standalone HfConfig. There is no +// `text_config` nesting: `NemotronHForCausalLM` is the top-level architecture. +struct NemotronHParams { + // --- shared geometry --- + int64_t hidden_size = 0; // 2688 + int64_t vocab_size = 0; // 131072 + int64_t max_position_embeddings = 0; // 1048576 + double layer_norm_epsilon = 1e-5; + bool tie_word_embeddings = false; + + // --- the layer schedule (the SOURCE OF TRUTH for depth) --- + // 52 entries: 23 mamba, 23 moe, 6 attention at indices 5,12,19,26,33,42. + // `num_hidden_layers` in the released config is DEPRECATED and ignored by + // the runtime config's setter (configuration_nemotron_h.py:233-238); we + // mirror that and only WARN-equivalent (refuse) on a genuine conflict. + std::vector layers_block_type; + // The MTP head's own schedule; ["attention","moe"] on this checkpoint and by + // default (configuration_nemotron_h.py:179-180). + std::vector mtp_layers_block_type; + int64_t num_nextn_predict_layers = 0; // 1 + + // --- attention (32 q / 2 kv heads, head_dim 128) --- + int64_t num_attention_heads = 0; + int64_t num_key_value_heads = 0; + int64_t head_dim = 0; + double rope_theta = 10000.0; + double partial_rotary_factor = 1.0; + bool attention_bias = false; + std::optional sliding_window = std::nullopt; + + // --- Mamba2 --- + int64_t mamba_num_heads = 0; // 64 + int64_t mamba_head_dim = 0; // 64 + int64_t n_groups = 0; // 8 (legacy alias `mamba_n_groups`) + int64_t ssm_state_size = 0; // 128 + int64_t conv_kernel = 0; // 4 (legacy alias `mamba_d_conv`) + int64_t chunk_size = 0; // 128 (legacy alias `mamba_chunk_size`) + int64_t expand = 0; // 2 (legacy alias `mamba_expand`) + std::string mamba_hidden_act = "silu"; + // The recurrent (SSM) state cache dtype, resolved INDEPENDENTLY of the + // convolution-state dtype (mamba_utils.py:99-104). "float32" here. + std::string mamba_ssm_cache_dtype; + bool use_conv_bias = true; // legacy alias `mamba_conv_bias` + bool use_bias = false; + bool mamba_proj_bias = false; + double time_step_min = 1e-3; // legacy alias `mamba_dt_min` + double time_step_max = 1e-1; // legacy alias `mamba_dt_max` + double time_step_floor = 1e-4; // legacy alias `mamba_dt_init_floor` + + // --- MoE --- + int64_t n_routed_experts = 0; // 128 + int64_t num_experts_per_tok = 0; // 6 + int64_t moe_intermediate_size = 0; // 1856 + int64_t n_shared_experts = 0; // 1 + int64_t moe_shared_expert_intermediate_size = 0; // 3712 + int64_t n_group = 1; + int64_t topk_group = 1; + double routed_scaling_factor = 1.0; // 2.5, applied to the OUTPUT (:246) + bool norm_topk_prob = true; + bool moe_shared_expert_overlap = true; + // `moe_latent_size` is `null` in this checkpoint. Upstream's predicate is + // `getattr(config, "moe_latent_size", None) is not None` (nemotron_h.py:143), + // so an ABSENT key and an explicit `null` are the SAME state — both mean "no + // latent MoE". nullopt covers both; a real value REFUSES (spec §0 puts + // `fc1_latent_proj`/`fc2_latent_proj` out of scope). + std::optional moe_latent_size = std::nullopt; + + // --- dense MLP block (`-`); this checkpoint has none --- + int64_t intermediate_size = 0; + bool mlp_bias = false; + std::string mlp_hidden_act = "relu2"; + + NemotronHQuantSurface quant; + + // Depth is the SCHEDULE's length, never the deprecated scalar + // (configuration_nemotron_h.py:225-238). + int64_t num_hidden_layers() const { + return static_cast(layers_block_type.size()); + } + // MambaMixer2 is fed `mamba_num_heads * mamba_head_dim` (nemotron_h.py:377): + // 64 * 64 = 4096. This is the SSM `intermediate_size`, NOT the MLP one. + int64_t mamba_intermediate_size() const { + return mamba_num_heads * mamba_head_dim; + } + // mamba_utils.py:190 `conv_dim = intermediate_size + 2 * n_groups * + // state_size` = 4096 + 2*8*128 = 6144. Falsifiable straight off disk: the + // released `mixer.conv1d.weight` is BF16 [6144, 1, 4]. + int64_t conv_dim() const { + return mamba_intermediate_size() + 2 * n_groups * ssm_state_size; + } + // The fused zxbcdt projection width: z (intermediate) + xBC (conv_dim) + dt + // (num_heads) = 4096 + 6144 + 64 = 10304. Released `mixer.in_proj.weight` is + // F8_E4M3 [10304, 2688]. + int64_t in_proj_out_features() const { + return mamba_intermediate_size() + conv_dim() + mamba_num_heads; + } + int64_t q_proj_out_features() const { return num_attention_heads * head_dim; } + int64_t kv_proj_out_features() const { return num_key_value_heads * head_dim; } + // The layer indices carrying `block`, in ascending order. + std::vector LayerIndices(NemotronHBlock block) const; +}; + +// Resolve + validate NemotronHParams from a standalone HfConfig. Pure/host, so +// it is unit-testable against the committed released-config fixture with no +// checkpoint present. Throws with a precise message naming the offending key on +// anything this bring-up cannot represent. +NemotronHParams ParseNemotronHParams(const HfConfig& config); + +// Per-family config hook (registry `parse_config`). The resolve IS the +// validation. +void ParseNemotronHConfig(const HfConfig& config); + +// The TEMPORAL/SSM state cache dtype — mirror of +// `MambaStateDtypeCalculator._mamba_state_dtype` (mamba_utils.py:93-106): it is +// resolved from `mamba_ssm_cache_dtype` INDEPENDENTLY of the convolution-state +// dtype, and only "auto"/absent falls back to `conv_dtype`. +// +// It is resolved from `NemotronHParams` rather than through the shared +// `detail::ResolveMambaSsmCacheDType` on purpose. That helper reads +// `HfConfig::mamba_ssm_dtype`, which `hf_config.cpp:439` parses from the key +// **`mamba_ssm_dtype`** — Qwen3.5/3.6's spelling. NemotronH ships +// **`mamba_ssm_cache_dtype`** (transformers' own field name, +// configuration_nemotron_h.py:121), so the shared helper sees an empty string +// here and silently returns the CONVOLUTION dtype. That is precisely the +// "recurrent state collapsed to the activation dtype" defect: numerically +// plausible, half the bytes, and invisible to a token gate. +vt::DType NemotronHSsmCacheDType(const NemotronHParams& params, + vt::DType conv_dtype); + +// One enumerated checkpoint tensor and the named thing that consumes it. A +// tensor with no entry here is UNCLAIMED, which the enumeration gate reports as +// a failure rather than tolerating — "nobody thought of it" is not a state. +struct NemotronHTensor { + std::string name; // the name as it ships on disk + std::string consumer; // stable tag: what will read it +}; + +// The on-disk weight name map of `NemotronHForCausalLM`, by the names the +// checkpoint SHIPS (q/k/v separate, `backbone.` prefix, `A_log` not `A`) rather +// than the upstream module names. Ordered: root, backbone layers ascending, +// then the MTP tower. +std::vector EnumerateNemotronHTensors( + const NemotronHParams& params); + +// The HETEROGENEOUS KV topology: TWO groups. +// (1) a full-attention paged group over the 6 GQA layers +// (num_kv_heads=2, head_size=128); +// (2) a Mamba2 recurrent-state group over the 23 mamba layers +// (mamba_utils.py:173-198 / :93-106). +// Both groups carry their REAL per-layer names, not a single tag: a mamba +// group's page size is multiplied by `layer_names.size()` when the runner sizes +// recurrent state (kv_cache_utils.cpp:979), and an attention group's page by +// the same count when the block budget is computed +// (kv_cache_interface.cpp:151-158). A one-element tag would under-count both by +// 23x and 6x. +v1::KVCacheConfig MakeNemotronHKVCache(const HfConfig& config, int block_size, + int num_blocks); + +} // namespace vllm diff --git a/src/vllm/model_executor/models/nemotron_h_registry.cpp b/src/vllm/model_executor/models/nemotron_h_registry.cpp new file mode 100644 index 000000000..e3bac3f82 --- /dev/null +++ b/src/vllm/model_executor/models/nemotron_h_registry.cpp @@ -0,0 +1,181 @@ +// Nemotron-H (`NemotronHForCausalLM`) registry TU — the ADDITIVE +// self-registration seam for the W3 structural bring-up (#517, +// .agents/specs/nemotron-h-model.md §4 W3). Follows the +// kimi_linear_registry.cpp / deepseek_v2_registry.cpp seam exactly: a NEW +// translation unit with ONE REGISTER_VLLM_MODEL line and ZERO edit to any +// shared array. It owns the arch entry points: the config hook, the +// HETEROGENEOUS KV-cache spec (a full-attention group over the 6 GQA layers + +// a Mamba2 recurrent-state group over the 23 mamba layers), the LoadedModel +// subclass and the factory. +// +// W3 registers the arch so it RESOLVES, parses its config and enumerates its +// checkpoint. It does NOT forward: `ForwardNemotronHForCausalLM` REFUSES BY +// NAME (VT_CHECK(false), exactly like kimi_linear / deepseek_v4 / kimi_k3), so +// the TU builds and the structure is unit-testable while a forward LOUDLY +// reports the pending brick instead of returning a silent wrong answer. The +// GGUF arm refuses by name too — it is OWED (spec §5b W7), and a silent +// dequantization to a supported path is exactly what a token gate cannot see. +// The model-matrix row stays INVENTORIED until W4-W6 land. +#include "vllm/model_executor/models/model_registry.h" + +#include +#include +#include +#include +#include + +#include "vllm/model_executor/models/nemotron_h.h" +#include "vllm/model_executor/models/qwen3_5.h" // ForwardLogits +#include "vllm/v1/kv_cache_dtype.h" +#include "vllm/v1/kv_cache_interface.h" +#include "vt/dtype.h" + +namespace vllm { +namespace { + +// registry.py `_ModelInfo` for NemotronH (registry.py:179 -> +// models/nemotron_h.py::NemotronHForCausalLM): text generation, HYBRID (23 +// Mamba2 layers ⇒ a recurrent-state KV group), not multimodal. Upstream's class +// carries `HasInnerState` + `IsHybrid` (nemotron_h.py:700-712); our ModelInfo is +// a consumed subset whose only `has_inner_state` reader short-circuits on +// `is_hybrid`, so this follows the established hybrid-recurrent registration +// convention (kQwen3_5Info, kKimiLinearInfo). +inline constexpr ModelInfo kNemotronHInfo{ + .is_text_generation_model = true, + .is_pooling_model = false, + .is_hybrid = true, + .has_inner_state = false, + .supports_multimodal = false, + .score_type = "bi-encoder", +}; + +class NemotronHLoadedModel final : public LoadedModel { + public: + NemotronHLoadedModel(const ModelRegistration& registration, + NemotronHParams params) + : LoadedModel(registration), params_(std::move(params)) {} + const NemotronHParams& params() const { return params_; } + + private: + NemotronHParams params_; +}; + +std::unique_ptr LoadNemotronHForCausalLM( + const ModelRegistration& registration, const HfConfig& config, + const ModelSource& source) { + if (source.kind != ModelSource::Kind::kSafetensors) { + // AGENTS.md makes GGUF k-quants a standing requirement, not a per-model + // choice; it is OWED here (spec §5b, W7) and refused BY NAME rather than + // routed to a supported path behind the caller's back. + throw std::runtime_error( + "Model architecture NemotronHForCausalLM does not support GGUF weights " + "yet: the GGUF k-quant/i-quant arm is not ported (see " + ".agents/specs/nemotron-h-model.md §5b W7)"); + } + // The config descent IS the validation, and it refuses by name on anything + // this bring-up cannot represent. W4 owns materializing the tensors + // EnumerateNemotronHTensors names. + NemotronHParams params = ParseNemotronHParams(config); + return std::make_unique(registration, + std::move(params)); +} + +void PrepareNemotronHForCausalLM(LoadedModel& model, const HfConfig& config, + vt::Queue& queue) { + (void)model; + (void)config; + (void)queue; +} + +ForwardLogits ForwardNemotronHForCausalLM(LoadedModel& model, + const ModelForwardInput& input) { + (void)model; + (void)input; + // W4 owns the hybrid layer loop, the Mamba2 mixer wiring, the 6 attention + // layers and the MoE layers (spec §4 W4), and is itself blocked on the + // Mamba2 SSD CUDA arm (#496 W2). Refuse loudly rather than return zeros. + VT_CHECK(false, + "NemotronHForCausalLM forward is not implemented yet (W4 of " + ".agents/specs/nemotron-h-model.md, issue #517)"); + return {}; +} + +const ModelFactory kNemotronHFactory{ + .parse_config = &ParseNemotronHConfig, + .load_weights = &LoadNemotronHForCausalLM, + .prepare = &PrepareNemotronHForCausalLM, + .forward = &ForwardNemotronHForCausalLM, + .make_kv_cache = &MakeNemotronHKVCache, + .is_dense_model = false, +}; + +} // namespace + +v1::KVCacheConfig MakeNemotronHKVCache(const HfConfig& config, int block_size, + int num_blocks) { + const NemotronHParams p = ParseNemotronHParams(config); + + // Both state tensors' dtypes come from `_mamba_state_dtype` + // (mamba_utils.py:93-106): the CONVOLUTION state follows the cache dtype + // (the model dtype, bf16, unless overridden), and the TEMPORAL/SSM state is + // resolved INDEPENDENTLY from `mamba_ssm_cache_dtype` — "float32" on this + // checkpoint, so the two differ. Collapsing the SSM state to the activation + // dtype is a silent precision loss a token gate can absorb. See + // NemotronHSsmCacheDType for why the shared qwen3_5 resolver is NOT the right + // reader here (it is keyed on a different config spelling). + const vt::DType conv_dtype = vt::DType::kBF16; + const vt::DType ssm_dtype = NemotronHSsmCacheDType(p, conv_dtype); + + v1::KVCacheConfig kv; + kv.num_blocks = num_blocks; + + // (1) the 6 GQA full-attention layers. `FullAttentionSpec` sizes the paged + // K+V page; the fp8 KV scheme the checkpoint ships (k_scale/v_scale) is a + // W4/W6 storage decision and is deliberately NOT selected here. + std::vector attn_layers; + for (int64_t i : p.LayerIndices(NemotronHBlock::kAttention)) { + attn_layers.push_back("backbone.layers." + std::to_string(i) + ".mixer"); + } + kv.kv_cache_groups.emplace_back( + std::move(attn_layers), + std::make_shared( + block_size, static_cast(p.num_key_value_heads), + static_cast(p.head_dim), v1::ResolveKvCacheDType())); + + // (2) the 23 Mamba2 layers. mamba2_state_shape (mamba_utils.py:173-198) at + // tp_world_size 1: + // conv = (conv_dim, conv_kernel - 1 + num_spec) + // state = (num_heads, head_dim, state_size) + // with conv_dim = mamba_num_heads*mamba_head_dim + 2*n_groups*state_size + // = 4096 + 2*8*128 = 6144, confirmed on disk by the released + // `mixer.conv1d.weight` BF16 [6144, 1, 4]. + // + // LAYOUT NOTE: upstream's DEFAULT conv layout is "SD" = (state_len, dim) + // (mamba_utils.py:27-48, `VLLM_SSM_CONV_STATE_LAYOUT` unset ⇒ "SD"), + // while our local convention across qwen3_5_common.cpp:85 and + // kimi_linear_registry.cpp:156 is (dim, state_len). The BYTES are + // identical — same product, same page size — and this follows the local + // convention so the shared runner/manager code sees one orientation. The + // discrepancy is recorded here rather than left for W4 to rediscover. + // + // num_spec is 0: speculative decoding widens the conv row to + // (K-1)+k taps, and the MTP head is W5. + std::vector mamba_layers; + for (int64_t i : p.LayerIndices(NemotronHBlock::kMamba)) { + mamba_layers.push_back("backbone.layers." + std::to_string(i) + ".mixer"); + } + kv.kv_cache_groups.emplace_back( + std::move(mamba_layers), + std::make_shared( + block_size, + std::vector>{ + {p.conv_dim(), p.conv_kernel - 1}, + {p.mamba_num_heads, p.mamba_head_dim, p.ssm_state_size}}, + std::vector{conv_dtype, ssm_dtype})); + return kv; +} + +REGISTER_VLLM_MODEL(nemotron_h, "NemotronHForCausalLM", kNemotronHFactory, + kNemotronHInfo) + +} // namespace vllm diff --git a/src/vllm/model_executor/models/nemotron_h_weights.cpp b/src/vllm/model_executor/models/nemotron_h_weights.cpp new file mode 100644 index 000000000..b99d74e43 --- /dev/null +++ b/src/vllm/model_executor/models/nemotron_h_weights.cpp @@ -0,0 +1,553 @@ +// Nemotron-H W3: config descent + the on-disk weight name map. See nemotron_h.h +// for the port anchors and the scope boundary against W1/W4. +#include "vllm/model_executor/models/nemotron_h.h" + +#include + +#include +#include +#include +#include + +namespace vllm { +namespace { + +using nlohmann::json; + +[[noreturn]] void Refuse(const std::string& detail) { + throw std::runtime_error("NemotronHForCausalLM: " + detail); +} + +// `raw` is the whole config document; every NemotronH-specific key is read from +// it because HfConfig types only the shared subset. +const json& Raw(const HfConfig& config) { return config.raw; } + +bool Has(const json& doc, const char* key) { + return doc.contains(key) && !doc.at(key).is_null(); +} + +int64_t GetInt(const json& doc, const char* key, int64_t fallback) { + if (!Has(doc, key)) return fallback; + const json& v = doc.at(key); + if (!v.is_number_integer() && !v.is_number_unsigned()) { + Refuse(std::string("config key '") + key + "' must be an integer"); + } + return v.get(); +} + +double GetDouble(const json& doc, const char* key, double fallback) { + if (!Has(doc, key)) return fallback; + const json& v = doc.at(key); + if (!v.is_number()) { + Refuse(std::string("config key '") + key + "' must be a number"); + } + return v.get(); +} + +bool GetBool(const json& doc, const char* key, bool fallback) { + if (!Has(doc, key)) return fallback; + const json& v = doc.at(key); + if (!v.is_boolean()) { + Refuse(std::string("config key '") + key + "' must be a boolean"); + } + return v.get(); +} + +std::string GetString(const json& doc, const char* key, + const std::string& fallback) { + if (!Has(doc, key)) return fallback; + const json& v = doc.at(key); + if (!v.is_string()) { + Refuse(std::string("config key '") + key + "' must be a string"); + } + return v.get(); +} + +// The legacy-alias reads of configuration_nemotron_h.py:142-155: the modern key +// wins, the `mamba_*` alias fills in, and a checkpoint carrying only the alias +// must not silently deserialize to the class default. +int64_t GetIntAliased(const json& doc, const char* key, const char* legacy, + int64_t fallback) { + if (Has(doc, key)) return GetInt(doc, key, fallback); + return GetInt(doc, legacy, fallback); +} + +double GetDoubleAliased(const json& doc, const char* key, const char* legacy, + double fallback) { + if (Has(doc, key)) return GetDouble(doc, key, fallback); + return GetDouble(doc, legacy, fallback); +} + +bool GetBoolAliased(const json& doc, const char* key, const char* legacy, + bool fallback) { + if (Has(doc, key)) return GetBool(doc, key, fallback); + return GetBool(doc, legacy, fallback); +} + +NemotronHBlock BlockFromName(const std::string& name) { + if (name == "mamba") return NemotronHBlock::kMamba; + if (name == "attention") return NemotronHBlock::kAttention; + if (name == "moe") return NemotronHBlock::kMoe; + if (name == "mlp") return NemotronHBlock::kMlp; + // Mirror of validate_layer_type (configuration_nemotron_h.py:195-204). + Refuse("layers_block_type contains the unsupported block type '" + name + + "' (expected one of mamba, attention, moe, mlp)"); +} + +NemotronHBlock BlockFromPatternChar(char c) { + // configuration_nemotron_h.py:265-268. + switch (c) { + case 'M': + return NemotronHBlock::kMamba; + case 'E': + return NemotronHBlock::kMoe; + case '*': + return NemotronHBlock::kAttention; + case '-': + return NemotronHBlock::kMlp; + default: + Refuse(std::string("hybrid_override_pattern contains '") + c + + "' (expected one of M, E, *, -)"); + } +} + +// One layer schedule, resolved with upstream's precedence: the explicit list, +// else the legacy pattern string, else the class default. `list_key` / +// `pattern_key` are the modern/legacy pair, `fallback` the class default. +std::vector ResolveSchedule( + const json& doc, const char* list_key, const char* pattern_key, + const std::vector& fallback) { + if (Has(doc, list_key)) { + const json& v = doc.at(list_key); + if (!v.is_array()) { + Refuse(std::string("config key '") + list_key + "' must be a list"); + } + std::vector out; + out.reserve(v.size()); + for (const json& entry : v) { + if (!entry.is_string()) { + Refuse(std::string("config key '") + list_key + + "' must be a list of strings"); + } + out.push_back(BlockFromName(entry.get())); + } + return out; + } + if (Has(doc, pattern_key)) { + std::vector out; + for (char c : GetString(doc, pattern_key, "")) { + out.push_back(BlockFromPatternChar(c)); + } + return out; + } + return fallback; +} + +NemotronHQuantSurface ResolveQuantSurface(const json& doc) { + NemotronHQuantSurface q; + if (!Has(doc, "quantization_config")) return q; + const json& qc = doc.at("quantization_config"); + if (!qc.is_object()) Refuse("quantization_config must be an object"); + q.present = true; + q.quant_method = GetString(qc, "quant_method", ""); + q.quant_algo = GetString(qc, "quant_algo", ""); + // W3 resolves NO per-module algorithm — that is W1 (#517 W1). It only refuses + // a producer whose on-disk companion-tensor layout we have not read, because + // enumerating the wrong companions is exactly the silent-wrong-bytes failure + // a token gate cannot see. + if (q.quant_method != "modelopt") { + Refuse("quantization_config.quant_method '" + q.quant_method + + "' is not implemented (this row ports ModelOpt MIXED_PRECISION; see " + ".agents/specs/nemotron-h-model.md W1)"); + } + if (Has(qc, "kv_cache_scheme")) { + const json& kv = qc.at("kv_cache_scheme"); + const bool fp8 = kv.is_object() && GetInt(kv, "num_bits", 0) == 8 && + GetString(kv, "type", "") == "float"; + if (!fp8) { + Refuse("quantization_config.kv_cache_scheme is not the fp8 scheme this " + "row implements (expected num_bits 8, type float)"); + } + q.fp8_kv_cache = true; + } + if (Has(qc, "ignore")) { + const json& ignore = qc.at("ignore"); + if (!ignore.is_array()) Refuse("quantization_config.ignore must be a list"); + for (const json& entry : ignore) { + if (!entry.is_string()) continue; + const std::string name = entry.get(); + // The released list carries the wildcard `mtp*`, which is what leaves the + // whole MTP tower bf16 with no scale companions. + if (name.rfind("mtp", 0) == 0) q.mtp_ignored = true; + } + } + return q; +} + +// ─── enumeration helpers ───────────────────────────────────────────────────── + +void Claim(std::vector& out, std::string name, + std::string consumer) { + out.push_back(NemotronHTensor{std::move(name), std::move(consumer)}); +} + +// An NVFP4 W4A16 group-16 weight: the packed nibbles, the per-16-block e4m3 +// scale, and the fp32 global scale (spec §1; config_groups group_1 on the +// released checkpoint). WHICH kernel consumes them is W1/W4, not W3. +void ClaimNvfp4(std::vector& out, const std::string& prefix, + const std::string& consumer, bool quantized) { + Claim(out, prefix + ".weight", consumer); + if (!quantized) return; + Claim(out, prefix + ".weight_scale", consumer + ".weight_scale[nvfp4]"); + Claim(out, prefix + ".weight_scale_2", consumer + ".weight_scale_2[nvfp4]"); +} + +// An FP8 W8A8 static-scaled projection: the e4m3 weight, its fp32 weight scale +// and the fp32 static input scale (config_groups group_0, 46 targets). +void ClaimFp8(std::vector& out, const std::string& prefix, + const std::string& consumer) { + Claim(out, prefix + ".weight", consumer); + Claim(out, prefix + ".weight_scale", consumer + ".weight_scale[fp8]"); + Claim(out, prefix + ".input_scale", consumer + ".input_scale[fp8]"); +} + +// One Mamba2 mixer (MambaMixer2, nemotron_h.py:373-389). `use_conv_bias` and +// `mamba_proj_bias` gate the two optional biases exactly as upstream does. +void ClaimMamba(std::vector& out, const NemotronHParams& p, + const std::string& mixer) { + ClaimFp8(out, mixer + ".in_proj", "mamba2.in_proj"); + ClaimFp8(out, mixer + ".out_proj", "mamba2.out_proj"); + if (p.mamba_proj_bias) { + Claim(out, mixer + ".in_proj.bias", "mamba2.in_proj.bias"); + Claim(out, mixer + ".out_proj.bias", "mamba2.out_proj.bias"); + } + Claim(out, mixer + ".conv1d.weight", "mamba2.conv1d"); + if (p.use_conv_bias) Claim(out, mixer + ".conv1d.bias", "mamba2.conv1d.bias"); + // `A_log` on disk; the mapper renames it to `A` in-module + // (nemotron_h.py:719). + Claim(out, mixer + ".A_log", "mamba2.A_log"); + Claim(out, mixer + ".D", "mamba2.D"); + Claim(out, mixer + ".dt_bias", "mamba2.dt_bias"); + Claim(out, mixer + ".norm.weight", "mamba2.gated_rmsnorm"); +} + +// One GQA attention mixer (NemotronHAttention, nemotron_h.py:503). q/k/v ship +// SEPARATE on disk; upstream stacks them into `qkv_proj` at load +// (hf_to_vllm_mapper orig_to_new_stacked, :719-723). +void ClaimAttention(std::vector& out, + const NemotronHParams& p, const std::string& mixer, + bool fp8_kv) { + for (const char* proj : {"q_proj", "k_proj", "v_proj", "o_proj"}) { + Claim(out, mixer + "." + proj + ".weight", std::string("attn.") + proj); + if (p.attention_bias) { + Claim(out, mixer + "." + proj + ".bias", + std::string("attn.") + proj + ".bias"); + } + } + if (fp8_kv) { + Claim(out, mixer + ".k_proj.k_scale", "attn.k_scale[fp8-kv]"); + Claim(out, mixer + ".v_proj.v_scale", "attn.v_scale[fp8-kv]"); + } +} + +// One non-gated relu² MoE block (NemotronHMoE, nemotron_h.py:126-256). There is +// NO gate_proj anywhere: FusedMoE is built with +// ckpt_names=("up_proj","down_proj","") (:220). +void ClaimMoe(std::vector& out, const NemotronHParams& p, + const std::string& mixer, bool quantized) { + Claim(out, mixer + ".gate.weight", "moe.router"); + // The noaux_tc score-correction bias registered on the gate (:158). + Claim(out, mixer + ".gate.e_score_correction_bias", "moe.router.bias"); + for (int64_t e = 0; e < p.n_routed_experts; ++e) { + const std::string expert = mixer + ".experts." + std::to_string(e); + ClaimNvfp4(out, expert + ".up_proj", "moe.expert.up_proj", quantized); + ClaimNvfp4(out, expert + ".down_proj", "moe.expert.down_proj", quantized); + } + if (p.n_shared_experts > 0) { + const std::string shared = mixer + ".shared_experts"; + ClaimNvfp4(out, shared + ".up_proj", "moe.shared.up_proj", quantized); + ClaimNvfp4(out, shared + ".down_proj", "moe.shared.down_proj", quantized); + } +} + +// One dense relu² MLP block (NemotronHMLP, nemotron_h.py:86-123). Absent from +// the driver checkpoint (no `-` in its schedule) but reachable through the +// class default schedule, so it is enumerated rather than left to be discovered. +// +// HONEST DEBT: the quantized companion layout here is DERIVED, not verified. +// `up_proj`/`down_proj` are the same ColumnParallelLinear/RowParallelLinear +// pair the experts use under the same ModelOpt config, so the NVFP4 triple is +// the only consistent reading — but no in-scope released NemotronH checkpoint +// ships an `mlp` block, so the enumeration gate cannot confirm it. It is +// recorded here rather than discovered later. +void ClaimMlp(std::vector& out, const NemotronHParams& p, + const std::string& mixer, bool quantized) { + ClaimNvfp4(out, mixer + ".up_proj", "mlp.up_proj", quantized); + ClaimNvfp4(out, mixer + ".down_proj", "mlp.down_proj", quantized); + if (p.mlp_bias) { + Claim(out, mixer + ".up_proj.bias", "mlp.up_proj.bias"); + Claim(out, mixer + ".down_proj.bias", "mlp.down_proj.bias"); + } +} + +} // namespace + +std::string_view NemotronHBlockName(NemotronHBlock block) { + switch (block) { + case NemotronHBlock::kMamba: + return "mamba"; + case NemotronHBlock::kAttention: + return "attention"; + case NemotronHBlock::kMoe: + return "moe"; + case NemotronHBlock::kMlp: + return "mlp"; + } + return "unknown"; +} + +std::vector NemotronHParams::LayerIndices(NemotronHBlock block) const { + std::vector out; + for (size_t i = 0; i < layers_block_type.size(); ++i) { + if (layers_block_type[i] == block) out.push_back(static_cast(i)); + } + return out; +} + +NemotronHParams ParseNemotronHParams(const HfConfig& config) { + const json& doc = Raw(config); + NemotronHParams p; + + // --- the schedule, which IS the depth --- + p.layers_block_type = ResolveSchedule( + doc, "layers_block_type", "hybrid_override_pattern", + // configuration_nemotron_h.py:165. + {NemotronHBlock::kMamba, NemotronHBlock::kMoe, NemotronHBlock::kAttention, + NemotronHBlock::kMlp}); + if (p.layers_block_type.empty()) { + Refuse("layers_block_type resolved to an empty schedule"); + } + p.num_nextn_predict_layers = GetInt(doc, "num_nextn_predict_layers", 0); + p.mtp_layers_block_type = + ResolveSchedule(doc, "mtp_layers_block_type", + "mtp_hybrid_override_pattern", + // configuration_nemotron_h.py:180. + {NemotronHBlock::kAttention, NemotronHBlock::kMoe}); + if (p.num_nextn_predict_layers > 0 && p.mtp_layers_block_type.empty()) { + // Mirror of validate_layer_type (configuration_nemotron_h.py:206-212). + Refuse( + "mtp_layers_block_type is required when num_nextn_predict_layers > 0"); + } + + // --- shared geometry --- + p.hidden_size = GetInt(doc, "hidden_size", 4096); + p.vocab_size = GetInt(doc, "vocab_size", 131072); + p.max_position_embeddings = GetInt(doc, "max_position_embeddings", 4096); + p.layer_norm_epsilon = GetDouble(doc, "layer_norm_epsilon", 1e-5); + p.tie_word_embeddings = GetBool(doc, "tie_word_embeddings", false); + + // --- attention --- + p.num_attention_heads = GetInt(doc, "num_attention_heads", 32); + // Three states, not two (configuration_nemotron_h.py:97 default 8, :188-189 + // `if self.num_key_value_heads is None: = num_attention_heads`): ABSENT takes + // the class default 8, an explicit `null` takes the query-head count, a value + // is taken as-is. + if (Has(doc, "num_key_value_heads")) { + p.num_key_value_heads = GetInt(doc, "num_key_value_heads", 8); + } else if (doc.contains("num_key_value_heads")) { + p.num_key_value_heads = p.num_attention_heads; + } else { + p.num_key_value_heads = 8; + } + p.head_dim = GetInt(doc, "head_dim", 128); + p.rope_theta = GetDouble(doc, "rope_theta", 10000.0); + p.partial_rotary_factor = GetDouble(doc, "partial_rotary_factor", 1.0); + p.attention_bias = GetBool(doc, "attention_bias", false); + if (Has(doc, "sliding_window")) { + p.sliding_window = GetInt(doc, "sliding_window", 0); + } + + // --- Mamba2 (legacy aliases normalized, configuration_nemotron_h.py:142-155) --- + p.mamba_num_heads = GetInt(doc, "mamba_num_heads", 128); + p.mamba_head_dim = GetInt(doc, "mamba_head_dim", 64); + p.n_groups = GetIntAliased(doc, "n_groups", "mamba_n_groups", 8); + p.ssm_state_size = GetInt(doc, "ssm_state_size", 128); + p.conv_kernel = GetIntAliased(doc, "conv_kernel", "mamba_d_conv", 4); + p.chunk_size = GetIntAliased(doc, "chunk_size", "mamba_chunk_size", 128); + p.expand = GetIntAliased(doc, "expand", "mamba_expand", 2); + p.mamba_hidden_act = GetString(doc, "mamba_hidden_act", "silu"); + p.mamba_ssm_cache_dtype = GetString(doc, "mamba_ssm_cache_dtype", "float32"); + p.use_conv_bias = GetBoolAliased(doc, "use_conv_bias", "mamba_conv_bias", true); + p.use_bias = GetBool(doc, "use_bias", false); + p.mamba_proj_bias = GetBool(doc, "mamba_proj_bias", false); + p.time_step_min = GetDoubleAliased(doc, "time_step_min", "mamba_dt_min", 1e-3); + p.time_step_max = GetDoubleAliased(doc, "time_step_max", "mamba_dt_max", 1e-1); + p.time_step_floor = + GetDoubleAliased(doc, "time_step_floor", "mamba_dt_init_floor", 1e-4); + if (p.mamba_num_heads <= 0 || p.mamba_head_dim <= 0 || p.n_groups <= 0 || + p.ssm_state_size <= 0 || p.conv_kernel <= 1) { + Refuse("the Mamba2 geometry is degenerate (mamba_num_heads, mamba_head_dim, " + "n_groups, ssm_state_size must be positive and conv_kernel > 1)"); + } + + // --- MoE --- + p.n_routed_experts = GetInt(doc, "n_routed_experts", 8); + p.num_experts_per_tok = GetInt(doc, "num_experts_per_tok", 2); + p.moe_intermediate_size = GetInt(doc, "moe_intermediate_size", 7688); + p.n_shared_experts = GetInt(doc, "n_shared_experts", 1); + p.moe_shared_expert_intermediate_size = + GetInt(doc, "moe_shared_expert_intermediate_size", 7688); + p.n_group = GetInt(doc, "n_group", 1); + p.topk_group = GetInt(doc, "topk_group", 1); + p.routed_scaling_factor = GetDouble(doc, "routed_scaling_factor", 1.0); + p.norm_topk_prob = GetBool(doc, "norm_topk_prob", true); + p.moe_shared_expert_overlap = GetBool(doc, "moe_shared_expert_overlap", true); + p.mlp_hidden_act = GetString(doc, "mlp_hidden_act", "relu2"); + if (Has(doc, "moe_latent_size")) { + p.moe_latent_size = GetInt(doc, "moe_latent_size", 0); + // The `fc1_latent_proj`/`fc2_latent_proj` pair (nemotron_h.py:191-207) is + // out of scope for this row (spec §0), and silently ignoring the key would + // build a differently-shaped MoE with no error. + Refuse("moe_latent_size is set, but the latent MoE " + "(fc1_latent_proj/fc2_latent_proj) is out of scope for this row " + "(see .agents/specs/nemotron-h-model.md §0)"); + } + if (p.mlp_hidden_act != "relu2") { + Refuse("mlp_hidden_act '" + p.mlp_hidden_act + + "' is not implemented (this architecture is the non-gated relu2 " + "expert; see .agents/specs/nemotron-h-model.md W2)"); + } + + // --- dense MLP block --- + if (doc.contains("intermediate_size") && doc.at("intermediate_size").is_array()) { + // `get_nemotron_h_config_for_layer` / NemotronHPuzzleForCausalLM + // (nemotron_h.py:283-288) is explicitly out of scope (spec §0). + Refuse("a per-layer intermediate_size list (NemotronHPuzzleForCausalLM) is " + "out of scope for this row (see .agents/specs/nemotron-h-model.md §0)"); + } + p.intermediate_size = GetInt(doc, "intermediate_size", 21504); + p.mlp_bias = GetBool(doc, "mlp_bias", false); + + // The recurrent-cache dtype is resolved independently of the conv dtype + // (mamba_utils.py:99-104); refuse an alias we cannot represent HERE rather + // than at cache-allocation time. + const std::string& ssm = p.mamba_ssm_cache_dtype; + if (!(ssm.empty() || ssm == "auto" || ssm == "float32" || ssm == "float" || + ssm == "float16" || ssm == "half" || ssm == "bfloat16")) { + Refuse("mamba_ssm_cache_dtype '" + ssm + + "' is not supported (expected auto, float32/float, float16/half or " + "bfloat16)"); + } + + p.quant = ResolveQuantSurface(doc); + + // `num_hidden_layers` is DEPRECATED, and its setter ignores whatever the + // checkpoint says (configuration_nemotron_h.py:233-238) — depth is the + // schedule's LENGTH. Upstream only WARNS on a conflicting scalar + // (:167-175), so this mirrors that rather than refusing: the released + // config.json ships `num_hidden_layers: 52` alongside a 52-entry + // `layers_block_type`, and a checkpoint that ships only the scalar becomes a + // 4-block model upstream too. The behavior is pinned by a test so it stays a + // deliberate mirror rather than an accident. + (void)config.num_hidden_layers; + return p; +} + +void ParseNemotronHConfig(const HfConfig& config) { + (void)ParseNemotronHParams(config); +} + +vt::DType NemotronHSsmCacheDType(const NemotronHParams& params, + vt::DType conv_dtype) { + const std::string& dtype = params.mamba_ssm_cache_dtype; + if (dtype.empty() || dtype == "auto") return conv_dtype; + if (dtype == "float32" || dtype == "float") return vt::DType::kF32; + if (dtype == "float16" || dtype == "half") return vt::DType::kF16; + if (dtype == "bfloat16") return vt::DType::kBF16; + // ParseNemotronHParams already refused anything else by name; this is the + // unreachable arm kept so the mapping cannot silently widen. + Refuse("mamba_ssm_cache_dtype '" + dtype + "' is not supported"); +} + +std::vector EnumerateNemotronHTensors( + const NemotronHParams& params) { + const NemotronHParams& p = params; + // The backbone is quantized where the config groups say so; the MTP tower is + // covered by the `mtp*` entry in `ignore` and ships bf16 with no companions. + const bool quantized = p.quant.present; + const bool mtp_quantized = quantized && !p.quant.mtp_ignored; + + std::vector out; + out.reserve(4096); + + // --- root --- + Claim(out, "backbone.embeddings.weight", "embed_tokens"); + Claim(out, "backbone.norm_f.weight", "final_norm"); + if (!p.tie_word_embeddings) { + ClaimNvfp4(out, "lm_head", "lm_head", quantized); + } + + // --- the 52 backbone layers --- + for (size_t i = 0; i < p.layers_block_type.size(); ++i) { + const std::string layer = "backbone.layers." + std::to_string(i); + const std::string mixer = layer + ".mixer"; + // Every block kind carries the pre-mixer RMSNorm (nemotron_h.py:299, :339, + // :391, :521). + Claim(out, layer + ".norm.weight", "layer_norm"); + switch (p.layers_block_type[i]) { + case NemotronHBlock::kMamba: + ClaimMamba(out, p, mixer); + break; + case NemotronHBlock::kAttention: + ClaimAttention(out, p, mixer, quantized && p.quant.fp8_kv_cache); + break; + case NemotronHBlock::kMoe: + ClaimMoe(out, p, mixer, quantized); + break; + case NemotronHBlock::kMlp: + ClaimMlp(out, p, mixer, quantized); + break; + } + } + + // --- the MTP tower (nemotron_h_mtp.py:241-275) --- + // total_layers = num_nextn_predict_layers * pattern_len; the FIRST block of + // every step carries enorm/hnorm/eh_proj and the LAST carries + // final_layernorm (:65-86, :152-173). + const int64_t pattern_len = + static_cast(p.mtp_layers_block_type.size()); + const int64_t mtp_layers = p.num_nextn_predict_layers * pattern_len; + for (int64_t i = 0; i < mtp_layers; ++i) { + const int64_t rel = i % pattern_len; + const std::string layer = "mtp.layers." + std::to_string(i); + const std::string mixer = layer + ".mixer"; + Claim(out, layer + ".norm.weight", "mtp.layer_norm"); + if (rel == 0) { + Claim(out, layer + ".enorm.weight", "mtp.enorm"); + Claim(out, layer + ".hnorm.weight", "mtp.hnorm"); + Claim(out, layer + ".eh_proj.weight", "mtp.eh_proj"); + } + if (rel == pattern_len - 1) { + Claim(out, layer + ".final_layernorm.weight", "mtp.final_layernorm"); + } + switch (p.mtp_layers_block_type[static_cast(rel)]) { + case NemotronHBlock::kAttention: + // The MTP attention block is unquantized: no k_scale/v_scale ship for + // it, which is what `ignore: [... "mtp*"]` buys. + ClaimAttention(out, p, mixer, mtp_quantized && p.quant.fp8_kv_cache); + break; + case NemotronHBlock::kMoe: + ClaimMoe(out, p, mixer, mtp_quantized); + break; + case NemotronHBlock::kMamba: + ClaimMamba(out, p, mixer); + break; + case NemotronHBlock::kMlp: + ClaimMlp(out, p, mixer, mtp_quantized); + break; + } + } + return out; +} + +} // namespace vllm diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 77d4fda9a..ac8660d3b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -327,6 +327,28 @@ target_include_directories(test_muse_glimmer_gguf PRIVATE # .agents/specs/kimi-linear.md. vllm_cpp_add_test(test_kimi_linear_scaffold vllm/models/test_kimi_linear_scaffold.cpp) +# Nemotron-H (NemotronHForCausalLM) W3 STRUCTURAL gate (#517, +# .agents/specs/nemotron-h-model.md §4 W3): registry-resolve + config descent off +# the REAL released config.json (52-entry layers_block_type schedule, the legacy +# hybrid_override_pattern/mamba_* aliases, the refuse cases) + the on-disk name map +# accounted BOTH WAYS against all 18487 tensors of the released index + the +# heterogeneous KV topology (6-layer full-attention group + 23-layer Mamba2 group, +# shapes from mamba2_state_shape). The structural half runs off a COMMITTED +# headers-only fixture so CI never needs the 20.1 GiB NAS asset; a final case +# re-verifies that fixture against the LIVE checkpoint when VT_NEMOTRON35_SNAPSHOT +# names it, and SKIPs loudly otherwise. CPU-only, no forward (W4), no speed claim. +vllm_cpp_add_test(test_nemotron_h_scaffold vllm/models/test_nemotron_h_scaffold.cpp) +target_compile_definitions(test_nemotron_h_scaffold PRIVATE + NEMOTRON_H_CKPT_FIXTURE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/vllm/models/fixtures/nemotron_h_35_lightning") +# GATE-PIN-UNPINNED-SNAPSHOTS (#471): the live re-verification resolves the +# checkpoint through the pinned parity::HfSnapshot, so it needs tests/parity on +# the include path. +# nemotron_h.h is a MODEL-PRIVATE header under src/, not include/vllm/: W3 ships +# nothing on the public ABI and include/vllm/ is a USER_USAGE_PREFIXES surface +# (#515). Same call as the W1 resolver header. +target_include_directories(test_nemotron_h_scaffold PRIVATE + ${CMAKE_SOURCE_DIR}/tests/parity ${CMAKE_SOURCE_DIR}/src) + # Kimi-Linear CPU REFERENCE FORWARD gates (W2-W6): the per-op reference forwards # (KDA layer vs the kimi_kda host-ref composition; NoPE-MLA vs a materialized-MHA # reference; sigmoid noaux_tc router + block vs a hand-computed case) and the whole diff --git a/tests/vllm/models/test_model_registry.cpp b/tests/vllm/models/test_model_registry.cpp index 5bf5a790a..47505fdb3 100644 --- a/tests/vllm/models/test_model_registry.cpp +++ b/tests/vllm/models/test_model_registry.cpp @@ -44,10 +44,10 @@ HfConfig Config(std::vector architectures) { TEST_CASE("registry_imports: every registered architecture has a complete factory") { const auto registrations = ModelRegistry::Registrations(); - // 30 text archs + the 3 Parakeet transcription-only archs (ARCH-ONE-SURFACE + // 34 text archs + the 3 Parakeet transcription-only archs (ARCH-ONE-SURFACE // ROW 1: ParakeetForCTC/ForRNNT/ForTDT, SupportsTranscription mirror) + the // LlamaModel embedding arch (ARCH-ONE-SURFACE ROW 6, is_pooling_model). - REQUIRE(registrations.size() == 37); + REQUIRE(registrations.size() == 38); for (const ModelRegistration& registration : registrations) { CAPTURE(registration.architecture); @@ -139,7 +139,7 @@ TEST_CASE("self_registration: every arch self-registers from its own TU") { // with the kExampleConfigArchitectures ledger; adding a model appends its two // entries here. const std::vector supported = ModelRegistry::SupportedArchs(); - REQUIRE(supported.size() == 37); + REQUIRE(supported.size() == 38); CHECK(std::is_sorted(supported.begin(), supported.end())); // The full byte-order sequence. Note "MiniCPM3" < "MiniCPMF" and "Phi3" < // "PhiF" ('3' 0x33 < 'F' 0x46); "OPT" < "Olmo" ('P' 0x50 < 'l' 0x6C); and among @@ -169,6 +169,7 @@ TEST_CASE("self_registration: every arch self-registers from its own TU") { "MistralForCausalLM", "MuseGlimmerForCausalLM", "MuseGlimmerForConditionalGeneration", + "NemotronHForCausalLM", "OPTForCausalLM", "Olmo2ForCausalLM", "Olmo3ForCausalLM", @@ -241,6 +242,16 @@ TEST_CASE("registry_model_property: Qwen registrations match pinned _ModelInfo") // registrations (Kimi-K3 = KDA+MLA hybrid backbone + MoonViT-V2 vision). CHECK(registration.info.is_hybrid); CHECK(registration.info.supports_multimodal); + } else if (registration.architecture == "NemotronHForCausalLM") { + // Nemotron-H (MODEL-NEMOTRON-H W3, #517): text-only HYBRID -- 23 Mamba2 + // layers carry a recurrent-state KV group alongside 6 GQA full-attention + // layers. Upstream's class carries HasInnerState + IsHybrid + // (nemotron_h.py:700-712); has_inner_state stays false here under the + // blanket assertion above, following the kQwen3_5Info / kKimiLinearInfo + // convention (our ModelInfo is a consumed subset whose only reader + // short-circuits on is_hybrid). Text-only: no vision or audio tower. + CHECK(registration.info.is_hybrid); + CHECK_FALSE(registration.info.supports_multimodal); } else if (registration.architecture == "KimiLinearForCausalLM") { // Kimi-Linear-48B-A3B: text-only HYBRID (20 KDA linear-attn layers ⇒ a GDN // recurrent-state KV group + 7 NoPE-MLA layers) — is_hybrid YES, @@ -579,7 +590,7 @@ TEST_CASE("Qwen3.5 SSM cache dtype accepts upstream torch aliases exactly") { TEST_CASE("hf_registry_coverage: every registration has an example config fixture") { // C++ fixture registry for the currently implemented subset. Keep this list // alias-for-alias with the central ordered table, mirroring HF_EXAMPLE_MODELS. - constexpr std::array kExampleConfigArchitectures{ + constexpr std::array kExampleConfigArchitectures{ "CohereForCausalLM", "DeepseekV2ForCausalLM", "DeepseekV4ForCausalLM", @@ -603,6 +614,7 @@ TEST_CASE("hf_registry_coverage: every registration has an example config fixtur "MistralForCausalLM", "MuseGlimmerForCausalLM", "MuseGlimmerForConditionalGeneration", + "NemotronHForCausalLM", "OPTForCausalLM", "Olmo2ForCausalLM", "Olmo3ForCausalLM", @@ -696,6 +708,7 @@ TEST_CASE("raise_for_unsupported: subset default message and order match oracle" "'LagunaForCausalLM', " "'LlamaForCausalLM', 'LlamaModel', " "'MiniCPM3ForCausalLM', 'MiniCPMForCausalLM', 'MistralForCausalLM', 'MuseGlimmerForCausalLM', 'MuseGlimmerForConditionalGeneration', " + "'NemotronHForCausalLM', " "'OPTForCausalLM', 'Olmo2ForCausalLM', 'Olmo3ForCausalLM', " "'ParakeetForCTC', 'ParakeetForRNNT', 'ParakeetForTDT', " "'Phi3ForCausalLM', 'PhiForCausalLM', 'Qwen3ForCausalLM', " @@ -718,6 +731,7 @@ TEST_CASE("raise_for_unsupported: subset default message and order match oracle" "'LagunaForCausalLM', " "'LlamaForCausalLM', 'LlamaModel', " "'MiniCPM3ForCausalLM', 'MiniCPMForCausalLM', 'MistralForCausalLM', 'MuseGlimmerForCausalLM', 'MuseGlimmerForConditionalGeneration', " + "'NemotronHForCausalLM', " "'OPTForCausalLM', 'Olmo2ForCausalLM', 'Olmo3ForCausalLM', " "'ParakeetForCTC', 'ParakeetForRNNT', 'ParakeetForTDT', " "'Phi3ForCausalLM', 'PhiForCausalLM', 'Qwen3ForCausalLM', " diff --git a/tests/vllm/models/test_nemotron_h_scaffold.cpp b/tests/vllm/models/test_nemotron_h_scaffold.cpp new file mode 100644 index 000000000..5a06229af --- /dev/null +++ b/tests/vllm/models/test_nemotron_h_scaffold.cpp @@ -0,0 +1,582 @@ +// Nemotron-H (`NemotronHForCausalLM`) W3 STRUCTURAL gate — issue #517, spec +// `.agents/specs/nemotron-h-model.md` §4 W3. +// +// Proves the four things W3 can prove on CPU with no GPU and no 20.1 GiB +// checkpoint, plus one live re-verification that runs only where the checkpoint +// is staged: +// (1) the arch RESOLVES through the registry (the additive TU registered it); +// (2) the config PARSES off the REAL released config.json (committed as a +// fixture): the 52-entry schedule with 23 mamba / 23 moe / 6 attention at +// indices 5,12,19,26,33,42, the mamba/attention/MoE geometry, the coarse +// quantization surface — and the legacy layouts +// (`hybrid_override_pattern`, `mamba_n_groups`, `mamba_d_conv`, ...) +// normalize to the SAME params, while an unrepresentable config REFUSES +// BY NAME; +// (3) the on-disk NAME MAP is faithful: every one of the 18487 tensors in the +// released `model.safetensors.index.json` is CLAIMED by a named consumer, +// and nothing is enumerated that the checkpoint does not ship; +// (4) the HETEROGENEOUS KV topology matches `mamba2_state_shape` / +// `_mamba_state_dtype` and carries the REAL per-layer names. +// The forward REFUSES BY NAME (W4 owns it) and GGUF REFUSES BY NAME (W7). +// +// The fixture is a HEADERS-ONLY projection of the released checkpoint at the +// pinned revision `29f2d174`; see its `_provenance`. The live re-verification +// resolves the checkpoint through `parity::Nemotron35LightningSnapshot()` +// (env `VT_NEMOTRON35_SNAPSHOT`) and SKIPS loudly when it is absent. +#include "vllm/model_executor/models/nemotron_h.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hf_snapshot.h" +#include "vllm/model_executor/models/model_registry.h" +#include "vllm/transformers_utils/hf_config.h" +#include "vllm/v1/kv_cache_interface.h" +#include "vt/dtype.h" + +using vllm::EnumerateNemotronHTensors; +using vllm::HfConfig; +using vllm::LoadHfConfig; +using vllm::MakeNemotronHKVCache; +using vllm::ModelRegistry; +using vllm::NemotronHBlock; +using vllm::NemotronHParams; +using vllm::NemotronHTensor; +using vllm::ParseNemotronHParams; + +namespace { + +std::string FixtureDir() { +#ifdef NEMOTRON_H_CKPT_FIXTURE_DIR + return NEMOTRON_H_CKPT_FIXTURE_DIR; +#else + return "tests/vllm/models/fixtures/nemotron_h_35_lightning"; +#endif +} + +nlohmann::json ReadJson(const std::string& path) { + std::ifstream in(path); + REQUIRE_MESSAGE(in.good(), "cannot open " << path); + nlohmann::json j; + in >> j; + return j; +} + +// A throwaway config.json on disk, so the test drives the SAME LoadHfConfig the +// engine uses rather than hand-building an HfConfig. +class TempConfig { + public: + explicit TempConfig(const nlohmann::json& doc) { + static int counter = 0; + dir_ = std::filesystem::temp_directory_path() / + ("nemotron_h_cfg_" + std::to_string(counter++)); + std::filesystem::create_directories(dir_); + std::ofstream(dir_ / "config.json") << doc.dump(); + } + ~TempConfig() { + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + std::string path() const { return (dir_ / "config.json").string(); } + + private: + std::filesystem::path dir_; +}; + +nlohmann::json FixtureConfigDoc() { + return ReadJson(FixtureDir() + "/config.json"); +} + +NemotronHParams FixtureParams() { + TempConfig cfg(FixtureConfigDoc()); + return ParseNemotronHParams(LoadHfConfig(cfg.path())); +} + +// Expand the committed collapsed index into the full 18487 on-disk names. +// `{E}` is the routed-expert index and `count` says how many share the entry. +std::map>> +ExpandIndexFixture(const nlohmann::json& index) { + std::map>> out; + for (const auto& [pattern, meta] : index.at("tensors").items()) { + const auto dtype = meta.at("dtype").get(); + const auto shape = meta.at("shape").get>(); + const auto count = meta.at("count").get(); + const std::string marker = "{E}"; + const size_t at = pattern.find(marker); + if (at == std::string::npos) { + REQUIRE_MESSAGE(count == 1, "non-expert family with count != 1: " + << pattern); + out.emplace(pattern, std::make_pair(dtype, shape)); + continue; + } + for (int64_t e = 0; e < count; ++e) { + std::string name = pattern; + name.replace(at, marker.size(), std::to_string(e)); + out.emplace(std::move(name), std::make_pair(dtype, shape)); + } + } + return out; +} + +// Read the LIVE checkpoint's index + safetensors HEADERS (never a tensor byte; +// the checkpoint is 20.1 GiB). +std::map>> +ReadLiveHeaders(const std::string& dir) { + const nlohmann::json index = ReadJson(dir + "/model.safetensors.index.json"); + std::set shards; + for (const auto& [name, shard] : index.at("weight_map").items()) { + (void)name; + shards.insert(shard.get()); + } + std::map>> out; + for (const std::string& shard : shards) { + std::ifstream in(dir + "/" + shard, std::ios::binary); + REQUIRE_MESSAGE(in.good(), "cannot open shard " << shard); + unsigned char raw[8]; + in.read(reinterpret_cast(raw), 8); + uint64_t n = 0; + for (int i = 7; i >= 0; --i) n = (n << 8) | raw[i]; + std::string header(static_cast(n), '\0'); + in.read(header.data(), static_cast(n)); + // MATERIALIZE the parsed header: `json::parse(x).items()` binds a range + // to a TEMPORARY, which is destroyed before the loop body runs. It reads + // as a clean one-liner and it is undefined behaviour — here it surfaced as + // `[json.exception.type_error.304] cannot use at() with null` from inside + // the loop, nowhere near the parse. + const nlohmann::json hj = nlohmann::json::parse(header); + REQUIRE_MESSAGE(hj.is_object(), "shard header is not an object: " << shard); + for (const auto& [name, meta] : hj.items()) { + if (name == "__metadata__") continue; + out.emplace(name, std::make_pair(meta.at("dtype").get(), + meta.at("shape").get>())); + } + } + return out; +} + +const vllm::v1::MambaSpec& MambaGroup(const vllm::v1::KVCacheConfig& kv) { + for (const auto& group : kv.kv_cache_groups) { + const auto* spec = + dynamic_cast(group.kv_cache_spec.get()); + if (spec != nullptr) return *spec; + } + FAIL("no MambaSpec group"); + std::abort(); +} + +} // namespace + +TEST_CASE("NemotronH: the architecture resolves through the model registry") { + const std::vector archs{"NemotronHForCausalLM"}; + const vllm::ModelRegistration& reg = ModelRegistry::Resolve(archs); + CHECK(std::string(reg.architecture) == "NemotronHForCausalLM"); + CHECK(reg.info.is_text_generation_model); + CHECK(reg.info.is_hybrid); + CHECK_FALSE(reg.info.supports_multimodal); + CHECK(reg.factory != nullptr); + CHECK(reg.factory->make_kv_cache != nullptr); +} + +TEST_CASE("NemotronH config: the REAL released config.json parses") { + const NemotronHParams p = FixtureParams(); + + // Depth comes from the SCHEDULE, not from the deprecated scalar. + CHECK(p.num_hidden_layers() == 52); + CHECK(p.LayerIndices(NemotronHBlock::kMamba).size() == 23); + CHECK(p.LayerIndices(NemotronHBlock::kMoe).size() == 23); + CHECK(p.LayerIndices(NemotronHBlock::kMlp).empty()); + CHECK(p.LayerIndices(NemotronHBlock::kAttention) == + std::vector{5, 12, 19, 26, 33, 42}); + + CHECK(p.hidden_size == 2688); + CHECK(p.vocab_size == 131072); + CHECK(p.max_position_embeddings == 1048576); + CHECK_FALSE(p.tie_word_embeddings); + CHECK(p.layer_norm_epsilon == 1e-5); + + // Attention: 32 q / 2 kv heads, head_dim 128, full rotary, no window. + CHECK(p.num_attention_heads == 32); + CHECK(p.num_key_value_heads == 2); + CHECK(p.head_dim == 128); + CHECK(p.rope_theta == 10000.0); + CHECK(p.partial_rotary_factor == 1.0); + CHECK_FALSE(p.attention_bias); + CHECK_FALSE(p.sliding_window.has_value()); + CHECK(p.q_proj_out_features() == 4096); + CHECK(p.kv_proj_out_features() == 256); + + // Mamba2. conv_dim carries the 2*n_groups*state_size term; dropping it + // yields 4096 and the released conv1d.weight is [6144, 1, 4]. + CHECK(p.mamba_num_heads == 64); + CHECK(p.mamba_head_dim == 64); + CHECK(p.n_groups == 8); + CHECK(p.ssm_state_size == 128); + CHECK(p.conv_kernel == 4); + CHECK(p.chunk_size == 128); + CHECK(p.expand == 2); + CHECK(p.mamba_hidden_act == "silu"); + CHECK(p.mamba_ssm_cache_dtype == "float32"); + CHECK(p.use_conv_bias); + CHECK_FALSE(p.use_bias); + CHECK_FALSE(p.mamba_proj_bias); + CHECK(p.mamba_intermediate_size() == 4096); + CHECK(p.conv_dim() == 6144); + CHECK(p.in_proj_out_features() == 10304); + + // MoE. `routed_scaling_factor` is applied to the OUTPUT (nemotron_h.py:246). + CHECK(p.n_routed_experts == 128); + CHECK(p.num_experts_per_tok == 6); + CHECK(p.moe_intermediate_size == 1856); + CHECK(p.n_shared_experts == 1); + CHECK(p.moe_shared_expert_intermediate_size == 3712); + CHECK(p.n_group == 1); + CHECK(p.topk_group == 1); + CHECK(p.routed_scaling_factor == 2.5); + CHECK(p.norm_topk_prob); + CHECK(p.moe_shared_expert_overlap); + CHECK(p.mlp_hidden_act == "relu2"); + // `null` and ABSENT are the same state upstream (nemotron_h.py:143). + CHECK_FALSE(p.moe_latent_size.has_value()); + + // MTP: 1 predict layer over a 2-block ["attention","moe"] pattern. + CHECK(p.num_nextn_predict_layers == 1); + CHECK(p.mtp_layers_block_type == + std::vector{NemotronHBlock::kAttention, + NemotronHBlock::kMoe}); + + // The coarse quantization surface (per-module algo resolution is W1). + CHECK(p.quant.present); + CHECK(p.quant.quant_method == "modelopt"); + CHECK(p.quant.quant_algo == "MIXED_PRECISION"); + CHECK(p.quant.fp8_kv_cache); + CHECK(p.quant.mtp_ignored); +} + +TEST_CASE("NemotronH config: the LEGACY layout normalizes to the same params") { + nlohmann::json doc = FixtureConfigDoc(); + // Swap every modern key for the legacy alias transformers still accepts + // (configuration_nemotron_h.py:142-190). A layout that silently + // deserializes to all-defaults is a wrong-shaped model with no error. + const std::vector modern = + ParseNemotronHParams(LoadHfConfig(TempConfig(doc).path())) + .layers_block_type; + + std::string pattern; + for (NemotronHBlock b : modern) { + pattern += b == NemotronHBlock::kMamba ? 'M' + : b == NemotronHBlock::kMoe ? 'E' + : b == NemotronHBlock::kAttention ? '*' + : '-'; + } + doc.erase("layers_block_type"); + doc["hybrid_override_pattern"] = pattern; + doc.erase("mtp_layers_block_type"); + doc["mtp_hybrid_override_pattern"] = "*E"; + doc["mamba_n_groups"] = doc["n_groups"]; + doc.erase("n_groups"); + doc["mamba_d_conv"] = doc["conv_kernel"]; + doc.erase("conv_kernel"); + doc["mamba_expand"] = doc["expand"]; + doc.erase("expand"); + doc["mamba_chunk_size"] = doc["chunk_size"]; + doc.erase("chunk_size"); + doc["mamba_conv_bias"] = doc["use_conv_bias"]; + doc.erase("use_conv_bias"); + doc["mamba_dt_min"] = doc["time_step_min"]; + doc.erase("time_step_min"); + doc["mamba_dt_max"] = doc["time_step_max"]; + doc.erase("time_step_max"); + doc["mamba_dt_init_floor"] = doc["time_step_floor"]; + doc.erase("time_step_floor"); + + TempConfig cfg(doc); + const NemotronHParams p = ParseNemotronHParams(LoadHfConfig(cfg.path())); + CHECK(p.layers_block_type == modern); + CHECK(p.LayerIndices(NemotronHBlock::kAttention) == + std::vector{5, 12, 19, 26, 33, 42}); + CHECK(p.n_groups == 8); + CHECK(p.conv_kernel == 4); + CHECK(p.expand == 2); + CHECK(p.chunk_size == 128); + CHECK(p.use_conv_bias); + CHECK(p.time_step_min == 1e-3); + CHECK(p.time_step_max == 1e-1); + CHECK(p.time_step_floor == 1e-4); + CHECK(p.conv_dim() == 6144); + CHECK(p.mtp_layers_block_type == + std::vector{NemotronHBlock::kAttention, + NemotronHBlock::kMoe}); +} + +TEST_CASE("NemotronH config: absent schedules take upstream's defaults") { + nlohmann::json doc = FixtureConfigDoc(); + doc.erase("layers_block_type"); + doc.erase("mtp_layers_block_type"); + TempConfig cfg(doc); + const NemotronHParams p = ParseNemotronHParams(LoadHfConfig(cfg.path())); + // configuration_nemotron_h.py:165 and :180. + CHECK(p.layers_block_type == + std::vector{NemotronHBlock::kMamba, NemotronHBlock::kMoe, + NemotronHBlock::kAttention, + NemotronHBlock::kMlp}); + CHECK(p.num_hidden_layers() == 4); + CHECK(p.mtp_layers_block_type == + std::vector{NemotronHBlock::kAttention, + NemotronHBlock::kMoe}); +} + +TEST_CASE("NemotronH config: unrepresentable configs REFUSE BY NAME") { + SUBCASE("an unknown block type") { + nlohmann::json doc = FixtureConfigDoc(); + doc["layers_block_type"][0] = "swa"; + TempConfig cfg(doc); + CHECK_THROWS_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), + std::runtime_error); + } + SUBCASE("a latent MoE") { + nlohmann::json doc = FixtureConfigDoc(); + doc["moe_latent_size"] = 512; + TempConfig cfg(doc); + CHECK_THROWS_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), + std::runtime_error); + } + SUBCASE("a heterogeneous per-layer intermediate_size (NemotronHPuzzle)") { + nlohmann::json doc = FixtureConfigDoc(); + doc["intermediate_size"] = nlohmann::json::array({1856, 1856}); + TempConfig cfg(doc); + CHECK_THROWS_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), + std::runtime_error); + } + SUBCASE("a quantization method we do not resolve") { + nlohmann::json doc = FixtureConfigDoc(); + doc["quantization_config"]["quant_method"] = "awq"; + TempConfig cfg(doc); + CHECK_THROWS_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), + std::runtime_error); + } + SUBCASE("an ssm cache dtype we cannot represent") { + nlohmann::json doc = FixtureConfigDoc(); + doc["mamba_ssm_cache_dtype"] = "float8_e4m3"; + TempConfig cfg(doc); + CHECK_THROWS_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), + std::runtime_error); + } + SUBCASE("an MTP head with no MTP schedule") { + nlohmann::json doc = FixtureConfigDoc(); + doc["mtp_layers_block_type"] = nlohmann::json::array(); + TempConfig cfg(doc); + CHECK_THROWS_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), + std::runtime_error); + } +} + +TEST_CASE( + "NemotronH enumeration: all 18487 released tensors are claimed, none " + "invented") { + const NemotronHParams p = FixtureParams(); + const std::vector enumerated = EnumerateNemotronHTensors(p); + + const nlohmann::json index = ReadJson(FixtureDir() + "/index.json"); + CHECK(index.at("total_tensors").get() == 18487); + const auto released = ExpandIndexFixture(index); + REQUIRE(released.size() == 18487); + + // Every enumerated name is on disk, exactly once, with a named consumer. + std::set seen; + std::vector invented; + for (const NemotronHTensor& t : enumerated) { + CHECK_MESSAGE(!t.consumer.empty(), "no named consumer for " << t.name); + CHECK_MESSAGE(seen.insert(t.name).second, + "enumerated twice: " << t.name); + if (released.count(t.name) == 0) invented.push_back(t.name); + } + CHECK_MESSAGE(invented.empty(), + "enumerated tensors the checkpoint does not ship, first: " + << (invented.empty() ? std::string("-") : invented.front()) + << " (" << invented.size() << " total)"); + + // ...and every tensor on disk is claimed. Zero unaccounted. + std::vector unaccounted; + for (const auto& [name, meta] : released) { + (void)meta; + if (seen.count(name) == 0) unaccounted.push_back(name); + } + CHECK_MESSAGE(unaccounted.empty(), + "UNCLAIMED checkpoint tensors, first: " + << (unaccounted.empty() ? std::string("-") + : unaccounted.front()) + << " (" << unaccounted.size() << " total)"); + CHECK(enumerated.size() == 18487); +} + +TEST_CASE("NemotronH enumeration: the shapes it implies are the ones on disk") { + // The three geometry facts W3 owns, checked against the RELEASED headers + // rather than against our own arithmetic. + const NemotronHParams p = FixtureParams(); + const auto released = ExpandIndexFixture(ReadJson(FixtureDir() + "/index.json")); + + const auto shape_of = [&](const std::string& name) { + const auto it = released.find(name); + REQUIRE_MESSAGE(it != released.end(), "absent from the index: " << name); + return it->second.second; + }; + + // conv_dim = intermediate + 2*n_groups*state_size. Without the second term + // this is 4096. + CHECK(shape_of("backbone.layers.0.mixer.conv1d.weight") == + std::vector{p.conv_dim(), 1, p.conv_kernel}); + CHECK(shape_of("backbone.layers.0.mixer.conv1d.bias") == + std::vector{p.conv_dim()}); + // in_proj = z + xBC + dt. + CHECK(shape_of("backbone.layers.0.mixer.in_proj.weight") == + std::vector{p.in_proj_out_features(), p.hidden_size}); + CHECK(shape_of("backbone.layers.0.mixer.out_proj.weight") == + std::vector{p.hidden_size, p.mamba_intermediate_size()}); + CHECK(shape_of("backbone.layers.0.mixer.A_log") == + std::vector{p.mamba_num_heads}); + // GQA: 32 q heads, 2 kv heads, head_dim 128. + CHECK(shape_of("backbone.layers.5.mixer.q_proj.weight") == + std::vector{p.q_proj_out_features(), p.hidden_size}); + CHECK(shape_of("backbone.layers.5.mixer.k_proj.weight") == + std::vector{p.kv_proj_out_features(), p.hidden_size}); + // NVFP4 W4A16 group_size 16: the packed weight halves the input dim and the + // per-block e4m3 scale divides it by 16. + const auto up = shape_of("backbone.layers.1.mixer.experts.0.up_proj.weight"); + const auto up_scale = + shape_of("backbone.layers.1.mixer.experts.0.up_proj.weight_scale"); + CHECK(up == std::vector{p.moe_intermediate_size, p.hidden_size / 2}); + CHECK(up_scale == + std::vector{p.moe_intermediate_size, p.hidden_size / 16}); + CHECK(shape_of("backbone.layers.1.mixer.gate.weight") == + std::vector{p.n_routed_experts, p.hidden_size}); + // The MTP fusion projection consumes [embed ; hidden]. + CHECK(shape_of("mtp.layers.0.eh_proj.weight") == + std::vector{p.hidden_size, 2 * p.hidden_size}); +} + +TEST_CASE("NemotronH KV: the het groups mirror mamba2_state_shape") { + TempConfig cfg(FixtureConfigDoc()); + const HfConfig config = LoadHfConfig(cfg.path()); + const NemotronHParams p = ParseNemotronHParams(config); + const vllm::v1::KVCacheConfig kv = MakeNemotronHKVCache(config, 16, 4); + + REQUIRE(kv.kv_cache_groups.size() == 2); + CHECK(kv.num_blocks == 4); + CHECK(kv.has_mamba_layers()); + + // (1) the full-attention group covers exactly the 6 attention layers. + const auto& attn_group = kv.kv_cache_groups[0]; + const auto* attn = dynamic_cast( + attn_group.kv_cache_spec.get()); + REQUIRE(attn != nullptr); + CHECK(attn->num_kv_heads == 2); + CHECK(attn->head_size == 128); + CHECK(attn_group.layer_names.size() == 6); + std::vector expected_attn; + for (int64_t i : p.LayerIndices(NemotronHBlock::kAttention)) { + expected_attn.push_back("backbone.layers." + std::to_string(i) + ".mixer"); + } + CHECK(attn_group.layer_names == expected_attn); + + // (2) the Mamba2 group covers exactly the 23 mamba layers, at the upstream + // state shapes and with the SSM dtype resolved INDEPENDENTLY of the conv + // dtype (mamba_ssm_cache_dtype: "float32"). + const auto& mamba_group = kv.kv_cache_groups[1]; + const vllm::v1::MambaSpec& mamba = MambaGroup(kv); + CHECK(mamba.shapes == + std::vector>{{6144, 3}, {64, 64, 128}}); + CHECK(mamba.dtypes == + std::vector{vt::DType::kBF16, vt::DType::kF32}); + CHECK(mamba_group.layer_names.size() == 23); + std::vector expected_mamba; + for (int64_t i : p.LayerIndices(NemotronHBlock::kMamba)) { + expected_mamba.push_back("backbone.layers." + std::to_string(i) + ".mixer"); + } + CHECK(mamba_group.layer_names == expected_mamba); + + // The 23x / 6x multipliers are the whole point of carrying real names. + const int64_t conv_bytes = 6144 * 3 * 2; // bf16 + const int64_t ssm_bytes = 64 * 64 * 128 * 4; // f32 + CHECK(mamba.page_size_bytes() == conv_bytes + ssm_bytes); + CHECK(vllm::v1::KVBytesPerBlock(kv) == attn->page_size_bytes() * 6); +} + +TEST_CASE("NemotronH: the unported arms REFUSE BY NAME") { + TempConfig cfg(FixtureConfigDoc()); + const HfConfig config = LoadHfConfig(cfg.path()); + const vllm::ModelRegistration& reg = ModelRegistry::Resolve(config); + + SUBCASE("GGUF k-quants are owed (W7), never silently dequantized") { + vllm::ModelSource source; + source.kind = vllm::ModelSource::Kind::kGguf; + CHECK_THROWS_WITH_AS(reg.factory->load_weights(reg, config, source), + doctest::Contains("NemotronHForCausalLM"), + std::runtime_error); + } +} + +TEST_CASE("NemotronH: the committed fixture matches the LIVE checkpoint") { + const std::string dir = parity::Nemotron35LightningSnapshot(); + if (dir.empty()) { + MESSAGE( + "SKIP: set VT_NEMOTRON35_SNAPSHOT to " + "$CHECKPOINT_ROOT/nemotron-3.5-lightning-30b-nvfp4 (revision " + "29f2d1746d8f41e316523194b19018707749b1b1) to re-verify the fixture"); + return; + } + + // config.json: identical key-by-key, and the ELIDED set is EXACTLY the two + // 5981-entry ModelOpt maps W1 owns. + const nlohmann::json live = ReadJson(dir + "/config.json"); + const nlohmann::json fixture = FixtureConfigDoc(); + for (const auto& [key, value] : live.items()) { + if (key == "quantization_config") continue; + CHECK_MESSAGE(fixture.contains(key), "fixture is missing key " << key); + if (fixture.contains(key)) { + CHECK_MESSAGE(fixture.at(key) == value, "fixture key drifted: " << key); + } + } + for (const auto& [key, value] : fixture.items()) { + (void)value; + CHECK_MESSAGE(live.contains(key), "fixture invented key " << key); + } + std::set elided; + for (const auto& [key, value] : live.at("quantization_config").items()) { + if (!fixture.at("quantization_config").contains(key)) { + elided.insert(key); + continue; + } + CHECK_MESSAGE(fixture.at("quantization_config").at(key) == value, + "fixture quantization_config key drifted: " << key); + } + CHECK(elided == std::set{"config_groups", "quantized_layers"}); + + // index.json: the expanded fixture is the live header set, both directions. + const auto expanded = ExpandIndexFixture(ReadJson(FixtureDir() + "/index.json")); + const auto live_headers = ReadLiveHeaders(dir); + CHECK(live_headers.size() == 18487); + CHECK(expanded.size() == live_headers.size()); + std::vector drifted; + for (const auto& [name, meta] : live_headers) { + const auto it = expanded.find(name); + if (it == expanded.end() || it->second != meta) drifted.push_back(name); + } + CHECK_MESSAGE(drifted.empty(), + "fixture drifted from the live checkpoint, first: " + << (drifted.empty() ? std::string("-") : drifted.front()) + << " (" << drifted.size() << " total)"); +} diff --git a/tests/vllm/test_model_loader_gguf.cpp b/tests/vllm/test_model_loader_gguf.cpp index 5874d450a..becac1d30 100644 --- a/tests/vllm/test_model_loader_gguf.cpp +++ b/tests/vllm/test_model_loader_gguf.cpp @@ -84,6 +84,7 @@ TEST_CASE("FromModelDir rejects an unknown dense architecture before loading") { "'LagunaForCausalLM', " "'LlamaForCausalLM', 'LlamaModel', " "'MiniCPM3ForCausalLM', 'MiniCPMForCausalLM', 'MistralForCausalLM', 'MuseGlimmerForCausalLM', 'MuseGlimmerForConditionalGeneration', " + "'NemotronHForCausalLM', " "'OPTForCausalLM', 'Olmo2ForCausalLM', 'Olmo3ForCausalLM', " "'ParakeetForCTC', 'ParakeetForRNNT', 'ParakeetForTDT', " "'Phi3ForCausalLM', 'PhiForCausalLM', 'Qwen3ForCausalLM', " From 3981de6a4ef33c90361aaa0f63f06ecb1c90207d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 13 Aug 2026 02:06:36 +0000 Subject: [PATCH 3/5] fix(MODEL-NEMOTRON-H W3): ClaimMamba ignored `quantized`, and the legacy-alias precedence was INVERTED (#517) Three review findings on `row/MODEL-NEMOTRON-H-W3C` @ `3295d0c1a` (PR #565). The verdict was PASS; this is the tightening pass. 1. `ClaimMamba` hard-coded its FP8 companions. Every other claimer (`ClaimNvfp4`, `ClaimMoe`, `ClaimMlp`, `ClaimAttention`'s `fp8_kv`) gates on `quantized`; `ClaimFp8` did not. The released config MINUS `quantization_config` -- the shape a released bf16 NemotronH safetensors checkpoint actually ships, which spec 5b already owes -- parsed without refusal and enumerated 92 tensors that checkpoint does not have (23 mamba blocks x 2 projections x 2 companions), first `backbone.layers.0.mixer.in_proj.weight_scale`. Nothing consumes the map yet, so no gated claim was wrong -- but an unimplemented arm is refused BY NAME, never silently mis-enumerated. `quantized` is now threaded through `ClaimFp8`/`ClaimMamba` and both call sites (backbone `quantized`, MTP `mtp_quantized`). The released checkpoint is quantized and its MTP schedule is {attention, moe}, so the 18487-tensor gate is unchanged. 2. `Get{Int,Double,Bool}Aliased` preferred the MODERN key. Upstream does the OPPOSITE for the `mamba_*` scalars: `configuration_nemotron_h.py:145-155` is `self.n_groups = kwargs.pop("mamba_n_groups") if "mamba_n_groups" in kwargs else self.n_groups`, which OVERWRITES an already-populated dataclass field, so LEGACY wins. Re-derived by RUNNING transformers @ 7d06b1a5, not by reading it: `NemotronHConfig(n_groups=8, mamba_n_groups=4, conv_kernel=4, mamba_d_conv=7)` -> `n_groups=4, conv_kernel=7`. The SCHEDULE pair is genuinely modern-wins (`:158-165` consults the pattern only when the list is None; `NemotronHConfig(layer_types=['mamba','mamba'], hybrid_override_pattern='*-')` -> `['mamba','mamba']`), and is left alone. Upstream disagrees with itself between the two families; both polarities are now mirrored and both are stated explicitly where they are implemented. Worse than the behavior was the RECORD: the comment asserted "the modern key wins" as if it were upstream's rule, which is what would mislead the next porter. No released checkpoint ships both spellings of one field, so this was a mirroring defect, not a live one -- which is exactly why it needed a test. 3. "the unported ARMS refuse by name" had one SUBCASE (GGUF). `ForwardNemotronHForCausalLM` is an unconditional `VT_CHECK`, which throws `std::runtime_error`, so it is directly callable with a stub `LoadedModel`. Now asserted, including that the message names W4 and the spec. 4. `NemotronHBlockName` had zero call sites. `BlockFromName` now maps both directions through it and builds its refusal's expected-list from the enum, so a fifth block kind cannot arrive with a message that still lists four. RED first for 1 and 2 on the pre-fix tree: 12 cases, 9 assertions failed -- `companions.empty()` reporting 92 with that exact first name, and `8 == 4`/`4 == 7`/`2 == 9`/`128 == 77`/`true`/`0.001 == 0.5`/`0.1 == 0.6`/ `0.0001 == 0.7`. GREEN after: 12 cases / 38284 assertions / 0 failed, `Status: SUCCESS!` (baseline was 10 / 38245). FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [Claude Code] --- src/vllm/model_executor/models/nemotron_h.h | 9 + .../models/nemotron_h_weights.cpp | 94 ++++++--- .../vllm/models/test_nemotron_h_scaffold.cpp | 186 ++++++++++++++++++ 3 files changed, 267 insertions(+), 22 deletions(-) diff --git a/src/vllm/model_executor/models/nemotron_h.h b/src/vllm/model_executor/models/nemotron_h.h index dac7c35a7..e926640c3 100644 --- a/src/vllm/model_executor/models/nemotron_h.h +++ b/src/vllm/model_executor/models/nemotron_h.h @@ -77,6 +77,7 @@ #include #include #include +#include #include #include "vllm/model_executor/models/model_registry.h" @@ -152,6 +153,14 @@ struct NemotronHParams { std::optional sliding_window = std::nullopt; // --- Mamba2 --- + // Each `legacy alias` below WINS over the modern spelling when both ship — + // `self.n_groups = kwargs.pop("mamba_n_groups") if "mamba_n_groups" in kwargs + // else self.n_groups` (configuration_nemotron_h.py:145-155) overwrites the + // already-populated dataclass field. That is the OPPOSITE of the two SCHEDULE + // pairs (`layers_block_type`/`hybrid_override_pattern` and the `mtp_` pair, + // :158-184), where the legacy pattern is consulted only when the modern list + // is absent. Upstream disagrees with itself between the two families; both + // polarities are mirrored as-is and pinned by the "PER-FAMILY" test case. int64_t mamba_num_heads = 0; // 64 int64_t mamba_head_dim = 0; // 64 int64_t n_groups = 0; // 8 (legacy alias `mamba_n_groups`) diff --git a/src/vllm/model_executor/models/nemotron_h_weights.cpp b/src/vllm/model_executor/models/nemotron_h_weights.cpp index b99d74e43..ae8a9d81b 100644 --- a/src/vllm/model_executor/models/nemotron_h_weights.cpp +++ b/src/vllm/model_executor/models/nemotron_h_weights.cpp @@ -63,35 +63,63 @@ std::string GetString(const json& doc, const char* key, return v.get(); } -// The legacy-alias reads of configuration_nemotron_h.py:142-155: the modern key -// wins, the `mamba_*` alias fills in, and a checkpoint carrying only the alias -// must not silently deserialize to the class default. +// The legacy-alias reads of the `mamba_*` SCALARS +// (configuration_nemotron_h.py:145-155). Upstream is: +// +// self.n_groups = kwargs.pop("mamba_n_groups") if "mamba_n_groups" in kwargs +// else self.n_groups +// +// The dataclass field already holds the modern value (or the class default) by +// the time `__post_init__` runs, and the legacy alias OVERWRITES it whenever it +// is present. So the precedence is LEGACY > modern > class default — the +// opposite of what reads naturally, and the opposite of the SCHEDULE pair below. +// +// Verified by RUNNING transformers @ 7d06b1a5 rather than by reading it: +// NemotronHConfig(n_groups=8, mamba_n_groups=4, +// conv_kernel=4, mamba_d_conv=7) -> n_groups=4, conv_kernel=7 +// +// A checkpoint carrying only the alias must likewise not silently deserialize +// to the class default. No released checkpoint ships both spellings of one +// field, so this is a mirroring obligation, not a live defect — pinned by +// "when BOTH spellings ship, the precedence is upstream's and it is PER-FAMILY" +// so it cannot drift back. int64_t GetIntAliased(const json& doc, const char* key, const char* legacy, int64_t fallback) { - if (Has(doc, key)) return GetInt(doc, key, fallback); - return GetInt(doc, legacy, fallback); + if (Has(doc, legacy)) return GetInt(doc, legacy, fallback); + return GetInt(doc, key, fallback); } double GetDoubleAliased(const json& doc, const char* key, const char* legacy, double fallback) { - if (Has(doc, key)) return GetDouble(doc, key, fallback); - return GetDouble(doc, legacy, fallback); + if (Has(doc, legacy)) return GetDouble(doc, legacy, fallback); + return GetDouble(doc, key, fallback); } bool GetBoolAliased(const json& doc, const char* key, const char* legacy, bool fallback) { - if (Has(doc, key)) return GetBool(doc, key, fallback); - return GetBool(doc, legacy, fallback); + if (Has(doc, legacy)) return GetBool(doc, legacy, fallback); + return GetBool(doc, key, fallback); } +// The four block spellings, in enum order. The single source of truth for both +// directions of the name<->enum map, so a fifth block kind cannot be added with +// a refusal message that still lists four. +constexpr NemotronHBlock kAllBlocks[] = { + NemotronHBlock::kMamba, NemotronHBlock::kAttention, NemotronHBlock::kMoe, + NemotronHBlock::kMlp}; + NemotronHBlock BlockFromName(const std::string& name) { - if (name == "mamba") return NemotronHBlock::kMamba; - if (name == "attention") return NemotronHBlock::kAttention; - if (name == "moe") return NemotronHBlock::kMoe; - if (name == "mlp") return NemotronHBlock::kMlp; + for (NemotronHBlock block : kAllBlocks) { + if (name == NemotronHBlockName(block)) return block; + } // Mirror of validate_layer_type (configuration_nemotron_h.py:195-204). + std::string expected; + for (NemotronHBlock block : kAllBlocks) { + if (!expected.empty()) expected += ", "; + expected += NemotronHBlockName(block); + } Refuse("layers_block_type contains the unsupported block type '" + name + - "' (expected one of mamba, attention, moe, mlp)"); + "' (expected one of " + expected + ")"); } NemotronHBlock BlockFromPatternChar(char c) { @@ -114,6 +142,20 @@ NemotronHBlock BlockFromPatternChar(char c) { // One layer schedule, resolved with upstream's precedence: the explicit list, // else the legacy pattern string, else the class default. `list_key` / // `pattern_key` are the modern/legacy pair, `fallback` the class default. +// +// NOTE the polarity, which is the OPPOSITE of the `mamba_*` scalars above and +// is deliberate on both sides. configuration_nemotron_h.py:158-165: +// +// if "hybrid_override_pattern" in kwargs: +// pattern = kwargs.pop("hybrid_override_pattern") +// if self.layer_types is None: +// self.layer_types = self._pattern_to_list(pattern) +// +// the legacy pattern is consulted ONLY when the modern list is absent, so here +// MODERN wins; :176-184 does the same for the MTP pair. Verified by running +// transformers @ 7d06b1a5: `NemotronHConfig(layer_types=['mamba','mamba'], +// hybrid_override_pattern='*-')` -> `['mamba','mamba']`. Do not "unify" the two +// families — upstream genuinely disagrees with itself here. std::vector ResolveSchedule( const json& doc, const char* list_key, const char* pattern_key, const std::vector& fallback) { @@ -204,19 +246,27 @@ void ClaimNvfp4(std::vector& out, const std::string& prefix, // An FP8 W8A8 static-scaled projection: the e4m3 weight, its fp32 weight scale // and the fp32 static input scale (config_groups group_0, 46 targets). +// `quantized` gates the companions exactly as `ClaimNvfp4` does: an UNQUANTIZED +// producer ships the bare bf16 weight and no scales at all. void ClaimFp8(std::vector& out, const std::string& prefix, - const std::string& consumer) { + const std::string& consumer, bool quantized) { Claim(out, prefix + ".weight", consumer); + if (!quantized) return; Claim(out, prefix + ".weight_scale", consumer + ".weight_scale[fp8]"); Claim(out, prefix + ".input_scale", consumer + ".input_scale[fp8]"); } // One Mamba2 mixer (MambaMixer2, nemotron_h.py:373-389). `use_conv_bias` and -// `mamba_proj_bias` gate the two optional biases exactly as upstream does. +// `mamba_proj_bias` gate the two optional biases exactly as upstream does, and +// `quantized` gates the in/out projection scale companions. Released bf16 +// NemotronH safetensors checkpoints ship NO `quantization_config` and no +// `mixer.{in,out}_proj.{weight_scale,input_scale}`; hard-coding the FP8 pair +// here enumerated 92 tensors such a checkpoint does not have (23 mamba blocks x +// 2 projections x 2 companions). void ClaimMamba(std::vector& out, const NemotronHParams& p, - const std::string& mixer) { - ClaimFp8(out, mixer + ".in_proj", "mamba2.in_proj"); - ClaimFp8(out, mixer + ".out_proj", "mamba2.out_proj"); + const std::string& mixer, bool quantized) { + ClaimFp8(out, mixer + ".in_proj", "mamba2.in_proj", quantized); + ClaimFp8(out, mixer + ".out_proj", "mamba2.out_proj", quantized); if (p.mamba_proj_bias) { Claim(out, mixer + ".in_proj.bias", "mamba2.in_proj.bias"); Claim(out, mixer + ".out_proj.bias", "mamba2.out_proj.bias"); @@ -367,7 +417,7 @@ NemotronHParams ParseNemotronHParams(const HfConfig& config) { p.sliding_window = GetInt(doc, "sliding_window", 0); } - // --- Mamba2 (legacy aliases normalized, configuration_nemotron_h.py:142-155) --- + // --- Mamba2 (legacy aliases WIN, configuration_nemotron_h.py:145-155) --- p.mamba_num_heads = GetInt(doc, "mamba_num_heads", 128); p.mamba_head_dim = GetInt(doc, "mamba_head_dim", 64); p.n_groups = GetIntAliased(doc, "n_groups", "mamba_n_groups", 8); @@ -496,7 +546,7 @@ std::vector EnumerateNemotronHTensors( Claim(out, layer + ".norm.weight", "layer_norm"); switch (p.layers_block_type[i]) { case NemotronHBlock::kMamba: - ClaimMamba(out, p, mixer); + ClaimMamba(out, p, mixer, quantized); break; case NemotronHBlock::kAttention: ClaimAttention(out, p, mixer, quantized && p.quant.fp8_kv_cache); @@ -540,7 +590,7 @@ std::vector EnumerateNemotronHTensors( ClaimMoe(out, p, mixer, mtp_quantized); break; case NemotronHBlock::kMamba: - ClaimMamba(out, p, mixer); + ClaimMamba(out, p, mixer, mtp_quantized); break; case NemotronHBlock::kMlp: ClaimMlp(out, p, mixer, mtp_quantized); diff --git a/tests/vllm/models/test_nemotron_h_scaffold.cpp b/tests/vllm/models/test_nemotron_h_scaffold.cpp index 5a06229af..6165a173f 100644 --- a/tests/vllm/models/test_nemotron_h_scaffold.cpp +++ b/tests/vllm/models/test_nemotron_h_scaffold.cpp @@ -44,8 +44,16 @@ #include "hf_snapshot.h" #include "vllm/model_executor/models/model_registry.h" +// The forward-refusal SUBCASE has to CALL the type-erased forward, so it needs +// the concrete definitions of the seam types `model_registry.h` only forward- +// declares. `nemotron_h_registry.cpp:28` reaches for the same header for the +// same reason. +#include "vllm/model_executor/models/qwen3_5.h" // ForwardLogits, *KvCache #include "vllm/transformers_utils/hf_config.h" +#include "vllm/v1/attention/backend.h" // CommonAttentionMetadata +#include "vllm/v1/attention/backends/gdn_attn.h" // GDNAttentionMetadata #include "vllm/v1/kv_cache_interface.h" +#include "vt/device.h" #include "vt/dtype.h" using vllm::EnumerateNemotronHTensors; @@ -346,6 +354,25 @@ TEST_CASE("NemotronH config: unrepresentable configs REFUSE BY NAME") { TempConfig cfg(doc); CHECK_THROWS_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), std::runtime_error); + // The refusal NAMES the offender and enumerates the four block spellings + // from `NemotronHBlockName`, so it cannot list four kinds after a fifth is + // added. `doctest::Contains` takes a `const char*` LITERAL here on purpose: + // doctest 2.5.2 stringifies a `const char*` VARIABLE as `1`. + CHECK_THROWS_WITH_AS(ParseNemotronHParams(LoadHfConfig(cfg.path())), + doctest::Contains("'swa'"), std::runtime_error); + for (NemotronHBlock block : {NemotronHBlock::kMamba, + NemotronHBlock::kAttention, + NemotronHBlock::kMoe, NemotronHBlock::kMlp}) { + const std::string name(vllm::NemotronHBlockName(block)); + CHECK_FALSE(name.empty()); + CHECK(name != "unknown"); + // Round-trip: every spelling the refusal offers must actually PARSE. + nlohmann::json ok = FixtureConfigDoc(); + ok["layers_block_type"][0] = name; + TempConfig ok_cfg(ok); + const NemotronHParams p = ParseNemotronHParams(LoadHfConfig(ok_cfg.path())); + CHECK(p.layers_block_type.at(0) == block); + } } SUBCASE("a latent MoE") { nlohmann::json doc = FixtureConfigDoc(); @@ -423,6 +450,126 @@ TEST_CASE( CHECK(enumerated.size() == 18487); } +TEST_CASE( + "NemotronH enumeration: an UNQUANTIZED producer claims NO scale " + "companions") { + // The released checkpoint is ModelOpt-quantized, so every `quantized` flag is + // true there and the 18487-tensor gate above cannot see a claimer that IGNORES + // the flag. Drop `quantization_config` — the shape a released bf16 NemotronH + // safetensors checkpoint actually ships (spec §5b) — and the scale companions + // must all disappear. A claimer that hard-codes its FP8/NVFP4 companions + // enumerates tensors that do not exist, which is the silent mis-enumeration + // AGENTS.md forbids ("an arm that is not implemented is refused by name"). + nlohmann::json doc = FixtureConfigDoc(); + doc.erase("quantization_config"); + TempConfig cfg(doc); + const NemotronHParams p = ParseNemotronHParams(LoadHfConfig(cfg.path())); + REQUIRE_FALSE(p.quant.present); + REQUIRE_FALSE(p.quant.fp8_kv_cache); + + const std::vector enumerated = EnumerateNemotronHTensors(p); + const auto ends_with = [](const std::string& s, const std::string& suffix) { + return s.size() >= suffix.size() && + s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; + }; + std::vector companions; + for (const NemotronHTensor& t : enumerated) { + if (ends_with(t.name, ".weight_scale") || + ends_with(t.name, ".weight_scale_2") || + ends_with(t.name, ".input_scale") || ends_with(t.name, ".k_scale") || + ends_with(t.name, ".v_scale")) { + companions.push_back(t.name); + } + } + CHECK_MESSAGE(companions.empty(), + "an unquantized producer still claims scale companions, first: " + << (companions.empty() ? std::string("-") + : companions.front()) + << " (" << companions.size() << " total)"); + + // The quantized arm is unchanged: the released config still claims all 18487. + const std::vector quantized = + EnumerateNemotronHTensors(FixtureParams()); + CHECK(quantized.size() == 18487); + // The bf16 arm is the same MODEL with the companions removed, nothing else: + // every unquantized name is also claimed by the quantized arm. + std::set quantized_names; + for (const NemotronHTensor& t : quantized) quantized_names.insert(t.name); + std::vector only_bf16; + for (const NemotronHTensor& t : enumerated) { + if (quantized_names.count(t.name) == 0) only_bf16.push_back(t.name); + } + CHECK_MESSAGE(only_bf16.empty(), + "the bf16 arm invented a name the quantized arm never claims, " + "first: " + << (only_bf16.empty() ? std::string("-") + : only_bf16.front())); +} + +TEST_CASE( + "NemotronH config: when BOTH spellings ship, the precedence is upstream's " + "and it is PER-FAMILY") { + // Re-derived by RUNNING transformers @ 7d06b1a5 (the pin this file's header + // names), not by reading it: + // NemotronHConfig(n_groups=8, mamba_n_groups=4, conv_kernel=4, + // mamba_d_conv=7) -> n_groups=4, conv_kernel=7 + // NemotronHConfig(layer_types=['mamba','mamba'], + // hybrid_override_pattern='*-') -> ['mamba','mamba'] + // The two families genuinely DISAGREE, and each is mirrored on its own terms: + // + // mamba_* SCALARS (configuration_nemotron_h.py:145-155) — LEGACY wins. + // `self.n_groups = kwargs.pop("mamba_n_groups") if "mamba_n_groups" in + // kwargs else self.n_groups`: the dataclass field already holds the modern + // value, and the legacy alias OVERWRITES it unconditionally. + // + // SCHEDULES (configuration_nemotron_h.py:158-165, :176-184) — MODERN wins. + // `if "hybrid_override_pattern" in kwargs: ... if self.layer_types is + // None: self.layer_types = _pattern_to_list(pattern)`: the legacy pattern + // is consulted ONLY when the modern list is absent. + // + // No released checkpoint ships both spellings of the same field, so this is a + // mirroring obligation rather than a live defect — which is exactly why it + // needs a test: nothing else can catch it drifting. + nlohmann::json doc = FixtureConfigDoc(); + const std::vector modern_schedule = + ParseNemotronHParams(LoadHfConfig(TempConfig(doc).path())) + .layers_block_type; + + // Every mamba_* scalar gets a legacy alias that DISAGREES with the modern key + // the fixture already ships. + doc["mamba_n_groups"] = 4; // modern `n_groups` is 8 + doc["mamba_d_conv"] = 7; // modern `conv_kernel` is 4 + doc["mamba_expand"] = 9; // modern `expand` is 2 + doc["mamba_chunk_size"] = 77; // modern `chunk_size` is 128 + doc["mamba_conv_bias"] = false; // modern `use_conv_bias` is true + doc["mamba_dt_min"] = 0.5; // modern `time_step_min` is 1e-3 + doc["mamba_dt_max"] = 0.6; // modern `time_step_max` is 1e-1 + doc["mamba_dt_init_floor"] = 0.7; // modern `time_step_floor` is 1e-4 + // ...and both schedules get a legacy pattern that disagrees too. + doc["hybrid_override_pattern"] = "*-"; + doc["mtp_hybrid_override_pattern"] = "M-"; + + TempConfig cfg(doc); + const NemotronHParams p = ParseNemotronHParams(LoadHfConfig(cfg.path())); + + // LEGACY wins for the scalars. + CHECK(p.n_groups == 4); + CHECK(p.conv_kernel == 7); + CHECK(p.expand == 9); + CHECK(p.chunk_size == 77); + CHECK_FALSE(p.use_conv_bias); + CHECK(p.time_step_min == 0.5); + CHECK(p.time_step_max == 0.6); + CHECK(p.time_step_floor == 0.7); + + // MODERN wins for the schedules — do NOT "unify" these with the scalars. + CHECK(p.layers_block_type == modern_schedule); + CHECK(p.layers_block_type.size() == 52); + CHECK(p.mtp_layers_block_type == + std::vector{NemotronHBlock::kAttention, + NemotronHBlock::kMoe}); +} + TEST_CASE("NemotronH enumeration: the shapes it implies are the ones on disk") { // The three geometry facts W3 owns, checked against the RELEASED headers // rather than against our own arithmetic. @@ -527,6 +674,45 @@ TEST_CASE("NemotronH: the unported arms REFUSE BY NAME") { doctest::Contains("NemotronHForCausalLM"), std::runtime_error); } + + SUBCASE("the forward is W4's, and REFUSES rather than returning zeros") { + // The case title says "arms", plural, but until now only the GGUF arm was + // exercised. `ForwardNemotronHForCausalLM` is an unconditional VT_CHECK, + // which throws std::runtime_error (vt/dtype.h:11-17), so it is directly + // callable: a stub LoadedModel is enough because the forward consumes + // neither the model nor the input. A forward that silently returned `{}` + // would produce zero logits and a plausible-looking garbage token. + struct StubModel : vllm::LoadedModel { + explicit StubModel(const vllm::ModelRegistration& r) : LoadedModel(r) {} + }; + StubModel model(reg); + const std::vector token_ids{0}; + const std::vector positions{0}; + const std::vector logits_indices{0}; + const vllm::v1::CommonAttentionMetadata attn_meta{}; + const vllm::v1::GDNAttentionMetadata gdn_meta{}; + std::vector attn_kv; + std::vector gdn_state; + vt::Queue queue{vt::Device{vt::DeviceType::kCPU, 0}, nullptr}; + const vllm::ModelForwardInput input{.token_ids = token_ids, + .positions = positions, + .attn_meta = attn_meta, + .gdn_meta = gdn_meta, + .attn_kv = attn_kv, + .gdn_state = gdn_state, + .config = config, + .queue = queue, + .logits_indices = logits_indices, + .num_reqs = 1}; + // The message must NAME the missing piece, not just fail. + CHECK_THROWS_WITH_AS(reg.factory->forward(model, input), + doctest::Contains("NemotronHForCausalLM forward is " + "not implemented yet"), + std::runtime_error); + CHECK_THROWS_WITH_AS(reg.factory->forward(model, input), + doctest::Contains("nemotron-h-model.md"), + std::runtime_error); + } } TEST_CASE("NemotronH: the committed fixture matches the LIVE checkpoint") { From 9e111f9027b614ec43f85a8fadaa09b367309a02 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 13 Aug 2026 02:23:44 +0000 Subject: [PATCH 4/5] spec(MODEL-NEMOTRON-H): 5d records the W3 repair, the 15-mutation sweep, and two residuals (#517) Records what neither the code nor git carries: which of the fresh review's findings were fixed and how, the two report-only items, and the full mutation proof re-run on the repaired tree. Also CORRECTS 5c. Its reconnaissance item 1 said both spellings are accepted "the modern one winning" as an unqualified rule. That is true of the SCHEDULE pair only. For the `mamba_*` scalars upstream is the OPPOSITE (configuration_nemotron_h.py:145-155 overwrites the already-populated field), re-derived by RUNNING transformers @ 7d06b1a5. A record that states the opposite of upstream is the part that misleads the next porter, so the generalization is scoped and 5d carries the evidence. Residuals recorded, NOT fixed here: - docs/FEATURES.md:171 says "27 of the 32 registered text-generation architectures" while :173 implies 38-3-1 = 34 and test_model_registry.cpp:47 says "34 text archs". W3's +1 was right; the BASE was already stale and no checker validates it. This branch has no docs/ authority, and repairing a pre-existing doc drift inside a scoped repair would hide it. - Fixture DTYPE drift is invisible offline by design: names and shapes are pinned in the committed projection, dtypes only by the live case, and CI has no checkpoint. Named so W4/W6 do not rediscover it. 5b's safetensors-bf16 row now says its enumeration is correct. Gate evidence, disk 68G free (85%) throughout: Release -Werror full build 0 warnings; scaffold 12/12 cases 38284/38284 offline and 12/12 39152/39152 live; Debug (-g0, asserts unmasked) identical on both; full ctest 100% passed, 0 failed out of 403. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [Claude Code] --- .agents/specs/nemotron-h-model.md | 140 +++++++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 2 deletions(-) diff --git a/.agents/specs/nemotron-h-model.md b/.agents/specs/nemotron-h-model.md index df0ff9d96..de6c0cffa 100644 --- a/.agents/specs/nemotron-h-model.md +++ b/.agents/specs/nemotron-h-model.md @@ -486,7 +486,7 @@ For this row that means: | Arm | State | |---|---| | ModelOpt NVFP4 W4A16 g16 + FP8 W8A8 (the shipped checkpoint) | W1-W6, the critical path | -| Safetensors bf16 | reachable via the same loader; owed a fixture | +| Safetensors bf16 | reachable via the same loader; owed a fixture. Its ENUMERATION is now correct (§5d finding 1): a producer with no `quantization_config` claims the bare weights and none of the 92 FP8/NVFP4 scale companions | | **GGUF k-quants / i-quants** through the shared GGUF loader | **OWED.** No NemotronH GGUF arm exists | Until the GGUF arm lands it is **refused by name** at load, naming the missing @@ -521,7 +521,8 @@ and disk:** property over `layers_block_type` whose setter discards the checkpoint's value (`configuration_nemotron_h.py:225-238`). §5a's live oracle run is what settles it on the real checkpoint. Both spellings are accepted here, the - modern one winning. + modern one winning — **for the SCHEDULE pair only**; see §5d, which corrects + the generalization this sentence originally made. 2. **`moe_latent_size`: absent and `null` are the SAME state.** Upstream's predicate is `getattr(config, "moe_latent_size", None) is not None` (`nemotron_h.py:143`), so a missing key and an explicit `null` both mean "no @@ -601,6 +602,141 @@ muse-glimmer precedent already used. **Not done here:** the forward (W4), the MTP head (W5), the e2e token gate (W6), the GGUF arm (W7). The row stays `INVENTORIED`. +## 5d. W3 repair — the review's four findings, and two residuals (2026-08-13) + +The fresh review of `row/MODEL-NEMOTRON-H-W3C` @ `3295d0c1a` (PR #565) returned +**PASS** with four MINOR/NIT findings and two report-only items. Repaired on +`row/MODEL-NEMOTRON-H-W3-FIX` (base `3295d0c1a` + `origin/main` re-merged, W1's +`MIXED_PRECISION` resolver having landed as `1bc5ef82c`). + +**1. `ClaimMamba` ignored `quantized` — FIXED.** Every other claimer +(`ClaimNvfp4`, `ClaimMoe`, `ClaimMlp`, and `ClaimAttention`'s `fp8_kv`) gates on +`quantized`; `ClaimFp8` did not. The released config MINUS `quantization_config` +parsed without refusal and enumerated **92 tensors an unquantized checkpoint +does not ship** — `backbone.layers.{mamba}.mixer.{in,out}_proj.{weight_scale, +input_scale}`, 23x4, first `backbone.layers.0.mixer.in_proj.weight_scale`. That +is exactly the shape §5b's owed **safetensors bf16 arm** will present. Nothing +consumes the map yet, so no gated claim was wrong — but an unimplemented arm is +refused BY NAME, never silently mis-enumerated. `quantized` now threads through +`ClaimFp8`/`ClaimMamba` and both call sites (backbone `quantized`, MTP +`mtp_quantized`). The 18487-tensor gate is unchanged: the released checkpoint is +quantized, and its MTP schedule is `{attention, moe}` with no mamba block. + +**2. Legacy-alias precedence was INVERTED — FIXED, and the RECORD corrected.** +`Get{Int,Double,Bool}Aliased` preferred the MODERN key. Upstream does the +opposite for the `mamba_*` SCALARS: `configuration_nemotron_h.py:145-155` is +`self.n_groups = kwargs.pop("mamba_n_groups") if "mamba_n_groups" in kwargs else +self.n_groups`, which OVERWRITES an already-populated dataclass field, so +**legacy wins**. Re-derived by RUNNING transformers @ `7d06b1a5` (the pin this +port names), not by reading it: + +``` +NemotronHConfig(n_groups=8, mamba_n_groups=4, conv_kernel=4, mamba_d_conv=7) + -> n_groups=4, conv_kernel=7 +NemotronHConfig(chunk_size=128, mamba_chunk_size=77, expand=2, mamba_expand=9, + use_conv_bias=True, mamba_conv_bias=False, + time_step_min=1e-3, mamba_dt_min=0.5) + -> chunk=77 expand=9 conv_bias=False dt_min=0.5 +NemotronHConfig(layer_types=['mamba','mamba'], hybrid_override_pattern='*-') + -> ['mamba', 'mamba'] +NemotronHConfig(mtp_layers_block_type=['mamba'], mtp_hybrid_override_pattern='*E') + -> ['mamba'] +``` + +So the two families **genuinely disagree** and each is mirrored on its own +terms: legacy-wins for the `mamba_*` scalars (`:145-155`), modern-wins for both +SCHEDULE pairs (`:158-165`, `:176-184`, where the legacy pattern is consulted +only when the modern list is `None`). Worse than the behavior was the record: +the code comment and §5c above asserted "the modern one wins" as if it were +upstream's rule, which is what would mislead the next porter. Both polarities +are now stated where they are implemented, each with its own upstream anchor and +an explicit "do not unify these" note. **No released checkpoint ships both +spellings of one field**, so this was a mirroring defect and a record defect, +never a live one — which is precisely why it needed a test. + +**3. The forward refusal was claimed but never exercised — FIXED.** The case +titled "the unported ARMS refuse by name" had one SUBCASE (GGUF). +`ForwardNemotronHForCausalLM` is an unconditional `VT_CHECK`, which throws +`std::runtime_error` (`vt/dtype.h:11-17`), so it is directly callable with a stub +`LoadedModel`. Now asserted, including that the message NAMES the missing piece +(W4 and this spec). + +**4. `NemotronHBlockName` had zero call sites — FIXED by using it.** +`BlockFromName` now maps both directions through it and builds its refusal's +expected-list from the enum, so a fifth block kind cannot arrive alongside a +message that still lists four. A round-trip assertion pins every spelling the +refusal offers to one that actually parses. + +**5. RESIDUAL, pre-existing, NOT fixed here — `docs/FEATURES.md:171` is off by +two.** It says "27 of the 32 registered text-generation architectures" while +`:173` implies 38 − 3 Parakeet − 1 `LlamaModel` = **34**, and +`tests/vllm/models/test_model_registry.cpp:47` now says "34 text archs". W3's +`+1` increment was correct; the BASE number was already stale before this row +touched it, and no checker validates it. Left for the operator to file — the +repair branch has no `docs/` authority, and fixing a pre-existing doc drift +inside a scoped repair would hide it. + +**6. RESIDUAL, accepted by design — fixture DTYPE drift is invisible offline.** +The committed index projection pins tensor NAMES and SHAPES offline; DTYPES are +only re-verified by the live case +(`test_nemotron_h_scaffold.cpp`, `VT_NEMOTRON35_SNAPSHOT`). CI has no +checkpoint, so a re-quantization that changed dtypes while preserving names and +shapes — which has happened before to an `unsloth` repo under an unchanged name +— would pass CI and fail only where the checkpoint is staged. That is the +declared design (the alternative is committing dtype metadata that nothing +offline can falsify), and it is named here so W4/W6 do not rediscover it. + +**Mutation proof, re-run in full on the repaired tree.** Each defect applied +alone to the restored tree, rebuilt, the gate run, then the file restored and +its SHA-256 re-verified byte-for-byte (`git status --porcelain` empty +afterwards). All fourteen turn it RED — the ten from the W3 review plus four +this pass adds: + +| Mutation | Result | +|---|---| +| `layers_block_type` `"moe"` mapped to `kAttention` (at `BlockFromName`) | 7 cases / 12 assertions FAILURE | +| `kMoe` claims attention tensors (at the enumeration switch) | 2 cases / 4 assertions FAILURE | +| `LayerIndices` shifts every index by +1 | 2 cases / 2 assertions FAILURE | +| `conv_dim` drops the `2*n_groups*state_size` term | 4 cases / 8 assertions FAILURE | +| SSM cache dtype collapsed to the conv/activation dtype | 1 case / 2 assertions FAILURE | +| mamba `dt_bias` left UNCLAIMED (23 tensors) | 2 cases / 3 assertions FAILURE | +| MTP `final_layernorm` dropped | 2 cases / 3 assertions FAILURE | +| attention KV group collapsed to ONE layer tag | 1 case / 1 assertion FAILURE | +| the `mtp*` `ignore` entry not honored | 3 cases / 4 assertions FAILURE (assertions 38284 -> **39320**) | +| mamba KV group collapsed to ONE layer tag | 1 case / 1 assertion FAILURE | +| MTP `enorm` dropped | 2 cases / 3 assertions FAILURE | +| **NEW** the forward returns `{}` instead of refusing (finding 3) | 1 case / 2 assertions FAILURE | +| **NEW** `NemotronHBlockName` mislabels `kMoe` (finding 4) | 8 cases / **0 assertions** FAILURE | +| **NEW** `ClaimFp8` ignores `quantized` again (finding 1) | 1 case / 1 assertion FAILURE | +| **NEW** the aliased getters prefer MODERN again (finding 2) | 1 case / 4 assertions FAILURE | + +Two of those rows are worth keeping in view. The `mtp*` mutation makes the +assertion COUNT go **up** by 1036 while the gate goes red — a changed count is +itself the signal. And the `NemotronHBlockName` mutation reports +**`assertions: 28 | 28 passed | 0 failed`** next to `8 failed` test cases: the +cases THREW, so `grep 'assertions:'` alone would have read that mutation as +clean. Read `Status:`. + +**Gate evidence (this repair).** Local CPU-only host (`VLLM_CPP_CUDA=OFF`), disk +**68G free / 85% used** at every measurement below. + +| Arm | Result | +|---|---| +| Release `-Werror`, full build | 0 warnings, 0 errors | +| Release `test_nemotron_h_scaffold`, offline | **12/12 cases, 38284/38284 assertions, `Status: SUCCESS!`** | +| Release `test_nemotron_h_scaffold`, `VT_NEMOTRON35_SNAPSHOT` live | **12/12, 39152/39152, `Status: SUCCESS!`** | +| Debug (`-g0`, asserts unmasked), offline | 12/12, 38284/38284, `Status: SUCCESS!` | +| Debug (`-g0`), live | 12/12, 39152/39152, `Status: SUCCESS!` | +| Full `ctest -j4` | **100% tests passed, 0 tests failed out of 403** (skipped: `test_modelopt_mixed_precision_checkpoint`, `test_voxtral_e2e` — neither has its asset here) | + +The W3 baselines were 10/38245 offline and 10/39113 live; the deltas (+2 cases, ++39 offline / +39 live assertions) are this pass's three new cases and the +`NemotronHBlockName` round-trip. RED-before on the pre-fix tree was +12 cases / 9 failed assertions, reporting `92` companions with first +`backbone.layers.0.mixer.in_proj.weight_scale`, and `8 == 4` / `4 == 7` / +`2 == 9` / `128 == 77` / `0.001 == 0.5` / `0.1 == 0.6` / `0.0001 == 0.7` for the +alias precedence. + ## 6. Risks / decisions - **Non-gated MoE must not become a parallel path.** If From cb37239d4eeef28793ad756df2504f8522056c67 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 13 Aug 2026 02:29:33 +0000 Subject: [PATCH 5/5] spec(MODEL-NEMOTRON-H): the one RED gate this branch cannot clear itself -- doc-checkpoint needs docs/ authority (#517) `scripts/agent-preflight.sh --staged` is green on every gate except `doc-checkpoint range`, which fails on `3981de6a4`: it touched `src/vllm/model_executor/models/` without touching `docs/FEATURES.md`. The rule is unconditional and PER-COMMIT -- `check-doc-checkpoint.py:79` classifies any `src/vllm/model_executor/models/` path as `feature_surface`, `:313` requires `docs/FEATURES.md` in the SAME commit, and `commits_in_range` walks `rev-list --reverse --no-merges origin/main..HEAD`, so W3C's own `docs/FEATURES.md` edit in `3295d0c1a` does not cover a later commit. `ci.yml:343` runs the identical invocation on the PR, so the PR is red for the same reason. The checker has no exemption mechanism, by design. This repair branch was dispatched with `docs/` explicitly excluded from its authority, so it does not take the obvious fix and does not weaken the checker to go green. Recorded in the spec with the exact minimal edit the operator can fold into `3981de6a4`: one truthful clause on the `NemotronHForCausalLM` row naming the bf16 enumeration this commit corrected. It disturbs neither W3's existing `docs/` edits nor the separate finding-5 count drift. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:claude-opus-5 [Claude Code] --- .agents/specs/nemotron-h-model.md | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.agents/specs/nemotron-h-model.md b/.agents/specs/nemotron-h-model.md index de6c0cffa..8a6998f6a 100644 --- a/.agents/specs/nemotron-h-model.md +++ b/.agents/specs/nemotron-h-model.md @@ -737,6 +737,41 @@ The W3 baselines were 10/38245 offline and 10/39113 live; the deltas (+2 cases, `2 == 9` / `128 == 77` / `0.001 == 0.5` / `0.1 == 0.6` / `0.0001 == 0.7` for the alias precedence. +**ONE GATE IS RED, and it needs the operator — `doc-checkpoint range`.** +`scripts/agent-preflight.sh --staged` reports every gate green EXCEPT: + +``` +Committed range vs origin/main: + ok now-current range + FAIL doc-checkpoint range + ERROR: commit 3981de6a4: changed feature_surface but did not update + docs/FEATURES.md. This change alters what the project supports or how it + is used, which its purpose-specific document has to reflect. +``` + +This is not a defect in the change, and it is not a checker to weaken. The rule +is unconditional and PER-COMMIT: `check-doc-checkpoint.py:79` classifies any +path under `src/vllm/model_executor/models/` as `feature_surface`, `:313` +requires `docs/FEATURES.md` in the SAME commit, and `commits_in_range` walks +`rev-list --reverse --no-merges origin/main..HEAD` — so W3C's own +`docs/FEATURES.md` edit in `3295d0c1a` does not cover a later commit. CI runs +the identical invocation on the PR (`ci.yml:343`), so the PR is red for the same +reason. The checker has no exemption mechanism, deliberately. + +The repair branch was dispatched with `docs/` explicitly excluded from its +authority, so it did not take the obvious fix. **The operator holds that +authority.** The minimal truthful edit is one clause on the +`NemotronHForCausalLM` row (`docs/FEATURES.md:139`) — a real feature-surface +fact this commit changed: + +> ... 18487/18487 released tensors claimed, 0 unaccounted; het-KV shapes match +> `mamba2_state_shape`. **A bf16 producer (no `quantization_config`) claims the +> bare weights and none of the 92 FP8/NVFP4 scale companions.** + +folded into `3981de6a4` (amend, or a squash-merge carrying both). It disturbs +neither W3's existing `docs/` edits nor the separate finding-5 count drift, +which stays open for its own issue. + ## 6. Risks / decisions - **Non-gated MoE must not become a parallel path.** If