From e80bcdcba52868ef9f348c81c9c418ef01d84032 Mon Sep 17 00:00:00 2001 From: three Date: Wed, 18 Mar 2026 11:56:16 -0500 Subject: [PATCH 1/9] added glm 47 to moe --- moe_registry.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/moe_registry.py b/moe_registry.py index 0501fe3..c19b5b5 100644 --- a/moe_registry.py +++ b/moe_registry.py @@ -2,6 +2,7 @@ Supports: - MiniMaxM2 (built-in transformers): MiniMaxM2Experts + - GLM-4 MoE (glm4_moe): Glm4MoeMoE + Glm4MoeNaiveMoe - GLM-5 / glm_moe_dsa (remote code): GlmMoeDsaMoE + GlmMoeDsaNaiveMoe - Qwen3.5 MoE: Qwen3_5MoeSparseMoeBlock + Qwen3_5MoeExperts @@ -119,6 +120,43 @@ def forward(self, hidden_states, top_k_index, top_k_weights): return final_hidden_states +# --------------------------------------------------------------------------- +# GLM-4 MoE (glm4_moe). +# --------------------------------------------------------------------------- + +class _QuantGlm4MoeMoE(_QuantSparseMoe): + @property + def num_experts(self): + return self.n_routed_experts + + def forward(self, hidden_states): + return super(_QuantSparseMoe, self).forward(hidden_states) + + +def register_glm4_7_moe_for_quantization(): + """Register GLM-4 MoE (glm4_moe) classes with modelopt.""" + try: + from transformers.models.glm4_moe.modeling_glm4_moe import ( + Glm4MoeMoE, + Glm4MoeNaiveMoe, + ) + except ImportError: + print("⚠ glm4_moe not in transformers, skipping GLM-4 registration") + return + + if QuantModuleRegistry.get(Glm4MoeMoE) is None: + QuantModuleRegistry.register( + {Glm4MoeMoE: "Glm4MoeMoE"} + )(_QuantGlm4MoeMoE) + + if QuantModuleRegistry.get(Glm4MoeNaiveMoe) is None: + QuantModuleRegistry.register( + {Glm4MoeNaiveMoe: "Glm4MoeNaiveMoe"} + )(_QuantFusedExperts) + + print("✓ Registered GLM-4 MoE for quantization") + + # --------------------------------------------------------------------------- # GLM-5 (glm_moe_dsa). # --------------------------------------------------------------------------- From 711fe9ebaef98130d850e038ede56f94d97b67f6 Mon Sep 17 00:00:00 2001 From: three Date: Wed, 18 Mar 2026 11:56:32 -0500 Subject: [PATCH 2/9] added to init --- models/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/__init__.py b/models/__init__.py index cc9a6ca..504090a 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -1,3 +1,4 @@ +from .glm4_7 import Glm4_7Config from .glm5 import Glm5Config from .minimax_m25 import MinimaxM25Config from .qwen3_5_122b import Qwen35_122BConfig @@ -5,6 +6,7 @@ from .qwen3_5_moe_noshared import Qwen35MoeNoSharedConfig _CONFIGS = { + "glm4_7": Glm4_7Config, "glm5": Glm5Config, "minimax_m25": MinimaxM25Config, "qwen3_5_122b": Qwen35_122BConfig, From 4fb60af6ea0a81faf6e3486e085e8f2096b98e0c Mon Sep 17 00:00:00 2001 From: three Date: Wed, 18 Mar 2026 11:56:58 -0500 Subject: [PATCH 3/9] mine --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ccfb0d2..7f879b6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ *.pyc uv.lock output/ +venv/ From 171058e46e1a42c6ad3a66ad4cc9013f5624dc7e Mon Sep 17 00:00:00 2001 From: three Date: Wed, 18 Mar 2026 11:58:32 -0500 Subject: [PATCH 4/9] glm 47 calib --- configs/calib_glm4_7.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 configs/calib_glm4_7.toml diff --git a/configs/calib_glm4_7.toml b/configs/calib_glm4_7.toml new file mode 100644 index 0000000..adf7c08 --- /dev/null +++ b/configs/calib_glm4_7.toml @@ -0,0 +1,9 @@ +# Calibration datasets for GLM-4.7. + +[[dataset]] +path = "text/agentic_coding_calib_v3.jsonl" +max_len = 4096 + +[[dataset]] +path = "text/deep_calib.jsonl" +max_len = 4096 From 784cdea1d4b9a397e5fa717f930da820d7fdc7dc Mon Sep 17 00:00:00 2001 From: three Date: Wed, 18 Mar 2026 12:01:21 -0500 Subject: [PATCH 5/9] model thingy --- models/glm4_7.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 models/glm4_7.py diff --git a/models/glm4_7.py b/models/glm4_7.py new file mode 100644 index 0000000..be8c372 --- /dev/null +++ b/models/glm4_7.py @@ -0,0 +1,20 @@ +from .base import ModelQuantConfig + + +Glm4_7Config = _Glm4_7Config( + model_id="zai-org/GLM-4.7", + trust_remote_code=True, + streaming=True, + extra_quant_overrides={ + "*mlp.gate*": {"enable": False}, + "*mtp*": {"enable": False}, + }, +) + + +class _Glm4_7Config(ModelQuantConfig): + def register_moe(self): + from moe_registry import register_glm4_7_moe_for_quantization + register_glm4_7_moe_for_quantization() + + From 747ba9ec7ff57e471d17f1e12aefe99c48e5db50 Mon Sep 17 00:00:00 2001 From: three Date: Wed, 18 Mar 2026 12:02:48 -0500 Subject: [PATCH 6/9] glm47 quant script --- scripts/quantize_glm4_7.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 scripts/quantize_glm4_7.sh diff --git a/scripts/quantize_glm4_7.sh b/scripts/quantize_glm4_7.sh new file mode 100755 index 0000000..7867797 --- /dev/null +++ b/scripts/quantize_glm4_7.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")/.." +source venv/bin/activate + +export SAFETENSORS_FAST_GPU=1 +export PYTORCH_ALLOC_CONF=expandable_segments:True + +python quantize.py \ + --model glm4_7 \ + --model-id /nvme/models/safetensors/GLM-4.7/ \ + --export-dir /nvme/models/temp/GLM-4.7-NVFP4 \ + --calib-config configs/calib_glm4_7.toml \ + --cpu-capacity 256GiB \ + --streaming \ + --floor-amaxes \ + --save-amax /nvme/models/temp/GLM-4.7-NVFP4/amaxes.safetensors \ + --batch-tokens 1572864 \ From 8b17beb8e1f2f276e998decc10b385aec7d1be24 Mon Sep 17 00:00:00 2001 From: three Date: Wed, 18 Mar 2026 12:07:53 -0500 Subject: [PATCH 7/9] made it look like lukes --- models/glm4_7.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/models/glm4_7.py b/models/glm4_7.py index be8c372..9dc854d 100644 --- a/models/glm4_7.py +++ b/models/glm4_7.py @@ -1,7 +1,7 @@ from .base import ModelQuantConfig -Glm4_7Config = _Glm4_7Config( +Glm4_7Config = ModelQuantConfig( model_id="zai-org/GLM-4.7", trust_remote_code=True, streaming=True, @@ -12,9 +12,9 @@ ) -class _Glm4_7Config(ModelQuantConfig): - def register_moe(self): - from moe_registry import register_glm4_7_moe_for_quantization - register_glm4_7_moe_for_quantization() +def _register_moe(): + from moe_registry import register_glm4_7_moe_for_quantization + register_glm4_7_moe_for_quantization() +Glm4_7Config.register_moe = _register_moe From 3fa1c8ababdfd9520d921fe5bac029d14202b840 Mon Sep 17 00:00:00 2001 From: 3 <3@3> Date: Wed, 18 Mar 2026 12:23:44 -0500 Subject: [PATCH 8/9] added diverse gpu size support --- streaming_loader.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/streaming_loader.py b/streaming_loader.py index 47512c3..546df3a 100644 --- a/streaming_loader.py +++ b/streaming_loader.py @@ -87,12 +87,13 @@ def __init__( self.num_gpus = torch.cuda.device_count() if gpu_capacity_gib is None: - # Auto-detect from first GPU - self.gpu_capacity_gib = ( - torch.cuda.get_device_properties(0).total_memory / 1024**3 - ) + # Auto-detect per-GPU capacity + self.gpu_capacities_gib = [ + torch.cuda.get_device_properties(i).total_memory / 1024**3 + for i in range(self.num_gpus) + ] else: - self.gpu_capacity_gib = gpu_capacity_gib + self.gpu_capacities_gib = [gpu_capacity_gib] * self.num_gpus self.snapshot_dir = _resolve_snapshot_dir(model_id) index_path = os.path.join(self.snapshot_dir, "model.safetensors.index.json") @@ -201,7 +202,6 @@ def _compute_storage_map(self, layer_sizes): # Reserve headroom per storage GPU for CUDA context, driver memory, # and PyTorch allocator fragmentation from loading many small tensors. gpu_headroom = 4.0 * 1024**3 - gpu_capacity = self.gpu_capacity_gib * 1024**3 - gpu_headroom cpu_capacity = self.cpu_capacity_gib * 1024**3 # GPU 0 is reserved for execution — start packing from GPU 1 @@ -214,6 +214,7 @@ def _compute_storage_map(self, layer_sizes): placed = False while current_gpu < self.num_gpus: + gpu_capacity = self.gpu_capacities_gib[current_gpu] * 1024**3 - gpu_headroom if gpu_used + size <= gpu_capacity: storage_map[i] = f"cuda:{current_gpu}" gpu_used += size From e73270612bcd7d5180cb81f77004b11310ee4025 Mon Sep 17 00:00:00 2001 From: 3 <3@3> Date: Wed, 18 Mar 2026 13:44:30 -0500 Subject: [PATCH 9/9] working on oom --- quantize.py | 2 ++ streaming_loader.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/quantize.py b/quantize.py index f37923a..103b401 100644 --- a/quantize.py +++ b/quantize.py @@ -51,6 +51,8 @@ help="Load amax checkpoint and resume calibration from where it left off.") parser.add_argument("--resume-batch", type=int, default=0, help="Skip batches before this number (1-indexed). Use with --resume-amax.") +parser.add_argument("--microbatch", type=int, default=None, + help="Split calibration batches into microbatches of this size to limit GPU 0 activation memory.") parser.add_argument("--calib-method", default="max", choices=["max", "quantile"], help="Calibration algorithm. 'quantile' uses P2 streaming quantile estimation.") parser.add_argument("--save-quantiles", type=str, default=None, diff --git a/streaming_loader.py b/streaming_loader.py index 546df3a..27b5c51 100644 --- a/streaming_loader.py +++ b/streaming_loader.py @@ -199,9 +199,6 @@ def _compute_layer_sizes(self, model): def _compute_storage_map(self, layer_sizes): """Assign each layer to a storage device: cuda:1-N, cpu, or meta (disk).""" storage_map = {} - # Reserve headroom per storage GPU for CUDA context, driver memory, - # and PyTorch allocator fragmentation from loading many small tensors. - gpu_headroom = 4.0 * 1024**3 cpu_capacity = self.cpu_capacity_gib * 1024**3 # GPU 0 is reserved for execution — start packing from GPU 1 @@ -214,7 +211,10 @@ def _compute_storage_map(self, layer_sizes): placed = False while current_gpu < self.num_gpus: - gpu_capacity = self.gpu_capacities_gib[current_gpu] * 1024**3 - gpu_headroom + # Reserve ~4% headroom for CUDA context, driver memory, + # and PyTorch allocator fragmentation. + gpu_total = self.gpu_capacities_gib[current_gpu] * 1024**3 + gpu_capacity = gpu_total * 0.96 if gpu_used + size <= gpu_capacity: storage_map[i] = f"cuda:{current_gpu}" gpu_used += size