From 67ac5cb713851ba3c63ac144d5e4920f7744b496 Mon Sep 17 00:00:00 2001 From: 1fanwang <1fannnw@gmail.com> Date: Tue, 12 May 2026 06:48:35 -0700 Subject: [PATCH] Skip MoE config conversion when target_modules is a regex string `_convert_peft_config_moe` called `set(peft_config.target_modules or [])` unconditionally. When a user passes `target_modules` as a regex string (e.g. ms-swift's `get_multimodal_target_regex` emits something like `^(thinker\.model(?=\.).*\.(q_proj|k_proj|...))$` for multimodal MoE models such as Qwen3-Omni), the `set()` iterates the string character by character. That produces a nonsense set of regex metacharacters and letters and fails downstream with: ValueError: Target modules {'i', 'e', 'p', 'q', 't', '(', 'o', '|', '.', ')', 'h', '_', 'j', 'r', '^', '\\', '$', 'd', 'k', '?', 'v', 'n', 'm', 'l', '=', '*'} not found in the base model. The MoE conversion loop further down assumes module-name granularity, so a regex pattern can't be remapped automatically. Detect the regex case explicitly, emit a UserWarning that points users at the v5 module names, and return without touching the config. Fixes #3229. --- .../utils/transformers_weight_conversion.py | 18 ++++++++ tests/test_integrations.py | 46 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/src/peft/utils/transformers_weight_conversion.py b/src/peft/utils/transformers_weight_conversion.py index 3350175481..94cceab955 100644 --- a/src/peft/utils/transformers_weight_conversion.py +++ b/src/peft/utils/transformers_weight_conversion.py @@ -15,6 +15,7 @@ # NOTE: don't import from this module unless transformers v5+ is used import copy import re +import warnings from typing import Any import torch @@ -368,6 +369,23 @@ def _convert_peft_config_moe(peft_config, model_type: str) -> None: if not fused_targets: return + # `target_modules` may be a regex string (e.g. when the user passes a pattern produced by an + # upstream framework like ms-swift's multimodal helper). In that case, iterating it would + # split the regex into individual characters, leading to a nonsense set like + # {'^', '(', 'q', '_', 'p', 'r', 'o', 'j', ...} and a downstream "Target modules not found" + # error. The MoE conversion below assumes module-name granularity, so it can't sensibly + # rewrite a regex; skip the conversion and leave the regex untouched. + if isinstance(peft_config.target_modules, str): + warnings.warn( + f"Skipping MoE target-module conversion for model_type={model_type!r}: " + "`target_modules` is a regex string and cannot be remapped automatically. " + "If this is a transformers v4 checkpoint loaded against a v5 architecture, " + "please update the regex to reference the v5 module names (e.g. `gate_up_proj`, " + "`down_proj`).", + stacklevel=2, + ) + return + peft_config.target_parameters = set(peft_config.target_parameters or []) peft_config.target_modules = set(peft_config.target_modules or []) if not hasattr(peft_config, "rank_pattern") or peft_config.rank_pattern is None: diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 844d1ee450..3b687926c3 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -391,3 +391,49 @@ def test_qwen3_moe_works(self): # target up_proj and gate_proj config = LoraConfig(target_modules=["gate_proj", "up_proj", "down_proj", "score"]) get_peft_model(model, config) # does not raise + + @pytest.mark.parametrize( + "model_type, regex", + [ + # The kind of pattern ms-swift's `get_multimodal_target_regex` helper produces for Qwen3-Omni; see + # https://github.com/huggingface/peft/issues/3229 and + # https://github.com/modelscope/ms-swift/issues/9321. + ( + "qwen3_omni_moe_thinker", + r"^(thinker\.model(?=\.).*\.(q_proj|k_proj|v_proj|gate_proj|up_proj|down_proj))$", + ), + # A simpler "ends with q_proj" pattern that still happens to contain MoE module names as substrings. + ("mixtral", r".*\.q_proj"), + ], + ) + def test_moe_conversion_preserves_regex_target_modules(self, model_type, regex): + # Regression test for https://github.com/huggingface/peft/issues/3229. The MoE config conversion previously did + # `set(peft_config.target_modules or [])`, which when `target_modules` is a regex string splits the regex into + # its individual characters (yielding `{'^', '(', 'q', '_', ...}`). The resulting per-character "targets" then + # caused "Target modules ... not found in the base model" downstream. + from peft.utils.transformers_weight_conversion import _convert_peft_config_moe + + config = LoraConfig(target_modules=regex, target_parameters=["foo.weight"]) + assert isinstance(config.target_modules, str) # sanity check on LoraConfig + + with pytest.warns(UserWarning, match="`target_modules` is a regex string"): + _convert_peft_config_moe(config, model_type) + + # The regex must be preserved verbatim — no `set(...)`-induced character splitting. + assert isinstance(config.target_modules, str) + assert config.target_modules == regex + # `target_parameters` is left untouched as well; the conversion is fully skipped. + assert config.target_parameters == ["foo.weight"] + + def test_moe_conversion_remaps_list_target_modules(self): + # Defensive companion to test_moe_conversion_preserves_regex_target_modules: a list of module names should + # still go through the normal MoE conversion path and remap `gate_proj` + `up_proj` to the fused + # `gate_up_proj`. + from peft.utils.transformers_weight_conversion import _convert_peft_config_moe + + config = LoraConfig(target_modules=["gate_proj", "up_proj"]) + # list -> set in LoraConfig.__post_init__ + assert isinstance(config.target_modules, set) + _convert_peft_config_moe(config, "qwen3_omni_moe_thinker") + # After conversion the original module names are remapped onto `gate_up_proj` via target_parameters. + assert "gate_up_proj" in config.target_parameters