From ff9ad5aaab27224bd10984f7d43c14f79a58eb29 Mon Sep 17 00:00:00 2001 From: isha Date: Sat, 9 May 2026 18:30:54 +0000 Subject: [PATCH 1/8] feat: registry-based dispatch for resolve_lora_variant in Linear layer --- src/peft/tuners/lora/config.py | 16 +++++++---- src/peft/tuners/lora/layer.py | 49 ++++++++++++++++------------------ 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/peft/tuners/lora/config.py b/src/peft/tuners/lora/config.py index 59d9c9b14e..52b963bc0a 100644 --- a/src/peft/tuners/lora/config.py +++ b/src/peft/tuners/lora/config.py @@ -709,7 +709,8 @@ class LoraConfig(PeftConfig): "magnitude is handled by a separate learnable parameter. This can improve the performance of LoRA, " "especially at low ranks. Right now, DoRA only supports linear and Conv2D layers. DoRA introduces a bigger" "overhead than pure LoRA, so it is recommended to merge weights for inference." - ) + ), + "is_variant": True, }, ) velora_config: Optional[Union[VeloraConfig, dict]] = field( @@ -718,7 +719,8 @@ class LoraConfig(PeftConfig): "help": ( "Enable VeLoRA as a LoRA variant by providing a VeloraConfig. VeLoRA swaps in a custom backward pass " "for the LoRA A projection that stores compressed activations instead of the full input activations." - ) + ), + "is_variant": True, }, ) alora_invocation_tokens: Optional[list[int]] = field( @@ -735,7 +737,8 @@ class LoraConfig(PeftConfig): "operations. Overall adapter inference speedups of an order of magnitude or more can occur on vLLM, " "depending on the length of the shared context. Note that merging is not possible due to the selective " "application of the weights." - ) + ), + "is_variant": True, }, ) use_qalora: bool = field( @@ -816,11 +819,14 @@ class LoraConfig(PeftConfig): "help": ( "Enable BD-LoRA (Block-Diagonal LoRA) by providing a BdLoraConfig. This technique uses block-diagonal matrices for LoRA-A or LoRA-B " "factors to enable faster multi-LoRA serving by eliminating communication overheads in distributed settings." - ) + ), + "is_variant": True, }, ) arrow_config: Optional[ArrowConfig] = field( - default=None, metadata={"help": "The necessary config to apply arrow routing on the model."} + default=None, metadata={"help": "The necessary config to apply arrow routing on the model.", + "is_variant": True, + }, ) ensure_weight_tying: bool = field( default=False, diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 006e8eb00b..da7a0a3683 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -795,33 +795,30 @@ def __init__( **kwargs, ) self.is_target_conv_1d_layer = is_target_conv_1d_layer - + + @property + def valid_variants(self): + from . import variants + return { + (): None, + ("use_dora",): variants.DoraLinearVariant, + ("arrow_config",): variants.ArrowLinearVariant, + ("use_bdlora",): variants.BdLoraLinearVariant, + ("alora_invocation_tokens",): variants.ALoraLinearVariant, + ("velora_config",): variants.VeloraLinearVariant, + } def resolve_lora_variant(self, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - if config.velora_config is not None: - from .variants import VeloraLinearVariant - - return VeloraLinearVariant() - - if config.arrow_config is not None: - from .variants import ArrowLinearVariant - - return ArrowLinearVariant() - - if config.use_bdlora is not None: - from .variants import BdLoraLinearVariant - - return BdLoraLinearVariant() - - use_alora = config.alora_invocation_tokens is not None - if not config.use_dora and not use_alora: - return None - - from .variants import ALoraLinearVariant, DoraLinearVariant - - if use_alora: - return ALoraLinearVariant() - else: - return DoraLinearVariant() + import dataclasses + from . import variants + + active = tuple(sorted( + f.name for f in dataclasses.fields(config) + if f.metadata.get("is_variant") and getattr(config, f.name) + )) + if active not in self.valid_variants: + raise ValueError(f"Invalid or unsupported variant combination: {active}") + variant_class = self.valid_variants[active] + return variant_class() if variant_class else None def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None: """ From 4bab6dbc184d21c6d9709bff1a5beb8513eda668 Mon Sep 17 00:00:00 2001 From: isha Date: Thu, 14 May 2026 19:13:15 +0000 Subject: [PATCH 2/8] Refactor LoRA variant resolution into base LoraLayer and add config sanity check --- src/peft/tuners/lora/config.py | 10 +-- src/peft/tuners/lora/layer.py | 136 ++++++++++++++++++--------------- tests/test_lora_variants.py | 16 +++- 3 files changed, 96 insertions(+), 66 deletions(-) diff --git a/src/peft/tuners/lora/config.py b/src/peft/tuners/lora/config.py index 52b963bc0a..38fc48ca0a 100644 --- a/src/peft/tuners/lora/config.py +++ b/src/peft/tuners/lora/config.py @@ -710,7 +710,7 @@ class LoraConfig(PeftConfig): "especially at low ranks. Right now, DoRA only supports linear and Conv2D layers. DoRA introduces a bigger" "overhead than pure LoRA, so it is recommended to merge weights for inference." ), - "is_variant": True, + "is_lora_variant": True, }, ) velora_config: Optional[Union[VeloraConfig, dict]] = field( @@ -720,7 +720,7 @@ class LoraConfig(PeftConfig): "Enable VeLoRA as a LoRA variant by providing a VeloraConfig. VeLoRA swaps in a custom backward pass " "for the LoRA A projection that stores compressed activations instead of the full input activations." ), - "is_variant": True, + "is_lora_variant": True, }, ) alora_invocation_tokens: Optional[list[int]] = field( @@ -738,7 +738,7 @@ class LoraConfig(PeftConfig): "depending on the length of the shared context. Note that merging is not possible due to the selective " "application of the weights." ), - "is_variant": True, + "is_lora_variant": True, }, ) use_qalora: bool = field( @@ -820,12 +820,12 @@ class LoraConfig(PeftConfig): "Enable BD-LoRA (Block-Diagonal LoRA) by providing a BdLoraConfig. This technique uses block-diagonal matrices for LoRA-A or LoRA-B " "factors to enable faster multi-LoRA serving by eliminating communication overheads in distributed settings." ), - "is_variant": True, + "is_lora_variant": True, }, ) arrow_config: Optional[ArrowConfig] = field( default=None, metadata={"help": "The necessary config to apply arrow routing on the model.", - "is_variant": True, + "is_lora_variant": True, }, ) ensure_weight_tying: bool = field( diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index da7a0a3683..783db16481 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -139,20 +139,56 @@ def __init__(self, base_layer: nn.Module, ephemeral_gpu_offload: bool = False, * def _get_in_out_features(self, module: nn.Module) -> tuple[int, int] | tuple[None, None]: return _get_in_out_features(module) + + @property + def lora_variants(self): + """ + A dictionary mapping the active LoRA variants to their respective classes. - def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - """Return a matching LoRA variant for this layer type. - - Given the init arguments of this layer, return the correct LoRA variant, if any. E.g., if `use_dora=True`, this - method should return the DoRA variant for the given layer. If `use_alora=True`, same for aLoRA. - - If there is no fitting variant, return None. - - Note: If this layer type does not support the LoRA variant at all, please raise an error during __init__ as is - convention, and not here. + To extend this, subclasses should override this property and return a dictionary + where the keys are tuples of variant field names (from LoraConfig) and the + values are the specific LoraVariant subclasses. + Tuples are used as keys because they are immutable and hashable, allowing us + to safely map combinations of active variants (e.g., DoRA + another variant) + to a specific composed variant class. """ - return None + return {(): None} + + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + import dataclasses + from . import variants + + # Safely fetch the dictionary (defaults to empty if a subclass forgot to define it) + layer_variants = getattr(self, "lora_variants", {(): None}) + + # 1. Gather all valid variant field names from the config + tagged_fields = { + f.name for f in dataclasses.fields(config) + if f.metadata.get("is_lora_variant") + } + + # 2. SANITY CHECK: Ensure all keys in the layer's dictionary actually exist in the config + for variant_combo in layer_variants.keys(): + for variant_name in variant_combo: + if variant_name not in tagged_fields: + raise ValueError( + f"Variant '{variant_name}' found in lora_variants but not tagged with " + f"'is_lora_variant' in LoraConfig." + ) + + # 3. Figure out which variants are currently active + active = tuple(sorted( + f.name for f in dataclasses.fields(config) + if f.metadata.get("is_lora_variant") and getattr(config, f.name) + )) + + # 4. Route to the correct variant class + if active not in layer_variants: + raise ValueError(f"Invalid or unsupported variant combination: {active}") + + variant_class = layer_variants[active] + return variant_class() if variant_class else None def update_layer( self, @@ -797,7 +833,7 @@ def __init__( self.is_target_conv_1d_layer = is_target_conv_1d_layer @property - def valid_variants(self): + def lora_variants(self): from . import variants return { (): None, @@ -807,18 +843,6 @@ def valid_variants(self): ("alora_invocation_tokens",): variants.ALoraLinearVariant, ("velora_config",): variants.VeloraLinearVariant, } - def resolve_lora_variant(self, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - import dataclasses - from . import variants - - active = tuple(sorted( - f.name for f in dataclasses.fields(config) - if f.metadata.get("is_variant") and getattr(config, f.name) - )) - if active not in self.valid_variants: - raise ValueError(f"Invalid or unsupported variant combination: {active}") - variant_class = self.valid_variants[active] - return variant_class() if variant_class else None def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None: """ @@ -1055,15 +1079,13 @@ def __init__( config=config, ) - def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - if config.velora_config is not None: - raise ValueError("VeLoRA does not support adapting embedding layers.") - if not config.use_dora: - return None - - from .variants import DoraEmbeddingVariant - - return DoraEmbeddingVariant() + @property + def lora_variants(self): + from . import variants + return { + (): None, + ("use_dora",): variants.DoraEmbeddingVariant, + } def update_layer( self, @@ -1664,15 +1686,13 @@ def __init__(self, *args, **kwargs): raise ValueError(f"Conv2d layer kernel must have 4 dimensions, not {self._kernel_dim}") self.conv_fn = F.conv2d - def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - if config.velora_config is not None: - raise ValueError("VeLoRA does not support adapting conv layers.") - if not config.use_dora: - return None - - from .variants import DoraConv2dVariant - - return DoraConv2dVariant() + @property + def lora_variants(self): + from . import variants + return { + (): None, + ("use_dora",): variants.DoraConv2dVariant, + } class Conv1d(_ConvNd): @@ -1683,15 +1703,13 @@ def __init__(self, *args, **kwargs): raise ValueError(f"Conv1d layer kernel must have 3 dimensions, not {self._kernel_dim}") self.conv_fn = F.conv1d - def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - if config.velora_config is not None: - raise ValueError("VeLoRA does not support adapting conv layers.") - if not config.use_dora: - return None - - from .variants import DoraConv1dVariant - - return DoraConv1dVariant() + @property + def lora_variants(self): + from . import variants + return { + (): None, + ("use_dora",): variants.DoraConv1dVariant, + } class Conv3d(_ConvNd): @@ -1702,15 +1720,13 @@ def __init__(self, *args, **kwargs): raise ValueError(f"Conv3d layer kernel must have 5 dimensions, not {self._kernel_dim}") self.conv_fn = F.conv3d - def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - if config.velora_config is not None: - raise ValueError("VeLoRA does not support adapting conv layers.") - if not config.use_dora: - return None - - from .variants import DoraConv3dVariant - - return DoraConv3dVariant() + @property + def lora_variants(self): + from . import variants + return { + (): None, + ("use_dora",): variants.DoraConv3dVariant, + } class MultiheadAttention(nn.Module, LoraLayer): diff --git a/tests/test_lora_variants.py b/tests/test_lora_variants.py index bb93b2988e..177b57a84b 100644 --- a/tests/test_lora_variants.py +++ b/tests/test_lora_variants.py @@ -16,6 +16,7 @@ import torch from torch import nn from transformers import AutoModelForCausalLM +from unittest.mock import PropertyMock, patch from peft import LoraConfig, TaskType, get_peft_model from peft.tuners.lora.layer import Conv1d as LoraConv1d @@ -173,7 +174,20 @@ def test_dora_params_have_gradients(self): for layer in layer_names: assert getattr(peft_model.base_model.model, layer).lora_magnitude_vector["default"].weight.grad is not None - + + def test_unregistered_variant_raises_error(self): + # 1. Create a config and dummy linear layer + config = LoraConfig() + base_layer = nn.Linear(10, 10) + layer = LoraLinear(base_layer, "default", config, r=8, lora_alpha=8) + + # 2. Monkey-patch the lora_variants property to include a fake variant + with patch("peft.tuners.lora.layer.Linear.lora_variants", new_callable=PropertyMock) as mock_variants: + mock_variants.return_value = {("fake_unregistered_variant",): None} + + # 3. Assert that the sanity check catches it and throws the right error + with pytest.raises(ValueError, match="not tagged with 'is_lora_variant'"): + layer.resolve_lora_variant(config=config) class TestActivatedLora: @pytest.mark.parametrize( From da3bd921bdb66edb810ed29e08aac64aff918872 Mon Sep 17 00:00:00 2001 From: Sanjana soni Date: Tue, 19 May 2026 13:15:10 +0530 Subject: [PATCH 3/8] Update src/peft/tuners/lora/layer.py Co-authored-by: Benjamin Bossan --- src/peft/tuners/lora/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 783db16481..dd6ffe420c 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -173,7 +173,7 @@ def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[Lora for variant_name in variant_combo: if variant_name not in tagged_fields: raise ValueError( - f"Variant '{variant_name}' found in lora_variants but not tagged with " + f"Variant '{variant_name}' found in lora_variants but it is not tagged with " f"'is_lora_variant' in LoraConfig." ) From e59ed420070ac5fac386d7b0e1cf674749b37f27 Mon Sep 17 00:00:00 2001 From: isha Date: Tue, 19 May 2026 08:29:06 +0000 Subject: [PATCH 4/8] refactor: address review comments - rename variables, extract lora_variant_configs, add tests --- src/peft/tuners/lora/layer.py | 42 ++++++++++++++++------------------- tests/test_lora_variants.py | 34 +++++++++++++++++++--------- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index dd6ffe420c..6d8fb69677 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -14,6 +14,7 @@ from __future__ import annotations import copy +import dataclasses import math import warnings from collections.abc import Callable @@ -22,9 +23,8 @@ import torch import torch.distributed as dist -import torch.nn as nn import torch.nn.functional as F -from torch import svd_lowrank +from torch import nn, svd_lowrank from transformers.pytorch_utils import Conv1D from peft.import_utils import is_transformers_ge_v5_4_0 @@ -139,7 +139,7 @@ def __init__(self, base_layer: nn.Module, ephemeral_gpu_offload: bool = False, * def _get_in_out_features(self, module: nn.Module) -> tuple[int, int] | tuple[None, None]: return _get_in_out_features(module) - + @property def lora_variants(self): """ @@ -154,23 +154,19 @@ def lora_variants(self): to a specific composed variant class. """ return {(): None} - + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - import dataclasses - from . import variants # Safely fetch the dictionary (defaults to empty if a subclass forgot to define it) layer_variants = getattr(self, "lora_variants", {(): None}) + lora_variants_configs = [f for f in dataclasses.fields(config) if f.metadata.get("is_lora_variant")] # 1. Gather all valid variant field names from the config - tagged_fields = { - f.name for f in dataclasses.fields(config) - if f.metadata.get("is_lora_variant") - } + tagged_fields = { f.name for f in lora_variants_configs } # 2. SANITY CHECK: Ensure all keys in the layer's dictionary actually exist in the config - for variant_combo in layer_variants.keys(): - for variant_name in variant_combo: + for variant_keys in layer_variants.keys(): + for variant_name in variant_keys: if variant_name not in tagged_fields: raise ValueError( f"Variant '{variant_name}' found in lora_variants but it is not tagged with " @@ -178,16 +174,16 @@ def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[Lora ) # 3. Figure out which variants are currently active - active = tuple(sorted( - f.name for f in dataclasses.fields(config) - if f.metadata.get("is_lora_variant") and getattr(config, f.name) + active_variants = tuple(sorted( + f.name for f in lora_variants_configs + if getattr(config, f.name) )) - + # 4. Route to the correct variant class - if active not in layer_variants: - raise ValueError(f"Invalid or unsupported variant combination: {active}") - - variant_class = layer_variants[active] + if active_variants not in layer_variants: + raise ValueError(f"Invalid or unsupported variant combination: {active_variants}") + + variant_class = layer_variants[active_variants] return variant_class() if variant_class else None def update_layer( @@ -679,7 +675,7 @@ def _cache_pop(self, key: str) -> Any: value = self._caches.pop(key) return value - def set_scale(self, adapter: str, scale: float | int) -> None: + def set_scale(self, adapter: str, scale: float) -> None: """Set the scale of the given adapter to the initial scale multiplied by the provided factor The initial scale is determined by the configured `r` (rank) and `lora_alpha`. @@ -692,7 +688,7 @@ def set_scale(self, adapter: str, scale: float | int) -> None: else: self.scaling[adapter] = scale * self.lora_alpha[adapter] / self.r[adapter] - def scale_layer(self, scale: float | int) -> None: + def scale_layer(self, scale: float) -> None: """Multiply the current scale of all active adapters by the provided factor""" if scale == 1: return @@ -831,7 +827,7 @@ def __init__( **kwargs, ) self.is_target_conv_1d_layer = is_target_conv_1d_layer - + @property def lora_variants(self): from . import variants diff --git a/tests/test_lora_variants.py b/tests/test_lora_variants.py index 177b57a84b..33e2466b98 100644 --- a/tests/test_lora_variants.py +++ b/tests/test_lora_variants.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from unittest.mock import PropertyMock, patch + import pytest import torch from torch import nn from transformers import AutoModelForCausalLM -from unittest.mock import PropertyMock, patch from peft import LoraConfig, TaskType, get_peft_model from peft.tuners.lora.layer import Conv1d as LoraConv1d @@ -174,7 +175,7 @@ def test_dora_params_have_gradients(self): for layer in layer_names: assert getattr(peft_model.base_model.model, layer).lora_magnitude_vector["default"].weight.grad is not None - + def test_unregistered_variant_raises_error(self): # 1. Create a config and dummy linear layer config = LoraConfig() @@ -184,11 +185,26 @@ def test_unregistered_variant_raises_error(self): # 2. Monkey-patch the lora_variants property to include a fake variant with patch("peft.tuners.lora.layer.Linear.lora_variants", new_callable=PropertyMock) as mock_variants: mock_variants.return_value = {("fake_unregistered_variant",): None} - + # 3. Assert that the sanity check catches it and throws the right error - with pytest.raises(ValueError, match="not tagged with 'is_lora_variant'"): + with pytest.raises(ValueError, match="Variant 'fake_unregistered_variant' found in lora_variants but it is not tagged with 'is_lora_variant' in LoraConfig."): layer.resolve_lora_variant(config=config) + def test_invalid_variant_combination_raises_error(self): + # 1. Create a config with no variants active + config = LoraConfig() + base_layer = nn.Linear(10, 10) + layer = LoraLinear(base_layer, "default", config, r=8, lora_alpha=8) + + # 2. Monkey-patch lora_variants to include a valid tagged combo that isn't active + with patch("peft.tuners.lora.layer.Linear.lora_variants", new_callable=PropertyMock) as mock_variants: + mock_variants.return_value = { + ("use_dora",): None, # only use_dora is valid, empty combo not listed + } + # 3. Assert invalid combination error is raised + with pytest.raises(ValueError, match="Invalid or unsupported variant combination"): + layer.resolve_lora_variant(config=config) + class TestActivatedLora: @pytest.mark.parametrize( "input_ids, alora_invocation_tokens, expected_offsets", @@ -239,9 +255,8 @@ def test_alora_activation_matches_base_until_invocation(self): input_ids = torch.tensor([[0, 1, 2, 3]]) start = 2 - with lora_model.disable_adapter(): - with torch.no_grad(): - base_out = lora_model(X=input_ids) + with lora_model.disable_adapter(), torch.no_grad(): + base_out = lora_model(X=input_ids) kwargs = get_alora_offsets_for_forward(lora_model, input_ids) with torch.no_grad(): @@ -286,9 +301,8 @@ def test_num_beams_error(self): lora_model.eval() input_ids = torch.tensor([[0, 1, 2, 3]]) - with pytest.raises(ValueError) as e: - with torch.no_grad(): - lora_out = lora_model(X=input_ids, num_beams=2, alora_offsets=[3]) + with pytest.raises(ValueError) as e, torch.no_grad(): + lora_out = lora_model(X=input_ids, num_beams=2, alora_offsets=[3]) assert "Beam search not yet supported for aLoRA." in str(e.value) def test_gradient_checkpointing_double_forward_raises(self): From c5d90b70b5c86c7f57ef4119cf7d6162bf83f031 Mon Sep 17 00:00:00 2001 From: isha Date: Sun, 24 May 2026 15:25:53 +0000 Subject: [PATCH 5/8] style: fix indentation in resolve_lora_variant --- src/peft/tuners/lora/layer.py | 50 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 6d8fb69677..46f845737f 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -157,34 +157,34 @@ def lora_variants(self): def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - # Safely fetch the dictionary (defaults to empty if a subclass forgot to define it) - layer_variants = getattr(self, "lora_variants", {(): None}) - lora_variants_configs = [f for f in dataclasses.fields(config) if f.metadata.get("is_lora_variant")] - - # 1. Gather all valid variant field names from the config - tagged_fields = { f.name for f in lora_variants_configs } - - # 2. SANITY CHECK: Ensure all keys in the layer's dictionary actually exist in the config - for variant_keys in layer_variants.keys(): - for variant_name in variant_keys: - if variant_name not in tagged_fields: - raise ValueError( - f"Variant '{variant_name}' found in lora_variants but it is not tagged with " - f"'is_lora_variant' in LoraConfig." - ) + # Safely fetch the dictionary (defaults to empty if a subclass forgot to define it) + layer_variants = getattr(self, "lora_variants", {(): None}) + lora_variants_configs = [f for f in dataclasses.fields(config) if f.metadata.get("is_lora_variant")] + + # 1. Gather all valid variant field names from the config + tagged_fields = { f.name for f in lora_variants_configs } + + # 2. SANITY CHECK: Ensure all keys in the layer's dictionary actually exist in the config + for variant_keys in layer_variants.keys(): + for variant_name in variant_keys: + if variant_name not in tagged_fields: + raise ValueError( + f"Variant '{variant_name}' found in lora_variants but it is not tagged with " + f"'is_lora_variant' in LoraConfig." + ) - # 3. Figure out which variants are currently active - active_variants = tuple(sorted( - f.name for f in lora_variants_configs - if getattr(config, f.name) - )) + # 3. Figure out which variants are currently active + active_variants = tuple(sorted( + f.name for f in lora_variants_configs + if getattr(config, f.name) + )) - # 4. Route to the correct variant class - if active_variants not in layer_variants: - raise ValueError(f"Invalid or unsupported variant combination: {active_variants}") + # 4. Route to the correct variant class + if active_variants not in layer_variants: + raise ValueError(f"Invalid or unsupported variant combination: {active_variants}") - variant_class = layer_variants[active_variants] - return variant_class() if variant_class else None + variant_class = layer_variants[active_variants] + return variant_class() if variant_class else None def update_layer( self, From e400623cd547bc428e80a32d67acd42e4de05031 Mon Sep 17 00:00:00 2001 From: isha Date: Sat, 30 May 2026 19:50:00 +0000 Subject: [PATCH 6/8] adding MonteCLORA --- src/peft/tuners/lora/config.py | 3 ++- src/peft/tuners/lora/layer.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/peft/tuners/lora/config.py b/src/peft/tuners/lora/config.py index 64866da816..09cbab932e 100644 --- a/src/peft/tuners/lora/config.py +++ b/src/peft/tuners/lora/config.py @@ -772,7 +772,8 @@ class LoraConfig(PeftConfig): "The configuration of Monteclora (Monte Carlo Low-Rank Adaptation). If passed, Monteclora will be " "used to add variational Monte Carlo sampling on top of the LoRA adapters. See `MontecloraConfig` " "for details on the individual hyperparameters." - ) + ), + "is_lora_variant": True, }, ) # Enables replicating layers in a model to expand it to a larger model. diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index b656ea8c9e..619007aaaf 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -838,6 +838,7 @@ def lora_variants(self): ("use_bdlora",): variants.BdLoraLinearVariant, ("alora_invocation_tokens",): variants.ALoraLinearVariant, ("velora_config",): variants.VeloraLinearVariant, + ("monteclora_config",): variants.MontecloraLinearVariant, } def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None: From 67fc8bb0e5cbd5b88308ecf09560f86c064be1e9 Mon Sep 17 00:00:00 2001 From: isha Date: Thu, 4 Jun 2026 16:22:28 +0000 Subject: [PATCH 7/8] Address review comments: remove getattr and revert combined with statements --- src/peft/tuners/lora/layer.py | 4 ++-- tests/test_lora_variants.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 619007aaaf..4e8df53481 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -157,8 +157,8 @@ def lora_variants(self): def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: - # Safely fetch the dictionary (defaults to empty if a subclass forgot to define it) - layer_variants = getattr(self, "lora_variants", {(): None}) + # Fetch the dictionary of variants + layer_variants = self.lora_variants lora_variants_configs = [f for f in dataclasses.fields(config) if f.metadata.get("is_lora_variant")] # 1. Gather all valid variant field names from the config diff --git a/tests/test_lora_variants.py b/tests/test_lora_variants.py index a98420e229..ace6b3148f 100644 --- a/tests/test_lora_variants.py +++ b/tests/test_lora_variants.py @@ -255,8 +255,9 @@ def test_alora_activation_matches_base_until_invocation(self): input_ids = torch.tensor([[0, 1, 2, 3]]) start = 2 - with lora_model.disable_adapter(), torch.no_grad(): - base_out = lora_model(X=input_ids) + with lora_model.disable_adapter(): + with torch.no_grad(): + base_out = lora_model(X=input_ids) kwargs = get_alora_offsets_for_forward(lora_model, input_ids) with torch.no_grad(): @@ -301,8 +302,9 @@ def test_num_beams_error(self): lora_model.eval() input_ids = torch.tensor([[0, 1, 2, 3]]) - with pytest.raises(ValueError) as e, torch.no_grad(): - lora_out = lora_model(X=input_ids, num_beams=2, alora_offsets=[3]) + with pytest.raises(ValueError) as e: + with torch.no_grad(): + lora_model(X=input_ids, num_beams=2, alora_offsets=[3]) assert "Beam search not yet supported for aLoRA." in str(e.value) def test_gradient_checkpointing_double_forward_raises(self): From d2a45452e74c582e8aacc3e6f0507119fb1a14f9 Mon Sep 17 00:00:00 2001 From: isha Date: Thu, 4 Jun 2026 17:17:55 +0000 Subject: [PATCH 8/8] apply make quality formatting --- src/peft/tuners/lora/config.py | 4 +++- src/peft/tuners/lora/layer.py | 22 +++++++++++----------- tests/test_lora_variants.py | 24 ++++++++++++++---------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/peft/tuners/lora/config.py b/src/peft/tuners/lora/config.py index 09cbab932e..f46e0dd11c 100644 --- a/src/peft/tuners/lora/config.py +++ b/src/peft/tuners/lora/config.py @@ -836,7 +836,9 @@ class LoraConfig(PeftConfig): }, ) arrow_config: Optional[ArrowConfig] = field( - default=None, metadata={"help": "The necessary config to apply arrow routing on the model.", + default=None, + metadata={ + "help": "The necessary config to apply arrow routing on the model.", "is_lora_variant": True, }, ) diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 4e8df53481..3ea6433db4 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -145,13 +145,11 @@ def lora_variants(self): """ A dictionary mapping the active LoRA variants to their respective classes. - To extend this, subclasses should override this property and return a dictionary - where the keys are tuples of variant field names (from LoraConfig) and the - values are the specific LoraVariant subclasses. + To extend this, subclasses should override this property and return a dictionary where the keys are tuples of + variant field names (from LoraConfig) and the values are the specific LoraVariant subclasses. - Tuples are used as keys because they are immutable and hashable, allowing us - to safely map combinations of active variants (e.g., DoRA + another variant) - to a specific composed variant class. + Tuples are used as keys because they are immutable and hashable, allowing us to safely map combinations of + active variants (e.g., DoRA + another variant) to a specific composed variant class. """ return {(): None} @@ -162,7 +160,7 @@ def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[Lora lora_variants_configs = [f for f in dataclasses.fields(config) if f.metadata.get("is_lora_variant")] # 1. Gather all valid variant field names from the config - tagged_fields = { f.name for f in lora_variants_configs } + tagged_fields = {f.name for f in lora_variants_configs} # 2. SANITY CHECK: Ensure all keys in the layer's dictionary actually exist in the config for variant_keys in layer_variants.keys(): @@ -174,10 +172,7 @@ def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[Lora ) # 3. Figure out which variants are currently active - active_variants = tuple(sorted( - f.name for f in lora_variants_configs - if getattr(config, f.name) - )) + active_variants = tuple(sorted(f.name for f in lora_variants_configs if getattr(config, f.name))) # 4. Route to the correct variant class if active_variants not in layer_variants: @@ -831,6 +826,7 @@ def __init__( @property def lora_variants(self): from . import variants + return { (): None, ("use_dora",): variants.DoraLinearVariant, @@ -1079,6 +1075,7 @@ def __init__( @property def lora_variants(self): from . import variants + return { (): None, ("use_dora",): variants.DoraEmbeddingVariant, @@ -1686,6 +1683,7 @@ def __init__(self, *args, **kwargs): @property def lora_variants(self): from . import variants + return { (): None, ("use_dora",): variants.DoraConv2dVariant, @@ -1703,6 +1701,7 @@ def __init__(self, *args, **kwargs): @property def lora_variants(self): from . import variants + return { (): None, ("use_dora",): variants.DoraConv1dVariant, @@ -1720,6 +1719,7 @@ def __init__(self, *args, **kwargs): @property def lora_variants(self): from . import variants + return { (): None, ("use_dora",): variants.DoraConv3dVariant, diff --git a/tests/test_lora_variants.py b/tests/test_lora_variants.py index ace6b3148f..b5c354f8ea 100644 --- a/tests/test_lora_variants.py +++ b/tests/test_lora_variants.py @@ -177,18 +177,21 @@ def test_dora_params_have_gradients(self): assert getattr(peft_model.base_model.model, layer).lora_magnitude_vector["default"].weight.grad is not None def test_unregistered_variant_raises_error(self): - # 1. Create a config and dummy linear layer - config = LoraConfig() - base_layer = nn.Linear(10, 10) - layer = LoraLinear(base_layer, "default", config, r=8, lora_alpha=8) + # 1. Create a config and dummy linear layer + config = LoraConfig() + base_layer = nn.Linear(10, 10) + layer = LoraLinear(base_layer, "default", config, r=8, lora_alpha=8) - # 2. Monkey-patch the lora_variants property to include a fake variant - with patch("peft.tuners.lora.layer.Linear.lora_variants", new_callable=PropertyMock) as mock_variants: - mock_variants.return_value = {("fake_unregistered_variant",): None} + # 2. Monkey-patch the lora_variants property to include a fake variant + with patch("peft.tuners.lora.layer.Linear.lora_variants", new_callable=PropertyMock) as mock_variants: + mock_variants.return_value = {("fake_unregistered_variant",): None} - # 3. Assert that the sanity check catches it and throws the right error - with pytest.raises(ValueError, match="Variant 'fake_unregistered_variant' found in lora_variants but it is not tagged with 'is_lora_variant' in LoraConfig."): - layer.resolve_lora_variant(config=config) + # 3. Assert that the sanity check catches it and throws the right error + with pytest.raises( + ValueError, + match="Variant 'fake_unregistered_variant' found in lora_variants but it is not tagged with 'is_lora_variant' in LoraConfig.", + ): + layer.resolve_lora_variant(config=config) def test_invalid_variant_combination_raises_error(self): # 1. Create a config with no variants active @@ -205,6 +208,7 @@ def test_invalid_variant_combination_raises_error(self): with pytest.raises(ValueError, match="Invalid or unsupported variant combination"): layer.resolve_lora_variant(config=config) + class TestActivatedLora: @pytest.mark.parametrize( "input_ids, alora_invocation_tokens, expected_offsets",