Spaces:
Paused
Paused
File size: 8,660 Bytes
2f5127c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 |
# Copyright 2020-2025 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class ModelConfig:
"""
Configuration class for the models.
Using [`~transformers.HfArgumentParser`] we can turn this class into
[argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
command line.
Parameters:
model_name_or_path (`str` or `None`, *optional*, defaults to `None`):
Model checkpoint for weights initialization.
model_revision (`str`, *optional*, defaults to `"main"`):
Specific model version to use. It can be a branch name, a tag name, or a commit id.
torch_dtype (`Literal["auto", "bfloat16", "float16", "float32"]` or `None`, *optional*, defaults to `None`):
Override the default `torch.dtype` and load the model under this dtype. Possible values are
- `"bfloat16"`: `torch.bfloat16`
- `"float16"`: `torch.float16`
- `"float32"`: `torch.float32`
- `"auto"`: Automatically derive the dtype from the model's weights.
trust_remote_code (`bool`, *optional*, defaults to `False`):
Whether to allow for custom models defined on the Hub in their own modeling files. This option should only
be set to `True` for repositories you trust and in which you have read the code, as it will execute code
present on the Hub on your local machine.
attn_implementation (`str` or `None`, *optional*, defaults to `None`):
Which attention implementation to use. You can run `--attn_implementation=flash_attention_2`, in which case
you must install this manually by running `pip install flash-attn --no-build-isolation`.
use_peft (`bool`, *optional*, defaults to `False`):
Whether to use PEFT for training.
lora_r (`int`, *optional*, defaults to `16`):
LoRA R value.
lora_alpha (`int`, *optional*, defaults to `32`):
LoRA alpha.
lora_dropout (`float`, *optional*, defaults to `0.05`):
LoRA dropout.
lora_target_modules (`Union[str, list[str]]` or `None`, *optional*, defaults to `None`):
LoRA target modules.
lora_modules_to_save (`list[str]` or `None`, *optional*, defaults to `None`):
Model layers to unfreeze & train.
lora_task_type (`str`, *optional*, defaults to `"CAUSAL_LM"`):
Task type to pass for LoRA (use `"SEQ_CLS"` for reward modeling).
use_rslora (`bool`, *optional*, defaults to `False`):
Whether to use Rank-Stabilized LoRA, which sets the adapter scaling factor to `lora_alpha/√r`, instead of
the original default value of `lora_alpha/r`.
use_dora (`bool`, *optional*, defaults to `False`):
Enable [Weight-Decomposed Low-Rank Adaptation (DoRA)](https://huggingface.co/papers/2402.09353). This
technique decomposes the updates of the weights into two parts, magnitude and direction. Direction is
handled by normal LoRA, whereas the magnitude is handled by a separate learnable parameter. This can
improve the performance of LoRA, especially at low ranks. Right now, DoRA only supports linear and Conv2D
layers. DoRA introduces a bigger overhead than pure LoRA, so it is recommended to merge weights for
inference.
load_in_8bit (`bool`, *optional*, defaults to `False`):
Whether to use 8 bit precision for the base model. Works only with LoRA.
load_in_4bit (`bool`, *optional*, defaults to `False`):
Whether to use 4 bit precision for the base model. Works only with LoRA.
bnb_4bit_quant_type (`str`, *optional*, defaults to `"nf4"`):
Quantization type (`"fp4"` or `"nf4"`).
use_bnb_nested_quant (`bool`, *optional*, defaults to `False`):
Whether to use nested quantization.
"""
model_name_or_path: Optional[str] = field(
default=None,
metadata={"help": "Model checkpoint for weights initialization."},
)
model_revision: str = field(
default="main",
metadata={"help": "Specific model version to use. It can be a branch name, a tag name, or a commit id."},
)
torch_dtype: Optional[str] = field(
default=None,
metadata={
"help": "Override the default `torch.dtype` and load the model under this dtype.",
"choices": ["auto", "bfloat16", "float16", "float32"],
},
)
trust_remote_code: bool = field(
default=False,
metadata={
"help": "Whether to allow for custom models defined on the Hub in their own modeling files. This option "
"should only be set to `True` for repositories you trust and in which you have read the code, as it will "
"execute code present on the Hub on your local machine."
},
)
attn_implementation: Optional[str] = field(
default=None,
metadata={
"help": "Which attention implementation to use. You can run `--attn_implementation=flash_attention_2`, in "
"which case you must install this manually by running `pip install flash-attn --no-build-isolation`."
},
)
use_peft: bool = field(
default=False,
metadata={"help": "Whether to use PEFT for training."},
)
lora_r: int = field(
default=16,
metadata={"help": "LoRA R value."},
)
lora_alpha: int = field(
default=32,
metadata={"help": "LoRA alpha."},
)
lora_dropout: float = field(
default=0.05,
metadata={"help": "LoRA dropout."},
)
lora_target_modules: Optional[list[str]] = field(
default=None,
metadata={"help": "LoRA target modules."},
)
lora_modules_to_save: Optional[list[str]] = field(
default=None,
metadata={"help": "Model layers to unfreeze & train."},
)
lora_task_type: str = field(
default="CAUSAL_LM",
metadata={"help": "Task type to pass for LoRA (use 'SEQ_CLS' for reward modeling)."},
)
use_rslora: bool = field(
default=False,
metadata={
"help": "Whether to use Rank-Stabilized LoRA, which sets the adapter scaling factor to `lora_alpha/√r`, "
"instead of the original default value of `lora_alpha/r`."
},
)
use_dora: bool = field(
default=False,
metadata={
"help": "Enable Weight-Decomposed Low-Rank Adaptation (DoRA). This technique decomposes the updates of "
"the weights into two parts, magnitude and direction. Direction is handled by normal LoRA, whereas the "
"magnitude is handled by a separate learnable parameter. This can improve the performance of LoRA, "
"especially at low ranks. Right now, DoRA only supports linear and Conv2D layers. DoRA introduces a "
"bigger overhead than pure LoRA, so it is recommended to merge weights for inference."
},
)
load_in_8bit: bool = field(
default=False,
metadata={"help": "Whether to use 8 bit precision for the base model. Works only with LoRA."},
)
load_in_4bit: bool = field(
default=False,
metadata={"help": "Whether to use 4 bit precision for the base model. Works only with LoRA."},
)
bnb_4bit_quant_type: str = field(
default="nf4",
metadata={"help": "Quantization type.", "choices": ["fp4", "nf4"]},
)
use_bnb_nested_quant: bool = field(
default=False,
metadata={"help": "Whether to use nested quantization."},
)
def __post_init__(self):
if self.load_in_8bit and self.load_in_4bit:
raise ValueError("You can't use 8 bit and 4 bit precision at the same time")
if hasattr(self.lora_target_modules, "__len__") and len(self.lora_target_modules) == 1:
self.lora_target_modules = self.lora_target_modules[0]
|