biHddlmZddlZddlmZddlmZddlmZddl Z ddl m Z ddl m Z ddl mZddlmZmZmZdd lmZmZmZd d lmZd d lmZmZGd deZy)) annotationsN)asdict)Enum)Optional)tqdm)Conv1D) BaseTunerBaseTunerLayercheck_target_module_exists)4TRANSFORMERS_MODELS_TO_VBLORA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules) VBLoRAConfig)Linear VBLoRALayerceZdZUdZdZded<ddZddZddZe dZ d Z e d Z dd Z e d Zdfd Zdd dZd!d"dZd#dZd#dZd$dZe dZ d% d&dZd'dZ d( d)dZdZd*d+dZd#dZxZS), VBLoRAModela Creates VBLoRA model from a pretrained transformers model. The method is described in detail in https://huggingface.co/papers/2405.15179. Args: model ([`~transformers.PreTrainedModel`]): The model to be adapted. config ([`VBLoRAConfig`]): The configuration of the VBLoRA model. adapter_name (`str`): The name of the adapter, defaults to `"default"`. low_cpu_mem_usage (`bool`, `optional`, defaults to `False`): Create empty adapter weights on meta device. Useful to speed up the loading process. Returns: `torch.nn.Module`: The VBLoRA model. Example: ```py >>> from transformers import AutoModelForCausalLM >>> from peft import VBLoRAConfig, get_peft_model >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m") >>> config = VBLoRAConfig( ... task_type="SEQ_CLS", ... r=4, ... target_modules=["fc1", "fc2", "k_proj", "out_proj", "q_proj", "v_proj"], ... num_vectors=60, ... vector_length=256, ... save_only_topk_weights=True, ... ) >>> model = get_peft_model(base_model, config) ``` **Attributes**: - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted. - **peft_config** ([`VBLoRAConfig`]): The configuration of the VBLoRAConfig model. vblora_strprefixctj|j|j}tjj j ||j |j||j|<yN) torchzeros num_vectors vector_lengthnninituniform_init_vector_bank_boundvblora_vector_bank)selfconfig adapter_namer"s S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/vblora/model.py_init_vblora_vector_bankz$VBLoRAModel._init_vblora_vector_bankJsZ"[[););V=Q=QR  1F4Q4Q3QSYSpSpq0B -c8tji|_yr)r ParameterDictr")r#modelr$r%s r&_pre_injection_hookzVBLoRAModel._pre_injection_hookOs"$"2"22"6r(ct|jdkDr2|jdk7r"t|jj dyy)z A helper method to check the config when a new adapter is being added. Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters. rnonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)len peft_configbias ValueError __class____name__)r#r$s r&_check_new_adapter_configz%VBLoRAModel._check_new_adapter_configRsR   !A %FKK6,A>>**+,77 -B %r(ct||Sr)r ) vblora_configkeys r&_check_target_module_existsz'VBLoRAModel._check_target_module_existsbs)-==r(c  | tdt|dxr|jdu}|j|d}|j ||t |t r`|j||j|j|j|j|j|j|jy|jd||j||d|} ||j vr| j#d|j%||| |y)NzCurrent Key shouldn't be `None`r1)fan_in_fan_outr1)r%r"rtopkrrvblora_dropoutinit_logits_std)r7r"r%targetF)r2hasattrr1r;r' isinstancer update_layerr"r<r=rrr>r?_create_new_moduleactive_adapterrequires_grad__replace_module) r#r7r%r@ target_nameparent current_keyr1kwargs new_modules r&_create_and_replacezVBLoRAModel._create_and_replacefs  >? ?vv&B6;;d+B+::  %%m\B ff %   )#'#:#://"'')55+99,;; - = = 100+#'#:#:)   J4#6#66))%0  j& Ir(ct|||t|dr |j}t|ds.|j|_t|dr|j|_t |dd^t|dr|j |j_n|j |_|j|jjtjd|jD]R\}}d|vs tfd|jDr.|j|jjTy)N base_layerr1statemetarc3<K|]}|jk(ywr)device).0prRs r& z.VBLoRAModel._replace_module..sI188t+Is) setattrrBrPweightr1getattrrQtorTr named_modulesany parameters)rJ child_namerMchildnamemodulerRs @r&rHzVBLoRAModel._replace_modules J/ 5, '$$Ez<0 % J uf%"'**  5'4 ( 4z<0.3kk %%+#(;;  MM%,,-- .||F#&446 3LD&D IV5F5F5HIIIIell112 3r(c|jD]\}}|j|vsd|_|jD]}|j|j }|dk(r"|dk(r%|jD]\}}d|vs d|_L|dk(rR|j D]>}t|tst|ds!|j .d|j _@td|dy) NFr.allr1T vblora_onlyzRequested bias: z, is not implemented.) named_parametersr requires_gradactive_adaptersr0r1modulesrCrrBNotImplementedError)r#r+nrVrFr1ms r& _mark_only_adapters_as_trainablez,VBLoRAModel._mark_only_adapters_as_trainables**, (DAq{{!#"' (#22 ZN##N388Dv~u}!224/DAq{*./&4A!![1ga6HQVVM_/3,4*,r?rA)rCr get_base_layerrrrwarningswarnr;rr2r<rrr=r>r?)r7r"r%r@rLtarget_base_layerrMs r&rEzVBLoRAModel._create_new_modules* fn - & 5 5 7  &  ' 9&' 7KPO'(=+G )6 204F, -*+ wKON'(=+G )JJ   1%oo%11'55##(77)99    r(cz t||S#t$r|dk(rt|j|cYSwxYw)z1Forward missing attributes to the wrapped module.r+)super __getattr__AttributeErrorrZr+)r#rar3s r&rvzVBLoRAModel.__getattr__sB -7&t, , -w4::t, , -s %::c i}|jjD]U\}}t|jDcic]$\}}|t|tr |j n|&}}}|sQd|d<W|<|Scc}}w)NTinference_mode)r0itemsrrCrvalue)r# inference config_dictr8r{kvr$s r&get_peft_config_as_dictz#VBLoRAModel.get_peft_config_as_dicts **002 0JCKQRW=K^K^K`a41aaJq$$7Q>aFa+/'( 0" C bs)A<c|jjD]*}t|ttfs|j |,yr)r+rirCr r enable_adapters)r#enabledrbs r&_set_adapter_layerszVBLoRAModel._set_adapter_layerss<jj((* 0F&>3G"HI&&w/ 0r(c(|jdy)zyEnable all adapters. Call this if you have previously disabled all adapters and want to re-enable them. TrN)rr#s r&enable_adapter_layersz!VBLoRAModel.enable_adapter_layerss    .r(c|jD]<}|j|j}|dk7s"d|d}tj|>|j dy)zDisable all adapters. When disabling all adapters, the model output corresponds to the output of the base model. r.z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.FrN)rhr0r1rqrrr)r#rFvalmsgs r&disable_adapter_layersz"VBLoRAModel.disable_adapter_layers sr #22 #N"">277Cf}TUXTYZGG c" #    /r(c|jjD]U}t|ts|jr%t j d|j|j|W||_ y)aSet the active adapter(s). Additionally, this function will set the specified adapters to trainable (i.e., requires_grad=True). If this is not desired, use the following code. ```py >>> for name, param in model_peft.named_parameters(): ... if ...: # some check on name (ex. if 'lora' in name) ... param.requires_grad = False ``` Args: adapter_name (`str` or `list[str]`): Name of the adapter(s) to be activated. zJAdapter cannot be set when the model is merged. Unmerging the model first.N) r+rirCrmergedrqrrunmerge set_adapterrF)r#r%rbs r&rzVBLoRAModel.set_adapters`jj((* 1F&+.==MM"noNN$""<0  1 +r(c~|j0|dtvr tdtt|d|_|S)N model_typez0Please specify `target_modules` in `peft_config`)target_modulesr r2set)r0 model_configs r&_prepare_adapter_configz#VBLoRAModel._prepare_adapter_config0sK  % % -L)1ee !STT),D\R^E_`*K &r(c"|jjDcgc]\}}|j|vs|}}}d|rdndzdz}t|| |D]} t |j|\} } } t | dr8|r| j|||j| | | j| bt| tsst| | | j| j|jScc}}w#t $rYwxYw)Nz Unloading z and merging r+)disabledescrP) safe_merge adapter_names)r+r\rrrrwrBmergerHrprCr rXmodules_to_saverF) r#r progressbarrrr8_key_listrrJr@rIs r&_unload_and_optionally_mergez(VBLoRAModel._unload_and_optionally_merge:s '+jj&>&>&@[FCDKKWZDZC[[~B?'Ik/E \C .=djj#.N+ v|,LLJmLT$$V[&:O:O:QSYZF$89 V-C-CFDYDY-Z[ \zz#\ "  sC<C<D D Dc|t|jjvrtd|d|j|=|jj Dcgc]\}}|j |vs|}}}d}|D]P}t|j|\}}}t|ts.|j||B|jdd}R|xsg|_ |j||ycc}}w)z Deletes an existing adapter. Args: adapter_name (str): Name of the adapter to be deleted. zAdapter z does not existN)new_active_adapters) listr0keysr2r+r\rrrCrdelete_adapterrF_delete_auxiliary_adapter)r#r%r8rr new_adapterr@s r&rzVBLoRAModel.delete_adapterTs tD$4$4$9$9$;< <x ~_EF F   \ *&*jj&>&>&@[FCDKKWZDZC[[  ;C*4::s;LAvq&+.%%l3&"("7"7":K  ;*/R &&|&U\s C36C3c*|j|||S)aJ This method merges the VBLoRA layers into the base model. This is needed if someone wants to use the base model as a standalone model. Args: progressbar (`bool`): whether to show a progressbar indicating the unload and merge process safe_merge (`bool`): whether to activate the safe merging check to check if there is any potential Nan in the adapter weights adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Example: ```py >>> from transformers import AutoModelForCausalLM >>> from peft import PeftModel >>> base_model = AutoModelForCausalLM.from_pretrained("tiiuae/falcon-40b") >>> peft_model_id = "smangrul/falcon-40B-int4-peft-lora-sfttrainer-sample" >>> model = PeftModel.from_pretrained(base_model, peft_model_id) >>> merged_model = model.merge_and_unload() ``` )rrrr)r#rrrs r&merge_and_unloadzVBLoRAModel.merge_and_unloadks#:00# -1  r(c&|jdS)z Gets back the base model by removing all the VBLoRA modules without merging. This gives back the original base model. F)rrrs r&unloadzVBLoRAModel.unloads 00u0==r(cd}d}d}|jD]U\}}d|vr||jz }d|vr||jz }6|jsC||jz }W|j|jr|j|j }d}|dkrd}n|dkrd}n |d krd}nd }||j|j z |j|j dz z} ||j|j z |j|j z|z} t|| z| z} | |fS||z} | |fS) z` Returns the number of savable VB-LoRA parameters and other savable parameters. r vblora_logitsr"rg?ig?l)rfnumelrgr0save_only_topk_weightsrr=int) r#adapter logits_paramsvector_bank_params other_paramsraparamrfactortopk_weight_paramstopk_indices_params vblora_paramss r&get_nb_savable_parametersz%VBLoRAModel.get_nb_savable_parameterss  002 .KD%$&. %-"ekkm3"$$ -  .   G $ ; ;**73??KFT!u$u$ 0 0 9 E EEIYIYZaIbIgIgjkIkl  0 0 9 E EEHXHXY`HaHfHffioo  25G GJ] ]^Ml**/>Ml**r(cV|j\}}td|dd||zdy)z_ Prints the number of savable VB-LoRA parameters and total savable parameters. z1VB-LoRA params to-be-saved (float32-equivalent): z,dz || total params to-be-saved: N)rprint)r#rrs r&print_savable_parametersz$VBLoRAModel.print_savable_parameterssB'+&D&D&F# | ? b?QR,-:\-I2+N P r()r$rr%rreturnNone)r+ nn.Moduler$rr%rrr)r$rrr)r+rrr)rar)F)r|bool)T)rrrr)rr)r%zstr | list[str]rr)TFFN)rrrrrOptional[list[str]])r%rrr)FFN)rrrrrrrztorch.nn.Module)default)rztuple[int, int])r4 __module__ __qualname____doc__r__annotations__r'r,r5 staticmethodr9rNrHrmrErvrrrrrrrrrrrr __classcell__)r3s@r&rr!s$LFCC 7 >>*JX338Z(&&P-0 / 0+.! -1   + 4V0im  59 Re  B>"+H r(r) __future__rrq dataclassesrenumrtypingrrtorch.nnrrtransformers.pytorch_utilsrpeft.tuners.tuners_utilsr r r peft.utilsr r rr$rlayerrrrrAr(r&rsB# -ZZrr &^ )^ r(