bi0ddlmZddlZddlmZddlmZddlmZddl Z ddl m Z ddl m Z ddl mZmZmZddlmZmZmZd d lmZd d lmZmZGd d eZy)) annotationsN)asdict)Enum)Optional)tqdm) BaseTunerBaseTunerLayercheck_target_module_exists)3TRANSFORMERS_MODELS_TO_SHIRA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules) ShiraConfig)Linear ShiraLayerceZdZUdZdZded<ddZedZdZ edZ dd Z ed Z dfd Z ddd Zdd ZdZdZdZedZ d ddZddZ d ddZdZxZS) ShiraModela Creates a Sparse High Rank Adapter (SHiRA) Model from a pretrained model. Args: model ([`~transformers.PreTrainedModel`]): The model to be adapted. config ([`ShiraConfig`]): The configuration of the SHiRA model. adapter_name (`str`): The name of the adapter, defaults to `"default"`. Returns: `torch.nn.Module`: The SHiRA model. Example: ```py >>> from transformers import AutoModelForCausalLM >>> from peft import ShiraConfig, get_peft_model >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m") >>> config = ShiraConfig(r=32) >>> model = get_peft_model(base_model, config) ``` **Attributes**: - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted. - **peft_config** ([`ShiraConfig`]): The configuration of the SHiRA model. shira_strprefixcL|jjD]}||us y)z A helper method to check the config when a new adapter is being added. Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters. N) peft_configvalues)selfconfigexisting_configs R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/shira/model.py_check_new_adapter_configz$ShiraModel._check_new_adapter_configCs- $//668 O&( ct||SN)r ) shira_configkeys r_check_target_module_existsz&ShiraModel._check_target_module_existsOs),<? ?vv&B6;;d+Bv  ! !X -$0$<$z-ShiraModel._replace_module..sI188t+Is) setattrr+r0weightr&getattrrDtorGtorch named_modulesany parameters)r: child_namerAchildnamemodulerEs @rr6zShiraModel._replace_module|s J/ 5, '$$Ez<0 % J uf%"'**  5'4 ( 4z<0.3kk %%+#(;;  MM%,,-- .||F#&446 3LD&4IV5F5F5HIIIIell112 3rc`|jD]\}}|j|vsd|_y)NF)named_parametersr requires_grad)rmodelnrIs r _mark_only_adapters_as_trainablez+ShiraModel._mark_only_adapters_as_trainables1**, (DAq{{!#"' (rc |j}|jdd}t|tr|j }n|}t|t j jr!|r.tjddx}|_ntd|d|j|j||jfi|nd}t||||j|fd|ji|}|S)Nr&Fzjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.zTarget module zZ is not supported. Currently, only the following modules are supported: `torch.nn.Linear`.r))fan_in_fan_outpopr.r get_base_layerrOnnrwarningswarnr*r/r1r)) r"r7r8r=r^_target_base_layerr@rAs rr3zShiraModel._create_new_modules%44 JJvu % fn - & 5 5 7  &  ' 9 7@ED!< )%% ##/ !L !2LNN Mf M     NN   &22   rcz t||S#t$r|dk(rt|j|cYSwxYw)z1Forward missing attributes to the wrapped module.rZ)super __getattr__AttributeErrorrMrZ)rrU __class__s rrhzShiraModel.__getattr__sB -7&t, , -w4::t, , -s %::c i}|jjD]U\}}t|jDcic]$\}}|t|tr |j n|&}}}|sQd|d<W|<|Scc}}w)NTinference_mode)rr-rr.rvalue)r inference config_dictr#rmr>r?rs rget_peft_config_as_dictz"ShiraModel.get_peft_config_as_dicts **002 0JCKQRW=K^K^K`a41aaJq$$7Q>aFa+/'( 0" C bs)A<c|jjD]*}t|ttfs|j |,yr!)rZmodulesr.r r enable_adapters)renabledrVs r_set_adapter_layerszShiraModel._set_adapter_layerss<jj((* 0F&>3G"HI&&w/ 0rc(|jdy)NTrtrurs renable_adapter_layersz ShiraModel.enable_adapter_layerss    .rc(|jdy)NFrwrxrys rdisable_adapter_layersz!ShiraModel.disable_adapter_layerss    /rc|jjD]U}t|ts|jr%t j d|j|j|W||_ y)NzJAdapter cannot be set when the model is merged. Unmerging the model first.) rZrrr.rmergedrbrcunmerge set_adapterr4)rr7rVs rrzShiraModel.set_adapters^jj((* 1F&*-==MM"noNN$""<0  1 +rc~|j0|dtvr tdtt|d|_|S)N model_typez0Please specify `target_modules` in `peft_config`)target_modulesr r*set)r model_configs r_prepare_adapter_configz"ShiraModel._prepare_adapter_configsK  % % -L)1dd !STT),CLQ]D^_*K &rc|jjDcgc] \}}d|vs |}}}d|rdndzdz}t|| |D]} t|j|\} } } t | dr8|r| j |||j| | | j| bt| tsst| | | j| j|jScc}}w#t$rYwxYw) Nshiraz Unloading z and merging rZ)disabledescr0) safe_merge adapter_names)rZrPrr rir+merger6r`r.r rKmodules_to_saver4) rr progressbarrrr#rdkey_listrr:r8r9s r_unload_and_optionally_mergez'ShiraModel._unload_and_optionally_merges'+jj&>&>&@WFCGSVDVCWW~B?'Ik/E \C .=djj#.N+ v|,LLJmLT$$V[&:O:O:QSYZF$89 V-C-CFDYDY-Z[ \zz#X "  s C2C2C88 DDc|t|jjvrtd|d|j|=|jj Dcgc] \}}d|vs |}}}d}|D]P}t |j|\}}}t|ts.|j||B|jdd}R|xsg|_ ycc}}w)z Deletes an existing adapter. Args: adapter_name (str): Name of the adapter to be deleted. zAdapter z does not existrN) listrkeysr*rZrPr r.rdelete_adapterr4)rr7r#rdr new_adapterr8s rrzShiraModel.delete_adapters tD$4$4$9$9$;< <x ~_EF F   \ *'+jj&>&>&@WFCGSVDVCWW  ;C*4::s;LAvq&*-%%l3&"("7"7":K  ;*/RXs  C,Cc*|j|||S)aC This method merges the Shira layers into the base model. This is needed if someone wants to use the base model as a standalone model. Args: progressbar (`bool`): whether to show a progressbar indicating the unload and merge process safe_merge (`bool`): whether to activate the safe merging check to check if there is any potential Nan in the adapter weights adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Example: ```py >>> from transformers import AutoModelForCausalLM >>> from peft import ShiraConfig, get_peft_model >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m") >>> config = ShiraConfig(r=32) >>> model = get_peft_model(base_model, config) >>> ## [Train the adapter] ## >>> merged_model = model.merge_and_unload() ``` )rrrr)rrrrs rmerge_and_unloadzShiraModel.merge_and_unload*s#<00# -1  rc&|jdS)z Gets back the base model by removing all the Shira modules without merging. This gives back the original base model. F)rrrys runloadzShiraModel.unloadLs 00u0==r)rrreturnNone)rZz nn.Modulerr)rUr)F)rnbool)T)TFFN)rrrrrzOptional[list[str]])r7r)FFN)__name__ __module__ __qualname____doc__r__annotations__r staticmethodr$rBr6r\r3rhrprurzr|rrrrrr __classcell__)rjs@rrr%s6FC =='JR338( ''R-0 /0+! -1   + 600im    59  Re  D>rr) __future__rrb dataclassesrenumrtypingrrOtorch.nnrarpeft.tuners.tuners_utilsrr r peft.utilsr r r rrlayerrrrrrrsE# ZZ  %l>l>r