bi-ddlmZddlZddlmZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZddlmZmZmZdd lmZGd d e Zy) ) annotationsN)Optional)tqdm) PeftConfig) BaseTunerBaseTunerLayercheck_target_module_exists onload_layer)AuxiliaryTrainingWrapper_get_input_embeddings_name_get_submodules)TrainableTokensLayerc.eZdZUdZded<dfd ZdZ d dfd ZdZ ddZ dd Z dd Z e d Z d Zdd ZdddZddZddZddZd dZ d! d"dZ d# d$dZxZS)%TrainableTokensModeltrainable_tokens_strprefixcn t||S#t$rt|j|cYSwxYw)z1Forward missing attributes to the wrapped module.)super __getattr__AttributeErrorgetattrmodel)selfname __class__s ]/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/trainable_tokens/model.pyrz TrainableTokensModel.__getattr__"s8 -7&t, , -4::t, , -s 44cT|jt|jd|_|S)N embed_tokens)target_modulesr r)r peft_config model_configs r_prepare_adapter_configz,TrainableTokensModel._prepare_adapter_config)s(  % % -)CDJJP^)_K &c t|d ||||d||j|}|jddr1|jj t |jjtr|jj Dcgc]%}dj|jddd'}}|jjdD]\} } |D cgc]} | j| s| } } | s*t|| \} }}|j|j}|jj|d<|j!||||| | dyyyycc}wcc} w) N)r adapter_nameautocast_adapter_dtypelow_cpu_mem_usagetie_word_embeddingsF.)remove_duplicate tied_adapterr)rinject_adapterget_model_configgetr_tied_weights_keys isinstanceget_input_embeddingsrjoinsplit named_modulesendswithr r"to_dict_create_and_replace_dict)rrr'r(r)kwargsr#n module_keysrmodule target_key matched_keysparenttarget target_namer"rs rr0z#TrainableTokensModel.inject_adapter0sy  %#9/    ,,T2    2E : --94::::<>RS@D @]@]^1388AGGCL"$56^K^ $ 8 8% 8 P  f=HfzDMMZdLe f f2A%2N/FFK"&"2"2<"@"H"H"JK26**2Q2Q2SK/11#$#$Q   T: ; _ gs*E,#E1:E1cgSNr/)rargsr<s r_get_tied_target_modulesz-TrainableTokensModel._get_tied_target_modulesas  r%c|}t|tr|j|fi|y|j|||fi|}|j ||||y)z The same as `_create_and_replace` but takes a dictionary instead of a peft config so that we can add keys that are not present in the config, such as `tied_adapter`. N)r4r update_layer_create_new_module_replace_module) rr"r'rCrDrB current_keyr< new_modules rr;z-TrainableTokensModel._create_and_replace_dicthsY f2 3 F   7 7000lF]V\]J  j& Ir%cP|j}|j||||||y)zc A private method to create and replace the target module with the adapter module. N)r:r;)rr"r'rCrDrBrMr<s r_create_and_replacez(TrainableTokensModel._create_and_replace}s,$$& %%flFKQWYder%ct||SrF)r )rr"keys r_check_target_module_existsz0TrainableTokensModel._check_target_module_existss)+s;;r%c vt||fi|}|j||d|d|jdd|S)N init_weights token_indicesr.)rUrVr.)rrJr2)r"r'rCr<rNs rrKz'TrainableTokensModel._create_new_modulesO)&,I&I  / 1ND9  r%ct|||t|dr |j}t|ds.|j|_t|dr|j|_t |dd^t|dr|j |j_n|j |_|j|jjtjd|jD]\\}}|j|vstfd|jDr8|j|jj^y)N base_layerbiasstatemetac3<K|]}|jk(ywrF)device).0pr[s r z7TrainableTokensModel._replace_module..sI188t+Is)setattrhasattrrXweightrYrrZtor]torchr8rany parameters)rrB child_namerNchildrr?r[s @rrLz$TrainableTokensModel._replace_modules J/ 5, '$$Ez<0 % J uf%"'**  5'4 ( 4z<0.3kk %%+#(;;  MM%,,-- .||F#&446 3LD&{{d"IV5F5F5HIIIIell112 3r%c`|jD]\}}|j|vsd|_y)NF)named_parametersr requires_grad)rrr=r_s r _mark_only_adapters_as_trainablez5TrainableTokensModel._mark_only_adapters_as_trainables1**, (DAq{{!#"' (r%c|jjD]*}t|ttfs|j |,yrF)rmodulesr4rr enable_adapters)renabledr?s r_set_adapter_layersz(TrainableTokensModel._set_adapter_layerss<jj((* 0F&>3K"LM&&w/ 0r%c(|jdy)zyEnable all adapters. Call this if you have previously disabled all adapters and want to re-enable them. TrqNrrrs renable_adapter_layersz*TrainableTokensModel.enable_adapter_layerss    .r%c(|jdy)zDisable all adapters. When disabling all adapters, the model output corresponds to the output of the base model. FrtNrurvs rdisable_adapter_layersz+TrainableTokensModel.disable_adapter_layerss    /r%c|jjD]U}t|ts|jr%t j d|j|j|W||_ y)aSet the active adapter(s). Additionally, this function will set the specified adapters to trainable (i.e., requires_grad=True). If this is not desired, use the following code. ```py >>> for name, param in model_peft.named_parameters(): ... if ...: # some check on name (ex. if 'lora' in name) ... param.requires_grad = False ``` Args: adapter_name (`str` or `list[str]`): Name of the adapter(s) to be activated. zJAdapter cannot be set when the model is merged. Unmerging the model first.N) rror4rmergedwarningswarnunmerge set_adapteractive_adapter)rr'r?s rrz TrainableTokensModel.set_adaptersajj((* 1F&"67==MM"noNN$""<0  1 +r%c&|jdS)zh Gets back the base model by removing all the trainable tokens modules without merging. F)merge_unload_and_optionally_mergervs runloadzTrainableTokensModel.unloads00u0==r%c*|j|||S)a This method merges the trained tokens into the targeted embedding layer(s) of the base model. This is needed if someone wants to use the base model as a standalone model. Args: progressbar (`bool`): whether to show a progressbar indicating the unload and merge process safe_merge (`bool`): whether to activate the safe merging check to check if there is any potential Nan in the adapter weights adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. ) progressbar safe_merge adapter_namesr)rrrrs rmerge_and_unloadz%TrainableTokensModel.merge_and_unloads#"00# -1  r%c`|jjDcgc]\}}|j|vs|}}}d|rdndzdz}t|| |D]} t |j|\} } } t | 5t| dr)| j|||} |j| | | | nCt| dr7|r| j|| |j| | | j| ddd|jScc}}w#t $rYwxYw#1swYxYw) Nz Unloading z and merging r)disabledesc"unload_and_optionally_merge_module)rrrrX)rr) rr8rrr rr rbrrLrget_base_layer) rrrrrrR_key_listrrBrCrDunloaded_modules rrz1TrainableTokensModel._unload_and_optionally_mergesG'+jj&>&>&@[FCDKKWZDZC[[~B?'Ik/E _C .=djj#.N+ f% _6#GH&,&O&O# -'P'O((ovVV\2 - X((f>S>S>UW]^ _ _  _"zz'\ "   _ _s)DDDA9D$ D! D!$D- )rr)TF) r nn.Moduler'rr(boolr)rreturnNone)r"dictr'rrCrrDrrBrrMrrr)r"rr'rrCrrDrrBrrMrrr)r"rrRrrr)rrrr)T)rqrrr)rr)r'zstr | list[str]rr)rtorch.nn.Module)FFN)rrrrrOptional[list[str]]rr)TFFN)rrrrrr)__name__ __module__ __qualname__r__annotations__rr$r0rHr;rPrS staticmethodrKrLrmrrrwryrrrr __classcell__)rs@rrrs%FC%-(,"' ///!% /  / /bJJJ J  J  JJ J* f f f f  f  f f  f<  38( 0 /0+.>im  59 Re  .! -1   + r%r) __future__rr|typingrretorch.nnnnr peft.configrpeft.tuners.tuners_utilsrrr r peft.utilsr r r layerrrr/r%rrs9# "hh\\'|9|r%