
    bi	                      d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZmZ d dlmZmZmZmZ d dlZd dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlm Z  d dl!m"Z"m#Z#m$Z$m%Z%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z, d dl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5 ed        Z6d%dZ7 G d dejp                  e      Z9 G d de      Z:	 	 	 	 	 	 d&dZ; G d d      Z<d'dZ=d(d)d Z>d*d!Z?d+d,d"Z@d-d.d#ZAd/d$ZBy)0    )annotationsN)ABCabstractmethod)contextmanagernullcontext)AnyOptionalUnionoverload)AlignDevicesHook)named_module_tensorsoffload_state_dict)nn)PreTrainedModel)Conv1D)PEFT_TYPE_TO_PREFIX_MAPPING)INCLUDE_LINEAR_LAYERS_SHORTHAND)DUMMY_MODEL_CONFIGDUMMY_TARGET_MODULESEMBEDDING_LAYER_NAMES#MIN_TARGET_MODULES_FOR_OPTIMIZATIONSEQ_CLS_HEAD_NAMES)init_empty_weights)AuxiliaryTrainingWrappermatch_target_against_key set_additional_trainable_modules)PeftTypeTaskType   )
PeftConfig)_get_submodules   )
BufferDictc              #    K   g }| j                         D ]u  \  }}|dv rt        |d      st        |j                  t              s3|j                  j
                  sJ|j                  j                  |       |j                  |       w d}t        | d      rt        | j                  d      rt        | j                  j                  t              r| j                  j                  j
                  rt        j                  d      | j                  j                  j                  j                         v r"t        | j                  j                  j                  d      r| j                  j                  j                  j                  j                  }t!        t#        | j                  j                  j                  j                        j%                               d   }||   d   }g }t&        j(                  j+                  |      D ]*  }	d	|	v r|j                  |	        n|j                  |	       , t'        j(                  j,                  | }
|
d
z   }| j                  j                  j                  | j                         d}d |D ]1  }|j                  j/                  |t        j0                  g              3 |r,t3        | j                        D ci c]  \  }}||j5                  d       c}}| j                  j                  _        t        j                  d      | j                  j                  j                  j                         v rTt        | j                  j                  j                  d      r*t7        | j                  j                  j                         | j                  j                  j/                  | j                  t        j0                  g              yyc c}}w w)a  
    A utility for modifying a module containing one or more tuners and a base layer, any of which are offloaded to the
    CPU or disk. Moves a module's sub-modules to the execution device before some action is performed, after that the
    base layer state dictionary is re-assigned (if that layer was offloaded to the disk) and finally the parameters are
    offloaded.

    If the module has no offloaded sub-modules, this function does nothing.

    Args:
        layer ('torch.nn.Module'):
            layer with tuners to be merged
    ) 
base_layer_hf_hookFr&   metadatasetr   safetensors_filez--z-mergedTNcpu)named_moduleshasattr
isinstancer'   r   offloadpre_forwardappendr&   torchdeviceoriginal_devicesvaluesweights_mapr)   indexlistdictkeysospathsplitjoinpost_forwardtensorr   tor   )layeroffloaded_modulesnamemodulebase_layer_offloadr7   module_name	file_namebase_name_arri	base_namesafetensors_filenameparams                S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/tuners_utils.pyonload_layerrO   2   s)     ++- -f%%6:&:fooGW+X]c]l]l]t]tOO''/$$V,- ul#  *-u''002BC%%-- <<5#3#3#<#<#M#M#T#T#VV[b%%119\
 $$--99AAGGEtE$4$4$=$=$I$I$Q$QRWWYZ[\]Kk*+=>IMWW]]9- (19!((+$$Q'	(
 m4I#,y#8 !!--e.>.>?!	# ?$$VU\\"-=>?  6J%JZJZ5[1
&1dED%((5/!1
!!- <<5#3#3#<#<#M#M#T#T#VV[b%%119\
 3U5E5E5N5N5Z5Z[!!..u/?/?bAQR 1
s$   *P P P JP %O:C?P c                    h d}ddh}h d}| j                   |v rbt        |d      rUt        |j                  dd      |v r;||v r6t	        d| j                    d	| d
|j                  j
                   d| d	      yyyy)z`
    Prevent applying LoRA to incompatible modules in specific architectures (e.g., Mamba).
    >   LORAXLORAADALORARANDLORAout_projconv1d>   mambamamba2	falcon_h1falcon_mambaconfig
model_typeNz[PEFT:z
] Module 'z7' is incompatible with Mamba-based models (model_type='z'). Incompatible modules: zG. Please remove it from `target_modules` to avoid compatibility issues.)	peft_typer-   getattrr[   
ValueErrorr\   )peft_configmodeltarget_namelora_like_typesincompatible_modulesmamba_model_typess         rN    _check_lora_target_modules_mambarf   v   s    
 ?O&1H 	0E8$ELL,59JJ..../z+ G  % 7 788RSgRh iXX  / K % 	1    c                      e Zd ZdZ	 	 d	 	 	 	 	 	 	 	 	 d fdZedd       ZddZddZe	dd       Z
d dZe	d!d       Ze		 d"	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d#d	       Ze	d$d
       Ze	d%d       Ze	d%d       Zd&dZd'd(dZd Zd)dZ	 	 d%dZ	 	 	 d*	 	 	 	 	 	 	 	 	 	 	 d+dZ	 	 	 	 	 	 	 	 	 	 d,dZd-d.dZd Zd/dZd0dZed1d       Zd2dZ xZS )3	BaseTunera  
    A base tuner model that provides the common methods and attributes for all tuners that are injectable into a
    torch.nn.Module

    For adding a new Tuner class, one needs to overwrite the following methods:

    - **_prepare_adapter_config**:
        A private method to eventually prepare the adapter config, for example in case the field `target_modules` is
        missing.
    - **_create_and_replace**:
        A private method to create and replace the target module with the adapter module.
    - **_check_target_module_exists**:
        A private helper method to check if the passed module's key name matches any of the target modules in the
        adapter_config.

    The easiest is to check what is done in the `peft.tuners.lora.LoraModel` class.

    Attributes:
        model (`torch.nn.Module`):
            The model to which the adapter tuner layers will be attached.
        forward (`Callable`):
            The forward method of the model.
        peft_config (`Union[`PeftConfig`, dict[str, PeftConfig]]`):
            The adapter configuration object, it should be a dictionary of `str` to `PeftConfig` objects. One can also
            pass a PeftConfig object and a new adapter will be created with the default name `adapter` or create a new
            dictionary with a key `adapter_name` and a value of that peft config.
        config (`dict[str, Any]`):
            The model configuration object, it should be a dictionary of `str` to `Any` objects.
        targeted_module_names (`list[str]`):
            The list of module names that were actually adapted. Can be useful to inspect if you want to quickly
            double-check that the `config.target_modules` were specified correctly.
        targeted_parameter_names (`list[str]`):
            The list of parameter names that were actually adapted. Can be useful to inspect if you want to quickly
            double-check that the `config.target_parameters` were specified correctly.
    c                d   t         |           || _        g | _        g | _        t        | d      st        |t              r||in|| _        nPt        j                  d       t        |t              r|| j                  |<   n| j                  j                  |       || _        | j                  | j                  | j                  |   |       |t        j                  k7  s||   t        j                  k7  r| j!                  | j                  |||       | j                  | j                  _        y )Nr`   zAlready found a `peft_config` attribute in the model. This will lead to having multiple adapters in the model. Make sure to know what you are doing!)low_cpu_mem_usage
state_dict)super__init__ra   targeted_module_namestargeted_parameter_namesr-   r.   r    r`   warningswarnupdateactive_adapter_pre_injection_hookr   rR   inject_adapter)selfra   r`   adapter_namerk   rl   	__class__s         rN   rn   zBaseTuner.__init__   s    	
02"35% t]+>HV`>ak:grDMMG +z21<  .   ''4/;  T-=-=l-K\Z(..(K,E,W

LL]jtu "&!1!1

rg   c                h    t        | j                  t              r| j                  gS | j                  S Nr.   rt   strrw   s    rN   active_adapterszBaseTuner.active_adapters   -    d))3/''(("""rg   c                :     | j                   j                  |i |S r{   )ra   forward)rw   argskwargss      rN   r   zBaseTuner.forward   s    !tzz!!42622rg   c                     y)a  
        A hook to be called before the adapter is injected into the model. This method can be overridden by child
        classes to perform any pre-injection operations.

        Args:
            model (`nn.Module`):
                The model to be adapted.
            config (`PeftConfig`):
                The adapter config.
            adapter_name (`str`):
                The adapter name.
        N )rw   ra   r[   rx   s       rN   ru   zBaseTuner._pre_injection_hook   s     	rg   c                     y)a  
        A private method to eventually prepare the adapter config. For transformers based models, if
        `peft_config.target_modules` is None, we can automatically infer the target modules from the
        `TRANSFORMERS_MODELS_TO_XXX_TARGET_MODULES_MAPPING`. This method can be further refactored in the future to
        automatically infer it for all tuner models.

        Check out `peft.tuner.lora.LoraModel._prepare_adapter_config` for an example.

        Args:
            peft_config (`PeftConfig`):
                The adapter config.
            model_config (`dict`):
                The transformers model config, that config should contain the `model_type` key.
        Nr   )rw   r`   model_configs      rN   _prepare_adapter_configz!BaseTuner._prepare_adapter_config   s      	rg   c                     y)a_  
        A private method to modify the model structure before adapter is applied.

        See `peft.tuner.lora.LoraModel._prepare_model` for an example.

        Args:
            peft_config (`PeftConfig`):
                The prepared adapter config.
            model (`nn.Module`):
                The model that is going to be adapted.
        Nr   )rw   r`   ra   s      rN   _prepare_modelzBaseTuner._prepare_model  s     	rg   c                     y)aq  
        A helper private method to check if the passed module's key name matches any of the target modules in the
        `peft_config.target_modules` list. If it does, return `True`, else return `False`.

        Args:
            peft_config (`PeftConfig`):
                The adapter config.
            key (`str`):
                The module's key name.
        Nr   )r`   keys     rN   _check_target_module_existsz%BaseTuner._check_target_module_exists  s     	rg   c                     y)aQ  
        Inplace replacement of the target module with the adapter layer. This method needs to be overridden by all the
        tuner classes.

        Check `peft.tuners.lora.LoraModel._create_and_replace` for an example.

        Args:
            peft_config (`PeftConfig`):
                The adapter config.
            adapter_name (`str`):
                The adapter name.
            target (`nn.Module`):
                The target module.
            target_name (`str`):
                The target module's name.
            parent (`nn.Module`):
                The parent module.
            current_key (`str`):
                The key of the current target being adapted.
            parameter_name (`str`, *optional*)
                If, and only if, an `nn.Parameter` is being targeted, this is the name of the parameter.
        Nr   )rw   r`   rx   targetrb   parentcurrent_keyparameter_names           rN   _create_and_replacezBaseTuner._create_and_replace  s    B 	rg   c                     y)a)  
        A helper method to mark only the adapter layers as trainable (i.e. module.requires_grad = False) This needs to
        be overridden for all tuner classes to match the correct key names.

        Check `peft.tuners.lora.LoraModel._mark_only_adapters_as_trainable` for an example.
        Nr   )rw   ra   s     rN    _mark_only_adapters_as_trainablez*BaseTuner._mark_only_adapters_as_trainable@  s     	rg   c                     y)z0
        Disable all adapters in-place.
        Nr   r~   s    rN   disable_adapter_layersz BaseTuner.disable_adapter_layersJ      
 	rg   c                     y)z.
        Enable all adapters in-place
        Nr   r~   s    rN   enable_adapter_layerszBaseTuner.enable_adapter_layersQ  r   rg   c                     y)z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        Nr   )rw   r[   s     rN   _check_new_adapter_configz#BaseTuner._check_new_adapter_configX  s     	rg   c                ^   |syt         j                  t         j                  h}| j                  j	                         D ]l  }t        |t              s|j	                         D ]D  }t        |t        j                  t        j                  t        f      s4||vr9t        ||   t        j                        rF||   j                  |v r4||   j                  j                  t         j                        ||   _        t        ||   t         j                         r7||   j                  |v r%||   j                  t         j                        ||<   ||   j#                         D ]?  }|j                  |v s|j                  j                  t         j                        |_        A G o y)ar  
        A helper method to cast the adapter weights to the correct dtype.

        Currently, this only upcasts float16 and bfloat16 to float32.

        Args:
            adapter_name (`str`):
                The adapter name.
            autocast_adapter_dtype (`bool`, *optional*):
                Whether to autocast the adapter dtype. Defaults to `True`.

        N)r2   float16bfloat16ra   modulesr.   BaseTunerLayerr   
ModuleDictParameterDictr#   	ParameterdtypedatarA   float32Tensor
parameters)rw   rx   autocast_adapter_dtypedtypes_to_convert_to_fp32rE   	submodulerM   s          rN   _cast_adapter_dtypezBaseTuner._cast_adapter_dtypea  si    &%*]]ENN$C!jj((* 	BFfn5#^^- B	!)bmmR=M=Mz-Z[y0i5r||D .448QQ7@7N7S7S7V7VW\WdWd7e	,/4i5u||D .448QQ2;L2I2L2LU]]2[	,/&|4??A BE{{&??%*ZZ]]5==%A
B#B		Brg   c                    t        j                  d      }| j                  | j                        }|rt	        j
                  d|d|z          yy)zHelper method to check whether the adapter can be merged.

        Raise a ValueError if it is not possible to merge the adapter with the given configuration.
        a   
            ```python
            from transformers import AutoModelForCausalLM

            # Load original tied model
            model = AutoModelForCausalLM.from_pretrained("google/gemma-2-2b-it", tie_word_embeddings=False)

            # Set the randomly initialized lm_head to the previously tied embeddings
            model.lm_head.weight.data = model.model.embed_tokens.weight.data.clone()

            # Save the untied model
            untied_model_dir = "dir/for/untied/model"
            model.save_pretrained(untied_model_dir)
            model.config.save_pretrained(untied_model_dir)

            # Now use the original model but in untied format
            model = AutoModelForCausalLM.from_pretrained(untied_model_dir)
            ```
            BModel with `tie_word_embeddings=True` and the tied_target_modules=z are part of the adapter. This can lead to complications. You can opt to merge the adapter after cloning the weights (to untie the embeddings). You can untie the embeddings by loading the model with `tie_word_embeddings=False`. For example:N)textwrapdedent_get_tied_target_modulesra   rq   rr   )rw   example_codetied_target_moduless      rN   _check_merge_allowedzBaseTuner._check_merge_allowed  sb    
  
* #;;DJJGMMUAT@V Ws s 	 rg   c                    t        |||       y)zh
        Prevent applying LoRA to incompatible modules in specific architectures (e.g., Mamba).
        N)rf   )rw   r`   ra   rb   s       rN   !_check_target_module_compatiblityz+BaseTuner._check_target_module_compatiblity  s     	)e[Irg   c                F    t        | j                  j                   d      )Nz) does not support targeting nn.Parameter.)NotImplementedErrorry   __name__)rw   r`   rx   r   rb   r   r   s          rN   _create_and_replace_parameterz'BaseTuner._create_and_replace_parameter  s"     "T^^%<%<$==f"ghhrg   c           
        | j                   |   }g }g }g }	| j                  |       | j                  |      }
| j                  ||
      }| j	                  ||       t        |dg       r|rt        d      t        |j                               }|D cg c]  \  }}|	 }}}t        |dd      t        k(  }|rg }g }t        ||      }t        |j                  t        t        f      rt        |j                        t        k\  r|j                   t"        j$                  k7  rl|D cg c]"  t'        fd|j                  D              s$ }}t)        |j                  |      }t        |      t        |j                        k  r||_        i }|D ]  \  }}t        |t*              s|||<    t               }|8t,        |j                      }|D ch c]  }|j/                  d|z   d      d    }}|D ]  \  }}|s
|D ])  }|j1                  |dz         s|j3                  |        n |r	|d	   |k(  rC|| j5                  ||      }t        |t6              r|j3                  |       y|s|j3                  |       | j8                  j3                  |       t;        ||      \  }}}| j=                  |||       |rt>        nt@        } |       5  | jC                  ||||||
       ddd       ||vr|j3                  |       ns| j8                  j3                  |       t;        ||      \  }}}| j=                  |||       |rt>        nt@        } |       5  | jC                  ||||||
       ddd       | j5                  ||      s|	j3                  |        t        |dg       r| jE                  ||||       |vt        |	      }t        | j8                        }||z
  } ||z
  }!d}"| s|!rd}"| r|"dtG        |        dz  }"|!r|"dtG        |!       dz  }"|"rtI        jJ                  |"tL               | j8                  s| jN                  s|s|r|st        d      |s|r|j                  st        d      |s]|r[d|j                   d}#t        |dd      |#d|jP                   dz  }#t        |dd      |#d|jR                   dz  }#t        |#      d}#t        |dd      |#d|jP                   dz  }#t        |dd      |#d|jR                   dz  }#t        |#      tU        |d      r2|jV                  r&|s$tI        jJ                  d|jV                   d       n|s|j                  r5| j8                  s)tI        jJ                  d|j                   dtL               nAt        |dg       r4| jN                  s(tI        jJ                  d|jX                   d tL               | j[                  |!      }$|$rtI        jJ                  d"|$d#       | j]                  | j^                         | ja                  |       | j                   |   jb                  r$|je                         D ]  \  }%}&||%v sd$|&_3         ti        ||tj        j                  |       |%       yc c}}w c c}w c c}w # 1 sw Y   xY w# 1 sw Y   TxY w)&a$  
        Creates adapter layers and replaces the target modules with the adapter layers. This method is called under the
        hood by `peft.mapping.get_peft_model` if a non-prompt tuning adapter class is passed.

        The corresponding PEFT config is directly retrieved from the `peft_config` attribute of the BaseTuner class.

        Args:
            model (`nn.Module`):
                The model to be tuned.
            adapter_name (`str`):
                The adapter name.
            autocast_adapter_dtype (`bool`, *optional*):
                Whether to autocast the adapter dtype. Defaults to `True`.
            low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
                Create empty adapter weights on meta device. Useful to speed up the loading process.
            state_dict (`dict`, *optional*, defaults to `None`)
                If a state_dict is passed here, the adapters will be injected based on the entries of the state_dict.
                This can be useful when the exact `target_modules` of the PEFT method is unknown, for instance because
                the checkpoint was created without meta data. Note that the values from the state_dict are not used,
                only the keys are used to determine the correct layers that should be adapted.

        target_parameterszTrying to inject a PEFT adapter from a state_dict but the PEFT config uses `target_parameters`. This is not supported -- when using `target_parameters`, please inject the adapter without the state_dict.target_modulesNc              3  T   K   | ]  }|k(  xs j                  d |z          ! yw.Nendswith).0suffixrD   s     rN   	<genexpr>z+BaseTuner.inject_adapter.<locals>.<genexpr>  s*     tvDFNJt}}S6\/JJts   %(r   r"   r   )r   )r`   ra   rx   rk   r%   zWhile injecting the PEFT adapters, an inconsistency was discovered between the PEFT config and the provided state_dict. This is not necessarily an issue and can be ignored if this was the intent. z;The PEFT config contained these additional target modules: z. z:The state_dict contained these additional target modules: zAll modules were excluded. This is likely unintended. Check your `target_modules`, `exclude_modules` and `modules_to_save` configuration.znNo `target_modules` passed but also no `target_parameters` found. Please check the values for these arguments.zTarget modules zL not found in the base model. Please check the target modules and try again.layers_to_transformz, Note: You specified 'layers_to_transform': layers_patternz& You also specified 'layers_pattern': a"  No modules were targeted for adaptation. This might be caused by a combination of mismatched target modules and excluded modules. Please check your `target_modules` and `exclude_modules` configuration. You may also have only targeted modules that are marked to be saved (`modules_to_save`).exclude_modulesz You have passed exclude_modules=zS but no modules were excluded. Please check that exclude_modules was set correctly.ztarget_modules=z$ were set but no module was matched.ztarget_parameters=z' were set but no parameter was matched.)ra   r   z are part of the adapter. This can lead to complications, for example when merging the adapter or converting your model to formats other than safetensors. See for example https://github.com/huggingface/peft/issues/2018.F)ra   r`   r   rx   )6r`   r   get_model_configr   r   r^   r_   r8   r,   r    _maybe_include_all_linear_layersr.   r   setlenr   r]   r   IA3any_find_minimal_target_modulesr   r   rsplit
startswithr1   r   _ExcludedModulero   r!   r   r   r   r   _inject_parameterssortedrq   rr   RuntimeWarningrp   r   r   r-   r   r   r   set_adapterr   r   inference_modenamed_parametersrequires_gradr   ri   )'rw   ra   rx   r   rk   rl   r`   excluded_modulesunmatched_modules!targeted_modules_from_peft_configr   r,   r   _key_listuses_dummy_target_modulesrD   names_no_targetnew_target_modulesexisting_adapter_maprE   module_namesprefixkadapter_keyresultr   r   rb   ctxtargeted_set_from_peft_configtargeted_set_from_state_dictdiff_peft_configdiff_state_dictwarning_msg	error_msgr   nps'                   `                      rN   rv   zBaseTuner.inject_adapter  s   D &&|479) 	&&{3,,U322;MK/; 3R8Zx 
 U0023&34FCC44$+K9I4$PTh$h!$MH 7{EJ {11D#;?[//04WW&&(,,6 %tYdYsYstt O 
 ">k>X>XZi!j%&[-G-G)HH-?*  "( 	3KC&.1,2$S)	3
 "%!01F1FGFBLMQAHHS6\15a8MLM( .	BKC  4 >>+"34$++C0
  $4R$8C$?!99+sKfo6$++C0%,,S1..55c:2A%2M/FFK::;{[0A,{C 00'v{F`c 1   l*%,,S1..55c:2A%2M/FFK::;{[0A,{C 00'v{F`c 1  33KE5<<SA].	B` ; 3R8##'u<ct $  ! -00Q,R)+.t/I/I+J(<?[[:=ZZOK? 
  QRXYiRjQkkmn ![\bcr\s[ttvwwk>:))$2O2OXq(9 j  &*;KD^D^ '  &*; &k&@&@%A BE F  ;(=tDP#OP[PoPoOppq!rrI;(8$?K#I+JdJdIeef!ggI ++]  ;(=tDP#OP[PoPoOppq!rrI;(8$?K#I+JdJdIeef!ggI ++["349T9T]mMM2;3N3N2O PG G
 +
 ))$2L2L%k&@&@%AAefhv &92>tGdGd()F)F(GGno"
 #;;%;HMMUAT@V WS S 	--.--e4L)88..0 ,11$&+AO, 	)#"33D9%		
q 58, N8  s*   Z/('Z5Z:Z?"[?[		[	c                .    d  fd}t        j                        }t        |      }j                         D ]  \  }}	t	        |	d      rV|D ]P  j                  d      \  }
}}|
|k7  rt        |	|d      , |||        j                  j                         R h |      }|	j                  d      D ]J  \  }}| d| |v st        fd|D              s& |||        j                  j                         L  y)	z1Inject layers based on peft_config.target_modulesc                P    d}|| v r| j                  |      \  }}}||z   } || v r| S )Nz.base_layer)
rpartition)rG   rD   r   r   r   s        rN   strip_base_layer_from_namez@BaseTuner._inject_parameters.<locals>.strip_base_layer_from_name  sC    
 !D+%$/$:$:4$@!6$vo +% rg   c                   t        |       \  }}} |       }j                  |      }t        |t              r>|j                  j
                  dk7  r%t        d| dt        |      j
                   d      j                  |       
rt        nt        } |       5  j                  	|||||j                  d      d          d d d        y # 1 sw Y   y xY w)NParamWrapperz+Trying to wrap an `nn.Parameter` of layer 'z
' of type z, which is not a valid target. Make sure that this layer is not also targeted with `target_modules`. For some models, PEFT will do this automatically, try setting `target_modules=[]` to prevent it.r   r   )r   r   )r!   get_submoduler.   r   ry   r   r_   typer   r   r   r   r   )rG   r   
param_namer   r   rb   unwrapped_module_nameunwrapped_moduler   rx   rk   ra   r`   rw   r   s            rN   create_and_replace_paramz>BaseTuner._inject_parameters.<locals>.create_and_replace_param  s    *9%*M'FFK$>{$K!$223HI*N;@P@Z@Z@c@cgu@u ABWAXXbF|,,- .EE  22;{S(9${C 	((  ##-#8#8#=b#A ) 	 	 	s   !+CCparametrizationsr   NFrecursec              3  F   K   | ]  }j                  d |         ywr   r   r   
target_keyr   s     rN   r   z/BaseTuner._inject_parameters.<locals>.<genexpr>  s"     3rWaCLL1ZLAQ4R3r   !)r   r   r   r,   r-   r   r^   rp   r1   r   r   )rw   r`   ra   rx   rk   r   unsorted_target_namestarget_namesrG   rE   target_module_namer   r   r   rM   r   r   s   `````          @@rN   r   zBaseTuner._inject_parameters  s;   
			 	8 !$K$A$A B 34#(#6#6#8 	BKv12 ( >C8;s8K5&:)[8 vz48@ ,[#zJ1188=> )C;(O%)/)@)@)@)O B%J231ZLAC|+3req3r0r 1c:N55<<SAB'	Brg   c                    | j                          | j                  j                         D ]:  }t        |t              st        |      5  |j                  ||       ddd       < y# 1 sw Y   GxY w)a  
        This method merges the adapter layers into the base model.

        Merging adapters can lead to a speed up of the forward pass. A copy of the adapter weights is still kept in
        memory, which is required to unmerge the adapters. In order to merge the adapter weights without keeping them
        in memory, please call `merge_and_unload`.

        Args:
            adapter_names (`list[str]`, *optional*):
                The list of adapter names that should be merged. If `None`, all active adapters will be merged.
                Defaults to `None`.
            safe_merge (`bool`, *optional*):
                If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
        )adapter_names
safe_mergeN)r   ra   r   r.   r   rO   merge)rw   r  r  rE   s       rN   merge_adapterzBaseTuner.merge_adapter  so    . 	!!#jj((* 	UF&.1!&) ULL}LTU U	UU Us   
A))A2	c                    | j                   j                         D ]7  }t        |t              st	        |      5  |j                          ddd       9 y# 1 sw Y   DxY w)zU
        This method unmerges all merged adapter layers from the base model.
        N)ra   r   r.   r   rO   unmerge)rw   rE   s     rN   unmerge_adapterzBaseTuner.unmerge_adapter:  sV     jj((* 	%F&.1!&) %NN$% %	%% %s   AA	c                v    | j                         D ]&  }t        |t              s|j                  ||       ( y )N)new_active_adapters)r   r.   r   delete_adapter)rw   rx   r  rE   s       rN   _delete_auxiliary_adapterz#BaseTuner._delete_auxiliary_adapterC  s8    lln 	]F&":;%%lH[%\	]rg   c                     |xs  j                   }t         fd|D              }|rt        |      dkD  rt        d      y y )Nc              3  P   K   | ]  }j                   |   j                    y wr{   )r`   modules_to_save)r   adapterrw   s     rN   r   z.BaseTuner._unloading_checks.<locals>.<genexpr>J  s'      +
:ADW%55+
s   #&r"   z?Cannot unload multiple adapters that specify `modules_to_save`.)r   r   r   r_   )rw   r  adapters_to_consideris_modules_to_save_availables   `   rN   _unloading_checkszBaseTuner._unloading_checksH  sS    ,D0D0D'* +
EY+
 (
$ (C0D,E,I^__ -J'rg   c                `    t        | dt              }t        |d      r|j                         }|S )a|  
        This method gets the config from a model in dictionary form. If model has not attribute config, then this
        method returns a default config.

        Args:
            model (`nn.Module`):
                Model to get the config from.
            default (`dict|None`, *optional*)::
                What to return if model does not have a config attribute.
        r[   to_dict)r^   r   r-   r  )ra   r   s     rN   r   zBaseTuner.get_model_configP  s1     uh0BC<+'//1Lrg   c                    g }| j                  |      }|j                  d      r=| j                  D ].  }|j                  d      d   t        v s|j                  |       0 |S )Ntie_word_embeddingsr   r   )r   getro   r=   r   r1   )rw   ra   r   r   target_modules        rN   r   z"BaseTuner._get_tied_target_modulesa  sm     ,,U312!%!;!; > !&&s+B/3HH'..}=> #"rg   FN)
r`   z(Union[PeftConfig, dict[str, PeftConfig]]rx   r}   rk   boolrl   !Optional[dict[str, torch.Tensor]]returnNoner&  	list[str])r   r   r   r   )ra   	nn.Moduler[   r    rx   r}   r&  r'  )r`   r    r   r9   r&  r    )r`   r    ra   r*  )r`   r    r   r}   r&  r$  r{   )r`   r    rx   r}   r   r*  rb   r}   r   r*  r   r}   r   zOptional[str]r&  r'  )ra   r*  r&  r'  )r[   r    r&  r'  )T)rx   r}   r   r$  r&  r'  r`   r    ra   r*  rb   r}   )TFN)ra   r*  rx   r}   r   r$  rk   r$  rl   r%  r&  r'  )
r`   r    ra   r*  rx   r}   rk   r$  r&  r'  NF)r  Optional[list[str]]r  r$  r&  r'  )rx   r}   r  r.  r&  r'  )r  r.  )ra   r*  r&  r9   )ra   r*  r&  r)  ) r   
__module____qualname____doc__rn   propertyr   r   ru   r   r   r   r   r   r   r   r   r   r   r   r   r   rv   r   r  r  r  r  staticmethodr   r   __classcell__)ry   s   @rN   ri   ri      s   "R #(8<#2 >#2 	#2
  #2 6#2 
#2J # #3  "    )-     	 
       &  
   D      )BV"HJi	i (,"'8<U
U
 U
 !%	U

  U
 6U
 
U
nIB%IB.7IBGJIB_cIB	IBVU:%]
`   
#rg   ri   c                  :   e Zd ZU dZdZded<   dZded<   dZded<   d	Zd
ed<   g Z	ded<   d!dZ
ed"d       Zed"d       Zd#d$dZd%dZed&d       Zed&d       Zed'd       Zd(dZed        Zd)dZd*dZd+dZd,dZd-d.dZed/d       Zed0d       Zd1d Zy)2r   aK  
    A tuner layer mixin that provides the common methods and attributes for all tuners.

    Args:
        is_pluggable (`bool`, *optional*):
            Whether the adapter layer can be plugged to any pytorch module
        active_adapters (Union[List[`str`], `str`], *optional*):
            The name of the active adapter.
    r   ztuple[str, ...]adapter_layer_namesother_param_namesFr$  _disable_adaptersdefaultstr | list[str]_active_adapterr)  merged_adaptersc                T    | }t        |d      r|j                  }t        |d      r|S )z
        (Recursively) get the base_layer.

        This is necessary for the case that the tuner layer wraps another tuner layer.

        r&   )r-   r&   rw   r&   s     rN   get_base_layerzBaseTunerLayer.get_base_layer  s0     
j,/#..J j,/rg   c                r    | j                         }t        |d      r|j                  }|S |j                  }|S )Nqweight)r?  r-   rA  weight)rw   r&   rB  s      rN   rB  zBaseTunerLayer.weight  s@     ((*
:y)''F   &&Frg   c                :    | j                         }|j                  S r{   )r?  biasr>  s     rN   rD  zBaseTunerLayer.bias  s    ((*
rg   Nc                    t         r{   r   )rw   r  r  s      rN   r  zBaseTunerLayer.merge      !!rg   c                    t         r{   rF  r~   s    rN   r  zBaseTunerLayer.unmerge  rG  rg   c                ,    t        | j                        S r{   )r$  r<  r~   s    rN   mergedzBaseTunerLayer.merged  s    D(())rg   c                    | j                   S r{   )r8  r~   s    rN   disable_adapterszBaseTunerLayer.disable_adapters  s     %%%rg   c                    | j                   S r{   )r;  r~   s    rN   rt   zBaseTunerLayer.active_adapter  s     ###rg   c                    t               }| j                  D ]a  }t        | |      }t        |t        j
                  t        j                  f      s:|j                  t        |j                                      c |S )z:Return all adapter names that can be found on this module.)	r   r6  r^   r.   r   r   r   rs   r:   )rw   adapters
layer_namerE   s       rN   _get_available_adaptersz&BaseTunerLayer._get_available_adapters  sd    522 	0JT:.Ffr}}b6F6F&GHOOC./		0
 rg   c                h    t        | j                  t              r| j                  gS | j                  S r{   r|   r~   s    rN   r   zBaseTunerLayer.active_adapters  r   rg   c                    |r#| j                  | j                         d| _        y| j                  D ]  }t	        | |      }|j                  d       ! d| _        y)zToggle the enabling and disabling of adapters

        Takes care of setting the requires_grad flag for the adapter weights.

        Args:
            enabled (bool): True to enable adapters, False to disable adapters
        FTN)r   r   r8  r6  r^   requires_grad_)rw   enabledrP  rB   s       rN   enable_adapterszBaseTunerLayer.enable_adapters  s^     T112%*D" #66 ,
j1$$U+, &*D"rg   c                    t        |t              r|g}| j                  D ]M  }t        | |      }|j	                         D ],  \  }}||v r|j                  d       |j                  d       . O || _        y)a   Set the active adapter(s).

        Additionally, this function will set the specified adapters to trainable (i.e., requires_grad=True). If this is
        not desired, use the following code.

        ```py
        >>> for name, param in model_peft.named_parameters():
        ...     if ...:  # some check on name (ex. if 'lora' in name)
        ...         param.requires_grad = False
        ```

        Args:
            adapter_name (`str` or `List[str]`): Name of the adapter(s) to be activated.
        TFN)r.   r}   r6  r^   itemsrT  r;  )rw   r  rP  module_dictr   rB   s         rN   r   zBaseTunerLayer.set_adapter  s     mS)*OM 22 	0J!$
3K)//1 0
U-' ((.((/0	0  -rg   c                    t               }| j                  | j                  z   D ]:  }t        | |      }t	        |d      s|j                  |j                                < t        |      S )z3Return a sorted list of all available adapter namesr:   )r   r6  r7  r^   r-   rs   r:   r   )rw   r  rD   attrs       rN   _all_available_adapter_namesz+BaseTunerLayer._all_available_adapter_names  sb    ,,t/E/EE 	2D 4&DtV$$$TYY[1	2 m$$rg   c                   | j                   | j                  z   D ]  }|t        | |      v st        | |      |=   || j                  v r| j                  dd }|j	                  |       |r| j                  |       y| j                         }|s| j                  g        y|d   }t        j                  d| d| d       | j                  |d          yy)a  
        Delete an adapter from the layer

        This should be called on all adapter layers, or else we will get an inconsistent state.

        This method will also set a new active adapter if the deleted adapter was an active adapter. It is important
        that the new adapter is chosen in a deterministic way, so that the same adapter is chosen on all layers.

        Args:
            adapter_name (`str`): The name of the adapter to delete

        Nr   zAdapter z< was active which is now deleted. Setting active adapter to r   )	r6  r7  r^   r   remover   r\  rq   rr   )rw   rx   r[  r   remaining_adaptersnew_active_adapters         rN   r  zBaseTunerLayer.delete_adapter	  s     ,,t/E/EE 	6DwtT22D$'5	6 4///"2215O""<0  1 &*%F%F%H")$$R();A)>&MM"<.0l-.a1 $$%7%:;% 0rg   c                  	 |g| j                         }t        |t        j                        r|j                  }dD ]+  }t        ||d      }||j                  }|j                  } n yt        j                  d      	| j                  | j                  z   D ]  }t        | |d      }t        |t        j                  t        j                  t        f      s@||vrEt        	fd|j                         D              rhj                  j                   s|j                  j"                  r||   j%                  |      ||<   ||   j%                  |      ||<    y)zU
        Move the adapter of the given name to the device of the base layer.
        N)rB  rA  r(   c              3  <   K   | ]  }|j                   k(    y wr{   )r3   )r   r   r(   s     rN   r   zGBaseTunerLayer._move_adapter_to_device_of_base_layer.<locals>.<genexpr>L  s     H188t#Hs   r   )r?  r.   r   MultiheadAttentionrU   r^   r3   r   r2   r6  r7  r   r   r#   r   r   is_floating_point
is_complexrA   )
rw   rx   r3   r&   weight_namerB  r   adapter_layer_nameadapter_layerr(   s
            @rN   %_move_adapter_to_device_of_base_layerz4BaseTunerLayer._move_adapter_to_device_of_base_layer.  sJ    >,,.J*b&;&;<'00
4  [$?%#]]F"LLE ||F#
 #'":":T=S=S"S 	U#D*<dCMmbmmR=M=Mz-Z[=0H]-E-E-GHH ||--1H1H.;L.I.L.LV[`.L.al+.;L.I.L.LV.Tl+	Urg   c                     y r{   r   rw   xr   s      rN   _cast_input_dtypez BaseTunerLayer._cast_input_dtypeU  s    FIrg   c                     y r{   r   rl  s      rN   rn  z BaseTunerLayer._cast_input_dtypeX  s    VYrg   c                l    |yt        | dd      }|r|j                  |k(  r|S |j                  |      S )a|  
        Whether to cast the dtype of the input of the forward method.

        Usually, we want to enable this to align the input dtype with the dtype of the weight, but by setting
        layer.cast_input_dtype=False, this can be disabled if necessary.

        Enabling or disabling can be managed via the peft.helpers.disable_lora_input_dtype_casting context manager.
        Ncast_input_dtype_enabledTrc  )r^   r   rA   )rw   rm  r   rq  s       rN   rn  z BaseTunerLayer._cast_input_dtype[  s@     9#*41KT#R (agg.>Htt%t  rg   )r&  r*  )r&  torch.Tensorr#  )r  r$  r  r.  r&  r'  r+  )r&  r$  )r&  r:  )r&  set[str])rU  r$  r&  r'  )r  r:  r&  r'  r(  )rx   r}   r&  r'  r{   )rx   r}   r3   zOptional[torch.device]r&  r'  )rm  r'  r   torch.dtyper&  r'  )rm  rr  r   rt  r&  rr  )r   rt  )r   r/  r0  r1  r6  __annotations__r7  r8  r;  r<  r?  r2  rB  rD  r  r  rJ  rL  rt   rQ  r   rV  r   r\  r  rj  r   rn  r   rg   rN   r   r   n  s    ,.-)++ $t# (1O_0 "$OY#
    "" * * & & $ $ # #*$->	%#<J%UN I IY Y!rg   r   c                0   t        | t              s| st        d      t        |       } d| v rt        d      t        |      }| j	                  |      sd}t        |      d }|D ch c]  } ||      D ]  }|  }}}| D ci c]  }| ||       }}t               }t        |j                         d       D ]<  \  }	|	D ]2  }||v s||v rt        fd|D              r!|j                  |        < > |st        |       S |S c c}}w c c}w )	a   Find the minimal set of target modules that is sufficient to separate them from the other modules.

    Sometimes, a very large list of target_modules could be passed, which can slow down loading of adapters (e.g. when
    loaded from diffusers). It may be possible to condense this list from hundreds of items to just a handful of
    suffixes that are sufficient to distinguish the target modules from the other modules.

    Example:
        ```py
        >>> from peft.tuners.tuners_utils import _find_minimal_target_modules

        >>> target_modules = [f"model.decoder.layers.{i}.self_attn.q_proj" for i in range(100)]
        >>> target_modules += [f"model.decoder.layers.{i}.self_attn.v_proj" for i in range(100)]
        >>> other_module_names = [f"model.encoder.layers.{i}.self_attn.k_proj" for i in range(100)]
        >>> _find_minimal_target_modules(target_modules, other_module_names)
        {"q_proj", "v_proj"}
        ```

    Args:
        target_modules (`list[str]` | `set[str]`):
            The list of target modules.
        other_module_names (`list[str]` | `set[str]`):
            The list of other module names. They must not overlap with the target modules.

    Returns:
        `set[str]`:
            The minimal set of target modules that is sufficient to separate them from the other modules.

    Raises:
        ValueError:
            If `target_modules` is not a list or set of strings or if it contains an empty string. Also raises an error
            if `target_modules` and `other_module_names` contain common elements.
    z2target_modules should be a list or set of strings.r%   z2target_modules should not contain an empty string.ztarget_modules and other_module_names contain common elements, this should not happen, please open a GitHub issue at https://github.com/huggingface/peft/issues with the code to reproduce this issuec                    | j                  d      }t        t        |            D cg c]  }dj                  ||d         c}d d d   S c c}w )Nr   r   )r=   ranger   r>   )spartsrJ   s      rN   generate_suffixesz7_find_minimal_target_modules.<locals>.generate_suffixes  sD    -23u:->?qr#?"EE?s   Ac                    | d   S )Nr"   r   )tups    rN   <lambda>z._find_minimal_target_modules.<locals>.<lambda>  s    TWXYTZ rg   )r   c              3  F   K   | ]  }j                  d |z           ywr   r   )r   
req_suffixitems     rN   r   z/_find_minimal_target_modules.<locals>.<genexpr>  s     [:t}}S:%56[r  )	r.   r}   r_   r   
isdisjointr   rX  r   add)
r   other_module_namesmsgr{  r  r   other_module_suffixestarget_modules_suffix_maprequired_suffixessuffixess
       `     rN   r   r   m  sW   F .#&nMNN(N	^MNN/0$$%78v 	 oF
 1CiQbcgQhivViVii LZ Z4'8'>!> Z Z  !!:!@!@!BHZ[ 	h 	F**f8M.M[IZ[[!%%f-		 >""/ j ![s   *DDc                      e Zd ZdZd Zy)r   zp
    A private helper method used to represent excluded modules in the check_target_module_exists function.
    c                     yr-  r   r~   s    rN   __bool__z_ExcludedModule.__bool__  s    rg   N)r   r/  r0  r1  r  r   rg   rN   r   r     s    rg   r   c                T   t        | d      r| j                  rt        | j                  t              r*t	        j
                  | j                        rJt               S | j                  v r
t               S t        fd| j                  D              r
t               S t        | dd      }|rt        fd|D              r
t               S | j                  | j                  yt        | j                  t              rt        | j                        }|S | j                  v rd}|S t        fd| j                  D              }t        | d	d      }t        | d
d      }|duxr  t        |t              rt        |      dk7  nd}|r|rd}|t        |      dk(  rt	        j                  d      }n:t        |t              r|gn|}|D ]   }t	        j                  d| d      }|  n |d}|S t        |j!                  d            }t        |t              r||k(  }|S ||v }|S )a  A helper method to check if the passed module's key name matches any of the target modules in the adapter_config.

    Args:
        config (`LoraConfig` | `LycorisConfig`): A config to match target modules from
        key (`str`): A key to search any matches in config

    Returns:
        `bool` | `re.Match[str]` | `None`: True of match object if key matches any target modules from config, False or
        None if no match found
    r   c              3  F   K   | ]  }j                  d |         ywr   r   )r   exclude_keyr   s     rN   r   z-check_target_module_exists.<locals>.<genexpr>  s!     [[+/0[r  r  Nc              3  R   K   | ]  }t        j                  d | d         yw)z(^|.*\.)z($|\..*)N)rematch)r   mr   s     rN   r   z-check_target_module_exists.<locals>.<genexpr>  s%     O!rxx8A3h/5Os   $'FTc              3  F   K   | ]  }j                  d |         ywr   r   r  s     rN   r   z-check_target_module_exists.<locals>.<genexpr>  s!     !iZ#,,:,/?"@!ir  r   r   r   z.*\.[^.]*\.(\d+)\.z.*\.z	\.(\d+)\.r"   )r-   r   r.   r}   r  	fullmatchr   r   r^   r   r   r   r8   r   r  intgroup)	r[   r   r  target_module_foundlayer_indexesr   is_using_layer_indexeslayer_indexpatterns	    `       rN   check_target_module_existsr    sA    v()f.D.Df,,c2||F22C8&((F***"$$[FDZDZ[["$$ f&7>OOOO"$$%F,D,D,P&''-6v7L7LcRF E 
%%	%"@ = "!iSYShSh!ii(=tD )94@!.d!: "
'1-'FC!#D 	 "&9K %^)<)A hh'<cB5?PS5T.!1Zh- G"$((d7)9+Es"KK".
 "&+#  "+"3"3A"67mS1*5*F'  +6*F'rg   c                   | j                   |   }| j                  j                         D cg c]  \  }}|	 }}}g g d}|D ]=  }| j                  ||      r|d   j	                  |       *|d   j	                  |       ? |S c c}}w )zw
    A helper function to inspect the set of matched and unmatched modules for a PEFT model and the given adapter.
    )matched	unmatchedr  r  )r`   ra   r,   r   r1   )tunerrx   r[   r   r   r   rY  s          rN   inspect_matched_modulesr    s     |,F"'++";";"=>Q>H> r2K 1,,VS9	"))#.$++C0	1
  ?s   Bc                p   t        | d      s| S t        | j                  t              r!| j                  j	                         t
        k(  s| S t        j                  j                  t        f}d}t               }|j                         D ]Z  \  }t        |      r|j                  |       $t        t              s5t        fd|D              sJ|j                  |       \ t               }t        |t              r|j!                         }|=|j                         D cg c]  \  }}||u s| c}}d   }	|j                  |	       nu| j"                  t$        j&                  k(  rXt(        D ]O  }t+        ||d      }
|
|j                         D cg c]  \  }}||
u s| c}}d   }	|j                  |	        n |j                         D ]G  \  }t        t              sj                         D ]  \  }}|s	|j                  | d|          I ||z  }|| _        | S c c}}w c c}}w )z
    Helper function to update `target_modules` to all linear/Conv1D layers if provided as 'all-linear'. Adapted from
    the QLoRA repository: https://github.com/artidoro/qlora/blob/main/qlora.py
    r   )Linearc              3  L   K   | ]  }|t              j                  v   y wr{   )r   r   )r   r   rE   s     rN   r   z3_maybe_include_all_linear_layers.<locals>.<genexpr>3  s!     7iWXT&\=R=R8R7is   !$Nr   r   )r-   r.   r   r}   lowerr   r2   r   r  r   r   r,   r  r   r   r   get_output_embeddings	task_typer   SEQ_CLSr   r^   )r`   ra   linear_classeslinear_nameslinear_module_namesrD   module_names_to_exclude
output_embrE   last_module_namecls_headr   r   childs           `     rN   r   r     s)   
 ; 01 	;--s3&&,,.2QQhhoov.NL%++- *ffn-##D)/C7i\h7i4i  ##D)* "e%)002
!9>9L9L9NgvRX\fRfghij#''(89""h&6&66 + "5$5'AFATATAV'mvZ`dlZl'mno'p$+//0@A  --/ Ffn-!'!5!5!7 F+//6(!F80DEFF 22!4K-  h (ns   %H,2H,H2%H2c                   || j                   }t        |t              rt        d|d      | j                  rt        | j                        }|D cg c]	  }||vs| }}|rFt        j                  ddj                  | j                         ddj                  |       d       |S t        j                  d       |S c c}w )z
    Helper function to check which adapters should be merged.

    Only return those adapters that are not already merged. Give a warning if some or all of the adapters are already
    merged.

    z/adapter_names should be a list of strings, got r   z'Already following adapters were merged ,z#. You are now additionally merging z/All adapters are already merged, nothing to do.)
r   r.   r}   r_   rJ  r   r<  rq   rr   r>   )rE   r  r<  rD   s       rN   check_adapters_to_merger  \  s     ..-%J=J[[\]^^}}f445*7W$4;VWWMM9#((6CYCY:Z9[ \447HH]4K3LAO  MMKL Xs   	CCc                    t        j                  |       }dd}|r0| j                         D ]  \  }} |||j                  |              |S )zClone a module in a pytorch model.

    Clones a module of a model, optionally sharing all the parameters between the original and the clone. Simplifies
    reusing a module when manipulating the architecture of a model.
    c                \    | j                  d      D ]  \  }}|j                  ||        y )NFr  )r   register_parameter)srcdstrD   rM   s       rN   _share_weightsz$clone_module.<locals>._share_weights  s3    ///> 	0KD%""4/	0rg   )r  r*  r  r*  )copydeepcopyr,   r   )rE   share_weightscloner  rD   r   s         rN   clone_moduler  x  sV     MM&!E0 %335 	AOD)9e&9&9$&?@	A Lrg   c                   t        | d      r| j                  } t        | d      rt        | d      r| j                  } d}d}t        | d      rd}| j                  }nUt        | d      r/t        | j                  d      rd}| j                  j
                  }nt        | d      rd	}| j                  }|rt        |t        j                        st        d
      g }|D ]l  \  }}t        ||      D ]X  }t        |      }|j                  t        ||   d             |d   j                         D ]  }	t        |	d      s||	_         Z n t        j                  |      }|dk(  r|| _        n/|dk(  r|| j                  _        n|d	k(  r|| _        nt        d      t        | j"                  d      rt        |      | j"                  _        yy)a~  Replicate layers in a transfomer model with weight sharing.

    This function looks for a module list attribute at model[(.model)*].layers and replicates the layers in the module
    list according to the layer map. For example the map `[[0, 4], [2, 5]]` will take the set of layers `[0, 1, 2, 3,
    4]` and replace them with a module list containing `[0, 1, 2, 3, 2, 3, 4]`.
    ra   bertNlayersllamaencoderrB   hfalconzlCould not locate the layers attribute in the model. Expected Llama, Bert or Falcon compatible architectures.T)r  r   	layer_idxz@Unexpected model type, need to handle post-processing of layers.num_hidden_layers)r-   ra   r  r  r  rB   r  r.   r   
ModuleListr_   rx  r   r1   r  r   r  r[   r  )
ra   	layer_mapr\   r  
new_layersstartendrJ   current_idxr   s
             rN   replicate_layersr    s    %
! %
! uf

J Fuh
			"wu}}g'F
$$		
Z>G
 	

 J 6
suc" 	6Aj/Kl6!9DIJ'^335 6	9k2*5I'6		66 ]]:&FW	v	$	x	[\\u||01),Z& 2rg   r,  )r   list[str] | set[str]r  r  r&  rs  )r   r}   r&  zbool | re.Match[str] | None)r9  )r  ri   rx   r}   r&  r9   )r`   r    ra   r*  r&  r    r{   )rE   r   r  r.  r&  r)  )F)rE   r*  )ra   r*  r  zlist[tuple[int, int]])C
__future__r   r  r;   r  r   rq   abcr   r   
contextlibr   r   typingr   r	   r
   r   r2   accelerate.hooksr   accelerate.utilsr   r   r   transformersr   transformers.pytorch_utilsr   peft.mappingr   
peft.utilsr   peft.utils.constantsr   r   r   r   r   peft.utils.integrationsr   peft.utils.otherr   r   r   peft.utils.peft_typesr   r   r[   r    utilsr!   _buffer_dictr#   rO   rf   Moduleri   r   r   r   r  r  r   r  r  r  r   rg   rN   <module>r     s    #  	 	   # 2 1 1  - E  ( - 4 6  7 q q 4  # $ @S @SF,_#		3 _#D|!S |!~O(O>ROOd CL<~8&19rg   