
    bi                        d dl mZ d dlZd dlmZ d dlmZ d dlmZ d dl	m	Z	 d dl
mZ d dlmZmZmZ d d	lmZmZ d
dlmZ  G d de      Zy)    )annotationsN)Optional)nn)Module)tqdm)
PeftConfig)	BaseTuner_get_submodulescheck_target_module_exists)6TRANSFORMERS_MODELS_TO_LNTUNING_TARGET_MODULES_MAPPINGModulesToSaveWrapper   )LNTuningLayerc                       e Zd ZU dZdZded<   d fdZedd       Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ		 	 	 	 	 	 	 	 ddZ
dd	Zdd
ZddZddZddZddZddZ	 	 	 	 d	 	 	 	 	 ddZd Z	 d 	 	 	 	 	 	 	 d!dZd"d#dZ xZS )$LNTuningModela  
    Creates LayerNorm tuning from a pretrained transformer model.

    The method is described in detail in https://huggingface.co/papers/2312.11420.

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`LNTuningConfig`]): The configuration of the Lora model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            This option has no effect on LN tuning but exists for consistency with other PEFT methods.

    Returns:
        'torch.nn.Module': The adapted model with LayerNorm tuned on.

    Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import get_peft_model, TaskType, LNTuningConfig

        >>> peft_config = LNTuningConfig(
        ...     task_type=TaskType.CAUSAL_LM,
        ... )

        >>> model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
        >>> model = get_peft_model(model, peft_config)
        >>> model.print_trainable_parameters()
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`LNTuningConfig`]): The configuration of the Lora model.
    
ln_tuning_strprefixc                z    	 t         |   |      S # t        $ r |dk(  r t        | j                  |      cY S w xY w)z1Forward missing attributes to the wrapped module.model)super__getattr__AttributeErrorgetattrr   )selfname	__class__s     V/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/ln_tuning/model.pyr   zLNTuningModel.__getattr__D   sB    	-7&t,, 	-w4::t,,	-s    %::c                ~    | j                   0|d   t        vrt        d      t        t        |d            | _         | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   
ValueErrorset)peft_configmodel_configs     r   _prepare_adapter_configz%LNTuningModel._prepare_adapter_configN   sK    %%-L)1gg !STT),F|T`Gab*K&     c                    | j                  |||      }|| j                  k7  r|j                  d       | j                  ||||       y )NF)_create_new_moduleactive_adapterrequires_grad__replace_module)r   r$   adapter_nametargettarget_nameparentcurrent_key
new_modules           r   _create_and_replacez!LNTuningModel._create_and_replaceX   sH     ,,[&,O
4...%%e,V[*fEr'   c                ~    t        |t              st        ||      }|S |}|j                  |j                  |       |S N)
isinstancer   update_layer
base_layer)r   r$   r.   r-   r2   s        r   r)   z LNTuningModel._create_new_moduleg   sE     &-0&v|<J   J##F$5$5|Dr'   c                   t        |||       t        |d      r|j                  }t        |dd       ^t        |d      r|j                  |j                  _        n|j                  |_        |j                  |j                  j                         |j                         D ]D  \  }}t        |d      r|j                  n|j                  }|j                  |j                         F y )Nr8   stateqweight)
setattrhasattrr8   r   r:   toweightdevicenamed_modulesr;   )r   r0   
child_namer2   childr   moduler?   s           r   r,   zLNTuningModel._replace_modulet   s    
J/5,'$$E5'4(4z<0.3kk
%%+#(;;
 MM%,,--.&446 	%LD&&-eY&?U]]U\\FIIfmm$	%r'   c                n    |j                         D ]"  \  }}| j                  |vrd|_        d|_        $ y )NFT)named_parametersr   requires_grad)r   r   nps       r    _mark_only_adapters_as_trainablez.LNTuningModel._mark_only_adapters_as_trainable   s8    **, 	'DAq{{!#"'"&		'r'   c                    t        ||      S r5   )r   )r   r$   keys      r   _check_target_module_existsz)LNTuningModel._check_target_module_exists   s    )+s;;r'   c                    | j                   j                         D ]*  }t        |t        t        f      s|j                  |       , y r5   )r   modulesr6   r   r   enable_adapters)r   enabledrD   s      r   _set_adapter_layersz!LNTuningModel._set_adapter_layers   s<    jj((* 	0F&=2F"GH&&w/	0r'   c                (    | j                  d       y)zyEnable all adapters.

        Call this if you have previously disabled all adapters and want to re-enable them.
        TrQ   NrR   r   s    r   enable_adapter_layersz#LNTuningModel.enable_adapter_layers   s    
 	   .r'   c                (    | j                  d       y)zDisable all adapters.

        When disabling all adapters, the model output corresponds to the output of the base model.
        FrT   NrU   rV   s    r   disable_adapter_layersz$LNTuningModel.disable_adapter_layers   s    
 	   /r'   c                    | j                   j                         D ]U  }t        |t              s|j                  r%t        j                  d       |j                          |j                  |       W || _	        y )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
r   rO   r6   r   mergedwarningswarnunmergeset_adapterr*   )r   r-   rD   s      r   r_   zLNTuningModel.set_adapter   s^    jj((* 	1F&-0==MM"noNN$""<0	1 +r'   c                   | j                  |       | j                  j                         D cg c]  \  }}| j                  |vs| }}}d|rdndz   dz   }t	        || |      D ]_  }	 t        | j                  |      \  }	}
}t        |
d      s+|r|
j                  |       | j                  |	||
j                         |
       a | j                  S c c}}w # t        $ r Y w xY w)NzUnloading adapters zand merging  r   )disabledescr8   )_unloading_checksr   rA   r   r   r
   r   r=   merger,   get_base_layer)r   re   progressbar
safe_mergeadapter_namesrL   _key_listrc   r0   r.   r/   s               r   _unload_and_optionally_mergez*LNTuningModel._unload_and_optionally_merge   s     	}-&*jj&>&>&@[FCDKKWZDZC[[$%RH7Rk/E 		[C.=djj#.N+ v|,LL/$$V[&:O:O:QSYZ		[ zz \ " s   CC,C	C('C(c                &    | j                  d      S )NFre   rl   rV   s    r   unloadzLNTuningModel.unload   s    00u0==r'   c                &    | j                  d      S )NTrn   ro   )r   rg   rh   ri   s       r   merge_and_unloadzLNTuningModel.merge_and_unload   s     00t0<<r'   c                     y r5    )r   r-   autocast_adapter_dtypes      r   _cast_adapter_dtypez!LNTuningModel._cast_adapter_dtype   s     	r'   )r   r   )r$   r   r%   dictreturnr   )r$   r   r-   r   r.   r   r/   r   r0   r   r1   r   rx   None)r$   r   r.   r   r-   r   rx   r   )
r0   r   rB   r   r2   r   rC   r   rx   ry   )r   r   )r$   r   rL   r   rx   bool)rQ   rz   rx   ry   )rx   ry   )r-   r   rx   ry   )TFFN)rg   rz   rh   rz   ri   Optional[list[str]])FFN)rg   rz   rh   rz   ri   r{   rx   z	nn.Module)T)r-   r   ru   rz   rx   ry   )__name__
__module____qualname____doc__r   __annotations__r   staticmethodr&   r3   r)   r,   rJ   rM   rR   rW   rY   r_   rl   rp   rr   rv   __classcell__)r   s   @r   r   r      s9   !F FC-  FF F 	F
 F F F 
F  	
 
%"'<0
/0+ ! -1  	
 +0> im==59=Re=	=
 r'   r   )
__future__r   r\   typingr   torchr   torch.nn.modulesr   r   peft.configr   peft.tuners.tuners_utilsr	   r
   r   
peft.utilsr   r   layerr   r   rt   r'   r   <module>r      s6    #    #  " [ [ c  qI qr'   