
    bi8                         d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ d dl	m	Z	 d dl
mZmZmZmZ d dlmZmZmZ d	d
lmZ d	dlmZmZmZ  G d de      Zy)    N)asdict)Enum)Optional)nn)tqdm)	BaseTunerBaseTunerLayercheck_target_module_existsonload_layer)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )
BOFTConfig)	BOFTLayerConv2dLinearc            	           e Zd ZU dZdZeed<   deddfdZe	d        Z
d	 Zd
 Zdej                  ddfdZe	d        Zdef fdZddefdZd dZd Zd Zd Ze	d        Z	 	 	 	 d!dededeee      fdZdeddfdZ	 d"dededeee      dej                  j                  fdZdej                  j                  fdZ xZ S )#	BOFTModela  
    Creates BOFT and OFT model from a pretrained transformers model. Paper: https://huggingface.co/papers/2311.06243
    https://huggingface.co/papers/2306.07280

    Args:
        model ([`transformers.PreTrainedModel`]): The model to be adapted.
        config ([`BOFTConfig`]): The configuration of the BOFT model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The BOFT model.

    Example::

        >>> import transformers >>> from transformers import AutoModelForSeq2SeqLM, BOFTConfig >>> from peft import
        BOFTConfig, get_peft_model

        >>> config = BOFTConfig( ... boft_block_size=8, ... boft_n_butterfly_factor=1, ... target_modules=["query",
        "value", "key", "output.dense", "mlp.fc1", "mlp.fc2"], ... boft_dropout=0.1, ... bias="boft_only", ...
        modules_to_save=["classifier"], ... )

        >>> model = transformers.Dinov2ForImageClassification.from_pretrained( ... "facebook/dinov2-large", ...
        num_labels=100, ... ) >>> boft_model = get_peft_model(model, config)

    **Attributes**:
        - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`BOFTConfig`]): The configuration of the BOFT model.
    boft_prefixconfigreturnNc                     t        | j                        dkD  r2|j                  dk7  r"t        | j                  j
                   d      yy)z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueError	__class____name__)selfr   s     Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/boft/model.py_check_new_adapter_configz#BOFTModel._check_new_adapter_configM   sR       !A%FKK6,A>>**+ ,7 7  -B%    c                     t        | |      S N)r
   )boft_configkeys     r#   _check_target_module_existsz%BOFTModel._check_target_module_exists\   s    )+s;;r%   c                 .   |t        d      t        |d      xr |j                  d u}|j                  |j                  |j
                  |j                  |j                  |j                  d}	||	d<   t        |t              sI | j                  |||fi |	}
|| j                  vr|
j                  d       | j                  |||
|       y |j                  ||j                  |j                  |j
                  |j                  |j                         y )NzCurrent Key shouldn't be `None`r   )boft_block_sizeboft_block_numboft_n_butterfly_factorboft_dropoutfan_in_fan_outinit_weightsF)r,   r-   r.   r/   r1   )r   hasattrr   r,   r-   r.   r/   r0   r1   
isinstancer   _create_new_moduleactive_adaptersrequires_grad__replace_moduleupdate_layer)r"   r(   adapter_nametargettarget_nameparentcurrent_keyoptional_kwargsr   kwargs
new_modules              r#   _create_and_replacezBOFTModel._create_and_replace`   s    >??vv&B6;;d+B*::)88'2'J'J'44)88'44
 v &),000lF]V\]J4#7#77))%0  j&I + ; ;*99(3(K(K(55(55   r%   c                    t        |||       t        |d      r|j                  }t        |d      s.|j                  |_        t        |d      r|j                  |_        t        |dd       ^t        |d      r|j                  |j                  _        n|j                  |_        |j                  |j                  j                         t        j                  d      |j                         D ]\  \  }}| j                  |v st        fd|j                         D              r8|j                  |j                  j                         ^ y )N
base_layerr   statemetac              3   <   K   | ]  }|j                   k(    y wr'   )device).0prE   s     r#   	<genexpr>z,BOFTModel._replace_module.<locals>.<genexpr>   s     I188t+Is   )setattrr2   rC   weightr   getattrrD   torG   torchnamed_modulesr   any
parameters)r"   r<   
child_namer@   childnamemodulerE   s          @r#   r7   zBOFTModel._replace_module   s   
J/
 5,'$$Ez<0 %Juf%"'**
5'4(4z<0.3kk
%%+#(;;
 MM%,,--.||F#&446 	3LD&{{d"IV5F5F5HIIIIell112	3r%   modelc                    |j                         D ]  \  }}| j                  |vsd|_         | j                  D ]  }| j                  |   j
                  }|dk(  r"|dk(  r%|j                         D ]  \  }}d|v sd|_         L|dk(  rU|j                         D ]A  \  }}t        |t              st        |d      s$|j
                  1d|j
                  _        C t        d| d       y )	NFr   allr   T	boft_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradr5   r   r   rP   r3   r   r2   NotImplementedError)r"   rW   nrI   active_adapterr   rU   ms           r#    _mark_only_adapters_as_trainablez*BOFTModel._mark_only_adapters_as_trainable   s   **, 	(DAq{{!#"'	( #22 	ZN##N388Dv~u}!224 /DAq{*./ $$224 4GD!!!Y/GAv4F166K]/3,4 *,<TFBW*XYY	Zr%   c                 ~   t        |t              r|j                         }n|}t        |t        j                  j
                        r5|d   r!t        j                  d       dx|d<   | _        t        ||fi |}|S t        |t        j                  j                        rt        ||fi |}|S t        d| d      )Nr0   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.FzTarget module zY is not supported. Currently, only `torch.nn.Linear` and `torch.nn.Conv2d` are supported.)r3   r	   get_base_layerrO   r   r   warningswarnr0   r   r   )r(   r9   r:   r?   target_base_layerr@   s         r#   r4   zBOFTModel._create_new_module   s    fn- & 5 5 7 &'9&'7 INM'(;+E??J  )588??;??J    )Y Y r%   rU   c                 z    	 t         |   |      S # t        $ r |dk(  r t        | j                  |      cY S w xY w)z1Forward missing attributes to the wrapped module.rW   )super__getattr__AttributeErrorrM   rW   )r"   rU   r    s     r#   ri   zBOFTModel.__getattr__   sB    	-7&t,, 	-w4::t,,	-s    %::	inferencec           
         i }| j                   j                         D ]U  \  }}t        |      j                         D ci c]$  \  }}|t        |t              r|j
                  n|& }}}|sQd|d<   W |<   |S c c}}w )NTinference_mode)r   itemsr   r3   r   value)r"   rk   config_dictr)   ro   kvr   s           r#   get_peft_config_as_dictz!BOFTModel.get_peft_config_as_dict   s    **002 	0JCKQRW=K^K^K`a41aaJq$$7Q>aFa+/'(	0 "C	 bs   )A<c                     | j                   j                         D ]*  }t        |t        t        f      s|j                  |       , y r'   )rW   modulesr3   r	   r   enable_adapters)r"   enabledrV   s      r#   _set_adapter_layerszBOFTModel._set_adapter_layers   s<    jj((* 	0F&>3G"HI&&w/	0r%   c                 (    | j                  d       y )NTrw   )rx   r"   s    r#   enable_adapter_layerszBOFTModel.enable_adapter_layers   s       .r%   c                     | j                   D ]<  }| j                  |   j                  }|dk7  s"d| d}t        j                  |       > | j                  d       y )Nr   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Frz   )r5   r   r   rd   re   rx   )r"   r_   valmsgs       r#   disable_adapter_layersz BOFTModel.disable_adapter_layers   sp    "22 	#N"">277Cf}TUXTY ZG G  c"	# 	   /r%   c                     | j                   j                         D ]U  }t        |t              s|j                  r%t        j                  d       |j                          |j                  |       W || _	        y )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
rW   ru   r3   r   mergedrd   re   unmergeset_adapterr_   )r"   r9   rV   s      r#   r   zBOFTModel.set_adapter   s^    jj((* 	1F&),==MM"noNN$""<0	1 +r%   c                 ~    | j                   0|d   t        vrt        d      t        t        |d            | _         | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r   set)r   model_configs     r#   _prepare_adapter_configz!BOFTModel._prepare_adapter_config  sK    %%-L)1cc !STT),B<P\C]^*K& r%   progressbar
safe_mergeadapter_namesc                    |r| j                          | j                  j                         D cg c]  \  }}| j                  |vs| }}}d|rdndz   dz   }t	        || |      D ]  }	 t        | j                  |      \  }	}
}t        |
      5  t        |
d      r8|r|
j                  ||       | j                  |	||
j                         |
       ngt        |
t              rW|
j                  |
j                     }t        |d      r%|r|j                  ||       |j                         }t!        |	||       d d d         | j                  S c c}}w # t        $ r Y w xY w# 1 sw Y   
xY w)Nz
Unloading zand merging  rW   )disabledescrC   )r   r   )_check_merge_allowedrW   rP   r   r   r   rj   r   r2   merger7   rc   r3   r   modules_to_saver_   rK   )r"   r   r   r   r   r)   _key_listr   r<   r:   r;   r@   s                r#   _unload_and_optionally_mergez&BOFTModel._unload_and_optionally_merge  sm    %%'&*jj&>&>&@[FCDKKWZDZC[[~B?'Ik/E 	=C.=djj#.N+ f% =6<0
-X((f>S>S>UW]^(<=!'!7!78M8M!NJz<8 &,,
R_,`%/%>%>%@
FK<= =	=( zz- \
 " = =s*   EE-EB,E)	E&%E&)E3	r9   c                    |t        | j                  j                               vrt        d| d      | j                  |= | j                  j                         D cg c]  \  }}| j                  |vs| }}}d}|D ]P  }t        | j                  |      \  }}}t        |t              s.|j                  |       |B|j                  dd }R |xs g | _        | j                  ||       yc c}}w )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not existN)new_active_adapters)listr   keysr   rW   rP   r   r   r3   r   delete_adapterr5   r_   _delete_auxiliary_adapter)r"   r9   r)   r   r   new_adapterr:   s          r#   r   zBOFTModel.delete_adapter-  s     tD$4$4$9$9$;<<x~_EFF\*&*jj&>&>&@[FCDKKWZDZC[[ 	<C*4::s;LAvq&),%%l3&"("8"8";K	< */R&&|&U \s   C36C3c                 *    | j                  |||      S )a  
        This method merges the BOFT layers into the base model. This is needed if someone wants to use the base model
        as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.

        )r   r   r   r   )r"   r   r   r   s       r#   merge_and_unloadzBOFTModel.merge_and_unloadD  s#    $ 00#
- 1 
 	
r%   c                 &    | j                  d      S )z
        Gets back the base model by removing all the boft modules without merging. This gives back the original base
        model.
        F)r   r   r{   s    r#   unloadzBOFTModel.unloadZ  s    
 00u0==r%   )F)T)TFFN)FFN)!r!   
__module____qualname____doc__r   str__annotations__r   r$   staticmethodr*   rA   r7   r   Modulera   r4   ri   boolrs   rx   r|   r   r   r   r   r   r   r   rO   r   r   __classcell__)r    s   @r#   r   r   +   sY   > FC
 t  < <'R38Zbii ZD Z*  0- - 0
/	0+   ! -1    	 
  S	* DV3 V4 V0 im

59
RZ[_`c[dRe
	
,> >r%   r   )rd   dataclassesr   enumr   typingr   rO   r   r   peft.tuners.tuners_utilsr   r	   r
   r   
peft.utilsr   r   r   r   r   layerr   r   r   r    r%   r#   <module>r      sI   $           , ,t>	 t>r%   