
    bi8                        d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZ ddlmZ ddlmZmZ  G d de      Zy)    )annotationsN)asdict)Enum)chain)Optional)tqdm)Conv1D)	BaseTunerBaseTunerLayercheck_target_module_exists)7TRANSFORMERS_MODELS_TO_FOURIERFT_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )FourierFTConfig)FourierFTLayerFourierFTLinearc                       e Zd ZU dZdZded<   ddZed        Zd Z	d Z
dd	Zed
        Zd fdZdddZdddZddZddZddZed        Z	 	 	 	 d	 	 	 	 	 d dZd!dZ	 d"	 	 	 	 	 	 	 d#dZd$dZ xZS )%FourierFTModela*  
    Creates FourierFT model from a pretrained transformers model.

    The method is described in detail in https://huggingface.co/papers/2405.03003.

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`FourierFTConfig`]): The configuration of the FourierFT model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The FourierFT model.

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`FourierFTConfig`]): The configuration of the Fourier model.
    
fourierft_strprefixc                    t        | j                        dkD  r2|j                  dk7  r"t        | j                  j
                   d      yy)z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueError	__class____name__)selfconfigs     V/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/fourierft/model.py_check_new_adapter_configz(FourierFTModel._check_new_adapter_config=   sR       !A%FKK6,A>>**+ ,7 7  -B%    c                    t        | |      S N)r   )fourierft_configkeys     r#   _check_target_module_existsz*FourierFTModel._check_target_module_existsL   s    )*:C@@r%   c                   t        d      t        t        |j                  j	                                     }t        t        fd|            }	|j                  j                  |	|j                        }
|j                  }|j                  }t        |d      xr |j                  d u}|
||j                  |j                  |j                  d}||d<   t        |t               r |j#                  ||
||j                  |       y  | j$                  |||fi |}|| j&                  k7  r|j)                  d       | j+                  ||||       y )NzCurrent Key shouldn't be `None`c                8    t        j                  d|  d      S )Nz.*\.$)rematch)r)   current_keys    r#   <lambda>z4FourierFTModel._create_and_replace.<locals>.<lambda>^   s    "((d3%q>;2W r%   r   )n_frequencyscalingfan_in_fan_outinit_weightsrandom_loc_seedF)r   listr   n_frequency_patternkeysnextfiltergetr2   r3   r6   hasattrr   r4   r5   
isinstancer   update_layer_create_new_moduleactive_adapterrequires_grad__replace_module)r!   r(   adapter_nametargettarget_nameparentr0   optional_kwargspattern_keystarget_name_keyr2   r3   r6   r   kwargs
new_modules         `         r#   _create_and_replacez"FourierFTModel._create_and_replaceP   sK    >??E"2"F"F"K"K"MNOv&WYefhst&::>>P`PlPlm"***::vv&B6;;d+B&.==,99/??
 vfn- -- 1001A<QWb[abJt222))%0  j&Ir%   c                   t        |||       t        |d      r|j                  }t        |d      s.|j                  |_        t        |d      r|j                  |_        t        |dd       ^t        |d      r|j                  |j                  _        n|j                  |_        |j                  |j                  j                         t        j                  d      |j                         D ]R  \  }}d|v st        fd|j                         D              r.|j                  |j                  j                         T y )N
base_layerr   statemetar   c              3  <   K   | ]  }|j                   k(    y wr'   )device).0prQ   s     r#   	<genexpr>z1FourierFTModel._replace_module.<locals>.<genexpr>   s     I188t+Is   )setattrr=   rO   weightr   getattrrP   torS   torchnamed_modulesany
parameters)r!   rG   
child_namerL   childnamemodulerQ   s          @r#   rC   zFourierFTModel._replace_module{   s    
J/
 5,'$$Ez<0 %Juf%"'**
5'4(4z<0.3kk
%%+#(;;
 MM%,,--.||F#&446 	3LD&t#IV5F5F5HIIIIell112	3r%   c                   |j                         D ]  \  }}| j                  |vsd|_         | j                  D ]  }| j                  |   j
                  }|dk(  r"|dk(  r%|j                         D ]  \  }}d|v sd|_         L|dk(  rR|j                         D ]>  }t        |t              st        |d      s!|j
                  .d|j
                  _        @ t        d| d       y )	NFr   allr   Tfourier_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradactive_adaptersr   r   modulesr>   r   r=   NotImplementedError)r!   modelnrU   rA   r   ms          r#    _mark_only_adapters_as_trainablez/FourierFTModel._mark_only_adapters_as_trainable   s    **, 	(DAq{{!#"'	( #22 	ZN##N388Dv~u}!224 /DAq{*./ ' 4A!!^4F9KPQPVPVPb/3,4 *,<TFBW*XYY	Zr%   c                   t        |t              r|j                         }n|}t        |t        j                  j
                        r'|d   rmt        j                  d       dx|d<   | _        nKt        |t              r,d|d<   |d   s1t        j                  d       dx|d<   | _        nt        d| d      t        ||fi |}|S )	Nr4   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.FTis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module zZ is not supported. Currently, only the following modules are supported: `torch.nn.Linear`.)r>   r   get_base_layerr[   nnLinearwarningswarnr4   r	   r   r   )r(   rD   rE   rK   target_base_layerrL   s         r#   r@   z!FourierFTModel._create_new_module   s    fn- & 5 5 7 &'9&'7 NSR'(+;+J)6204F,-*+w NRQ'(+;+J  )% % 
 %V\DVD
r%   c                z    	 t         |   |      S # t        $ r |dk(  r t        | j                  |      cY S w xY w)z1Forward missing attributes to the wrapped module.rk   )super__getattr__AttributeErrorrY   rk   )r!   ra   r   s     r#   ry   zFourierFTModel.__getattr__   sB    	-7&t,, 	-w4::t,,	-s    %::c           
        i }| j                   j                         D ]U  \  }}t        |      j                         D ci c]$  \  }}|t        |t              r|j
                  n|& }}}|sQd|d<   W |<   |S c c}}w )NTinference_mode)r   itemsr   r>   r   value)r!   	inferenceconfig_dictr)   r~   kvr"   s           r#   get_peft_config_as_dictz&FourierFTModel.get_peft_config_as_dict   s    **002 	0JCKQRW=K^K^K`a41aaJq$$7Q>aFa+/'(	0 "C	 bs   )A<c                    | j                   j                         D ]*  }t        |t        t        f      s|j                  |       , y r'   )rk   ri   r>   r   r   enable_adapters)r!   enabledrb   s      r#   _set_adapter_layersz"FourierFTModel._set_adapter_layers   s<    jj((* 	0F&>3G"HI&&w/	0r%   c                (    | j                  d       y)zyEnable all adapters.

        Call this if you have previously disabled all adapters and want to re-enable them.
        Tr   N)r   r!   s    r#   enable_adapter_layersz$FourierFTModel.enable_adapter_layers   s    
 	   .r%   c                    | j                   D ]<  }| j                  |   j                  }|dk7  s"d| d}t        j                  |       > | j                  d       y)zDisable all adapters.

        When disabling all adapters, the model output corresponds to the output of the base model.
        r   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Fr   N)rh   r   r   rt   ru   r   )r!   rA   valmsgs       r#   disable_adapter_layersz%FourierFTModel.disable_adapter_layers   sr    
 #22 	#N"">277Cf}TUXTY ZG G  c"	# 	   /r%   c                    | j                   j                         D ]U  }t        |t              s|j                  r%t        j                  d       |j                          |j                  |       W || _	        y)zSet the active adapter(s).

        Args:
            adapter_name (`str` or `list[str]`): Name of the adapter(s) to be activated.
        zJAdapter cannot be set when the model is merged. Unmerging the model first.N)
rk   ri   r>   r   mergedrt   ru   unmergeset_adapterrA   )r!   rD   rb   s      r#   r   zFourierFTModel.set_adapter   s`     jj((* 	1F&.1==MM"noNN$""<0	1 +r%   c                ~    | j                   0|d   t        vrt        d      t        t        |d            | _         | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r   set)r   model_configs     r#   _prepare_adapter_configz&FourierFTModel._prepare_adapter_config  sK    %%-L)1hh !STT),GUaHbc*K& r%   c                "   | j                   j                         D cg c]  \  }}| j                  |vs| }}}d|rdndz   dz   }t        || |      D ]  }	 t	        | j                   |      \  }	}
}t        |
d      r8|r|
j                  ||       | j                  |	||
j                         |
       bt        |
t              sst        |	||
j                  |
j                             | j                   S c c}}w # t
        $ r Y w xY w)Nz
Unloading zand merging  rk   )disabledescrO   )
safe_mergeadapter_names)rk   r\   r   r   r   rz   r=   mergerC   rq   r>   r   rW   modules_to_saverA   )r!   r   progressbarr   r   r)   _key_listr   rG   rE   rF   s               r#   _unload_and_optionally_mergez+FourierFTModel._unload_and_optionally_merge  s
    '+jj&>&>&@[FCDKKWZDZC[[~B?'Ik/E 	\C.=djj#.N+ v|,LLJmLT$$V[&:O:O:QSYZF$89V-C-CFDYDY-Z[	\ zz! \
 " s   C<C<D	DDc                   |t        | j                  j                               vrt        d| d      | j                  |= | j                  j                         D cg c]  \  }}d|vs| }}}d}|D ]P  }t        | j                  |      \  }}}t        |t              s.|j                  |       |B|j                  dd }R |xs g | _
        | j                  ||       yc c}}w )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not exist	fourierftN)new_active_adapters)r7   r   r9   r   rk   r\   r   r>   r   delete_adapterrA   _delete_auxiliary_adapter)r!   rD   r)   r   r   new_adapterrE   s          r#   r   zFourierFTModel.delete_adapter)  s     tD$4$4$9$9$;<<x~_EFF\* '+jj&>&>&@[FCKWZDZC[[ 	;C*4::s;LAvq&.1%%l3&"("7"7":K	; */R&&|&U \s   C),C)c                *    | j                  |||      S )a  
        This method merges the Fourier layers into the base model. This is needed if someone wants to use the base
        model as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        )r   r   r   r   )r!   r   r   r   s       r#   merge_and_unloadzFourierFTModel.merge_and_unloadA  s#    " 00#
- 1 
 	
r%   c                &    | j                  d      S )z
        Gets back the base model by removing all the Fourier modules without merging. This gives back the original base
        model.
        F)r   r   r   s    r#   unloadzFourierFTModel.unloadV  s    
 00u0==r%   )r"   r   returnNone)rk   torch.nn.Moduler   r   )ra   r   )F)r   bool)T)r   r   r   r   )r   r   )rD   zstr | list[str]r   r   )TFFN)r   r   r   r   r   Optional[list[str]])rD   r   )FFN)r   r   r   r   r   r   r   r   )r   r   )r    
__module____qualname____doc__r   __annotations__r$   staticmethodr*   rM   rC   rn   r@   ry   r   r   r   r   r   r   r   r   r   r   __classcell__)r   s   @r#   r   r   &   s    ( FC A A)JV38Z*  <-0
/0+   ! -1  	
 +2V2 im

59
Re
	
*>r%   r   )
__future__r   r.   rt   dataclassesr   enumr   	itertoolsr   typingr   r[   r   transformers.pytorch_utilsr	   peft.tuners.tuners_utilsr
   r   r   
peft.utilsr   r   r   r"   r   layerr   r   r    r%   r#   <module>r      sK    # 	        - Z Z  $ 2u>Y u>r%   