
    bi4                         d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ d dl	m	Z	 d dl
mZmZmZ d dlmZmZmZ d	d
lmZ d	dlmZmZ  G d de      Zy)    N)asdict)Enum)Optional)nn)tqdm)	BaseTunerBaseTunerLayercheck_target_module_exists)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )
MissConfig)	MissLayer
MissLinearc            	           e Zd ZU dZdZeed<   deddfdZe	d        Z
d	 Zd
 Zdej                  ddfdZe	d        Zdef fdZddefdZd dZd Zd Zd Ze	d        Z	 	 	 	 d!dededeee      fdZdeddfdZ	 d"dededeee      dej                  j                  fdZdej                  j                  fdZ xZ S )#	MissModela  
    Creates Householder reflection adaptation (MiSS) model from a pretrained model. The method is described in
    https://huggingface.co/papers/2409.15371

    Args:
        model (`torch.nn.Module`): The model to which the adapter tuner layers will be attached.
        config ([`MissConfig`]): The configuration of the MiSS model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The MiSS model.

    Example:
        ```py
        >>> from diffusers import StableDiffusionPipeline
        >>> from peft import MissModel, MissConfig

        >>> config_te = MissConfig(
        ...     r=8,
        ...     target_modules=["k_proj", "q_proj", "v_proj", "out_proj", "fc1", "fc2"],
        ...     init_weights=True,
        ... )
        >>> config_unet = MissConfig(
        ...     r=8,
        ...     target_modules=[
        ...         "proj_in",
        ...         "proj_out",
        ...         "to_k",
        ...         "to_q",
        ...         "to_v",
        ...         "to_out.0",
        ...         "ff.net.0.proj",
        ...         "ff.net.2",
        ...     ],
        ...     init_weights=True,
        ... )

        >>> model = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
        >>> model.text_encoder = MissModel(model.text_encoder, config_te, "default")
        >>> model.unet = MissModel(model.unet, config_unet, "default")
        ```

    **Attributes**:
        - **model** ([`~torch.nn.Module`]) -- The model to be adapted.
        - **peft_config** ([`MissConfig`]): The configuration of the MiSS model.
    miss_prefixconfigreturnNc                     t        | j                        dkD  r2|j                  dk7  r"t        | j                  j
                   d      yy)z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueError	__class____name__)selfr   s     Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/miss/model.py_check_new_adapter_configz#MissModel._check_new_adapter_configW   sR       !A%FKK6,A>>**+ ,7 7  -B%    c                     t        | |      S N)r
   )miss_configkeys     r!   _check_target_module_existsz%MissModel._check_target_module_existsf   s    )+s;;r#   c                    |t        d      t        |d      xr |j                  d u}|j                  |j                  |j
                  |j                  d}	||	d<   t        |t              sI | j                  |||fi |	}
|| j                  vr|
j                  d       | j                  |||
|       y |j                  ||j                  |j                  |j
                  |j                         y )NzCurrent Key shouldn't be `None`r   )rmini_rmiss_dropoutinit_weightsF)r*   r-   r,   r+   )r   hasattrr   r*   r+   r,   r-   
isinstancer   _create_new_moduleactive_adaptersrequires_grad__replace_moduleupdate_layer)r    r&   adapter_nametargettarget_nameparentcurrent_keyoptional_kwargsr   kwargs
new_modules              r!   _create_and_replacezMissModel._create_and_replacej   s     >??vv&B6;;d+B!(('44'44	
 v &),000lF]V\]J4#7#77))%0  j&I--(55(55"))   r#   c                    t        |||       t        |d      r|j                  }t        |d      s.|j                  |_        t        |d      r|j                  |_        t        |dd       ^t        |d      r|j                  |j                  _        n|j                  |_        |j                  |j                  j                         t        j                  d      |j                         D ]\  \  }}| j                  |v st        fd|j                         D              r8|j                  |j                  j                         ^ y )N
base_layerr   statemetac              3   <   K   | ]  }|j                   k(    y wr%   )device).0prA   s     r!   	<genexpr>z,MissModel._replace_module.<locals>.<genexpr>   s     I188t+Is   )setattrr.   r?   weightr   getattrr@   torC   torchnamed_modulesr   any
parameters)r    r8   
child_namer<   childnamemodulerA   s          @r!   r3   zMissModel._replace_module   s   
J/
 5,'$$Ez<0 %Juf%"'**
5'4(4z<0.3kk
%%+#(;;
 MM%,,--.||F#&446 	3LD&{{d"IV5F5F5HIIIIell112	3r#   modelc                    |j                         D ]  \  }}| j                  |vsd|_         | j                  D ]  }| j                  |   j
                  }|dk(  r"|dk(  r%|j                         D ]  \  }}d|v sd|_         L|dk(  rU|j                         D ]A  \  }}t        |t              st        |d      s$|j
                  1d|j
                  _        C t        d| d       y )	NFr   allr   T	miss_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradr1   r   r   rL   r/   r   r.   NotImplementedError)r    rS   nrE   active_adapterr   rQ   ms           r!    _mark_only_adapters_as_trainablez*MissModel._mark_only_adapters_as_trainable   s   **, 	(DAq{{!#"'	( #22 	ZN##N388Dv~u}!224 /DAq{*./ $$224 4GD!!!Y/GAv4F166K]/3,4 *,<TFBW*XYY	Zr#   c                     t        |t              r|j                         }n|}t        |t        j                  j
                        rt        ||fi |}|S t        d| d      )NzTarget module zB is not supported. Currently, only `torch.nn.Linear` is supported.)r/   r	   get_base_layerrK   r   Linearr   r   )r&   r5   r6   r;   target_base_layerr<   s         r!   r0   zMissModel._create_new_module   si    fn- & 5 5 7 &'9#FLCFCJ 	  (jk r#   rQ   c                 z    	 t         |   |      S # t        $ r |dk(  r t        | j                  |      cY S w xY w)z1Forward missing attributes to the wrapped module.
base_model)super__getattr__AttributeErrorrI   rS   )r    rQ   r   s     r!   re   zMissModel.__getattr__   sC    	-7&t,, 	-|#4::t,,	-s    %::	inferencec           
         i }| j                   j                         D ]U  \  }}t        |      j                         D ci c]$  \  }}|t        |t              r|j
                  n|& }}}|sQd|d<   W |<   |S c c}}w )NTinference_mode)r   itemsr   r/   r   value)r    rg   config_dictr'   rk   kvr   s           r!   get_peft_config_as_dictz!MissModel.get_peft_config_as_dict   s    **002 	0JCKQRW=K^K^K`a41aaJq$$7Q>aFa+/'(	0 "C	 bs   )A<c                     | j                   j                         D ]*  }t        |t        t        f      s|j                  |       , y r%   )rS   modulesr/   r	   r   enable_adapters)r    enabledrR   s      r!   _set_adapter_layerszMissModel._set_adapter_layers   s<    jj((* 	0F&>3G"HI&&w/	0r#   c                 (    | j                  d       y )NTrs   )rt   r    s    r!   enable_adapter_layerszMissModel.enable_adapter_layers   s       .r#   c                     | j                   D ]<  }| j                  |   j                  }|dk7  s"d| d}t        j                  |       > | j                  d       y )Nr   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Frv   )r1   r   r   warningswarnrt   )r    r[   valmsgs       r!   disable_adapter_layersz MissModel.disable_adapter_layers   sp    "22 	#N"">277Cf}TUXTY ZG G  c"	# 	   /r#   c                     | j                   j                         D ]U  }t        |t              s|j                  r%t        j                  d       |j                          |j                  |       W || _	        y )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
rS   rq   r/   r   mergedrz   r{   unmergeset_adapterr[   )r    r5   rR   s      r!   r   zMissModel.set_adapter   s^    jj((* 	1F&),==MM"noNN$""<0	1 +r#   c                 ~    | j                   0|d   t        vrt        d      t        t        |d            | _         | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r   set)r   model_configs     r!   _prepare_adapter_configz!MissModel._prepare_adapter_config   sK    %%-L)1cc !STT),B<P\C]^*K& r#   progressbar
safe_mergeadapter_namesc                 D   | j                  |       | j                  j                         D cg c]  \  }}| j                  |vs| }}}d|rdndz   dz   }t	        || |      D ]  }	 t        | j                  |      \  }	}
}t        |
d      r8|r|
j                  ||       | j                  |	||
j                         |
       bt        |
t              sst        |	||
j                  |
j                             | j                  S c c}}w # t        $ r Y w xY w)Nz
Unloading zand merging  rS   )disabledescr?   )r   r   )_unloading_checksrS   rL   r   r   r   rf   r.   merger3   r_   r/   r   rG   modules_to_saver[   )r    r   r   r   r   r'   _key_listr   r8   r6   r7   s               r!   _unload_and_optionally_mergez&MissModel._unload_and_optionally_merge	  s    	}-&*jj&>&>&@[FCDKKWZDZC[[~B?'Ik/E 	\C.=djj#.N+ v|,LLJmLT$$V[&:O:O:QSYZF$89V-C-CFDYDY-Z[	\ zz! \
 " s   DD,D	DDr5   c                    |t        | j                  j                               vrt        d| d      | j                  |= | j                  j                         D cg c]  \  }}| j                  |vs| }}}d}|D ]P  }t        | j                  |      \  }}}t        |t              s.|j                  |       |B|j                  dd }R |xs g | _        | j                  ||       yc c}}w )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not existN)new_active_adapters)listr   keysr   rS   rL   r   r   r/   r   delete_adapterr1   r[   _delete_auxiliary_adapter)r    r5   r'   r   r   new_adapterr6   s          r!   r   zMissModel.delete_adapter#  s     tD$4$4$9$9$;<<x~_EFF\*&*jj&>&>&@[FCDKKWZDZC[[ 	<C*4::s;LAvq&),%%l3&"("8"8";K	< */R&&|&U \s   C36C3c                 *    | j                  |||      S )a  
        This method merges the MiSS layers into the base model. This is needed if someone wants to use the base model
        as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.

        )r   r   r   r   )r    r   r   r   s       r!   merge_and_unloadzMissModel.merge_and_unload:  s#    $ 00#
- 1 
 	
r#   c                 &    | j                  d      S )z
        Gets back the base model by removing all the miss modules without merging. This gives back the original base
        model.
        F)r   r   rw   s    r!   unloadzMissModel.unloadP  s    
 00u0==r#   )F)T)TFFN)FFN)!r   
__module____qualname____doc__r   str__annotations__r   r"   staticmethodr(   r=   r3   r   Moduler]   r0   re   boolro   rt   rx   r~   r   r   r   r   r   r   rK   r   r   __classcell__)r   s   @r!   r   r   #   sY   /b FC
 t  < <$L38Zbii ZD Z*  - - 0
/	0+   ! -1  	
  S	*4V3 V4 V0 im

59
RZ[_`c[dRe
	
,> >r#   r   )rz   dataclassesr   enumr   typingr   rK   r   r   peft.tuners.tuners_utilsr   r	   r
   
peft.utilsr   r   r   r   r   layerr   r   r    r#   r!   <module>r      sB           Z Z   (r>	 r>r#   