
    bi3                         d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ d dl	m	Z	 d dl
mZmZmZ d dlmZmZmZ d	d
lmZ d	dlmZmZ  G d de      Zy)    N)asdict)Enum)Optional)nn)tqdm)	BaseTunerBaseTunerLayercheck_target_module_exists)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )
BoneConfig)	BoneLayer
BoneLinearc            	           e Zd ZU dZdZeed<   deddfdZe	d        Z
d	 Zd
 Zdej                  ddfdZe	d        Zdef fdZddefdZd dZd Zd Zd Ze	d        Z	 	 	 	 d!dededeee      fdZdeddfdZ	 d"dededeee      dej                  j                  fdZdej                  j                  fdZ xZ S )#	BoneModela  
    Creates Householder reflection adaptation (Bone) model from a pretrained model. The method is described in
    https://huggingface.co/papers/2409.15371

    Args:
        model (`torch.nn.Module`): The model to which the adapter tuner layers will be attached.
        config ([`BoneConfig`]): The configuration of the Bone model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The Bone model.

    Example:
        ```py
        >>> from diffusers import StableDiffusionPipeline
        >>> from peft import BoneModel, BoneConfig

        >>> config_te = BoneConfig(
        ...     r=8,
        ...     target_modules=["k_proj", "q_proj", "v_proj", "out_proj", "fc1", "fc2"],
        ...     init_weights=True,
        ... )
        >>> config_unet = BoneConfig(
        ...     r=8,
        ...     target_modules=[
        ...         "proj_in",
        ...         "proj_out",
        ...         "to_k",
        ...         "to_q",
        ...         "to_v",
        ...         "to_out.0",
        ...         "ff.net.0.proj",
        ...         "ff.net.2",
        ...     ],
        ...     init_weights=True,
        ... )

        >>> model = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
        >>> model.text_encoder = BoneModel(model.text_encoder, config_te, "default")
        >>> model.unet = BoneModel(model.unet, config_unet, "default")
        ```

    **Attributes**:
        - **model** ([`~torch.nn.Module`]) -- The model to be adapted.
        - **peft_config** ([`BoneConfig`]): The configuration of the Bone model.
    bone_prefixconfigreturnNc                     t        | j                        dkD  r2|j                  dk7  r"t        | j                  j
                   d      yy)z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueError	__class____name__)selfr   s     Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/bone/model.py_check_new_adapter_configz#BoneModel._check_new_adapter_configW   sR       !A%FKK6,A>>**+ ,7 7  -B%    c                     t        | |      S N)r
   )bone_configkeys     r!   _check_target_module_existsz%BoneModel._check_target_module_existsf   s    )+s;;r#   c                    |t        d      t        |d      xr |j                  d u}|j                  |j                  d}	||	d<   t        |t              sI | j                  |||fi |	}
|| j                  vr|
j                  d       | j                  |||
|       y |j                  ||j                  |j                         y )NzCurrent Key shouldn't be `None`r   )rinit_weightsF)r   hasattrr   r*   r+   
isinstancer   _create_new_moduleactive_adaptersrequires_grad__replace_moduleupdate_layer)r    r&   adapter_nametargettarget_nameparentcurrent_keyoptional_kwargsr   kwargs
new_modules              r!   _create_and_replacezBoneModel._create_and_replacej   s     >??vv&B6;;d+B'44
 v &),000lF]V\]J4#7#77))%0  j&I--(55   r#   c                    t        |||       t        |d      r|j                  }t        |d      s.|j                  |_        t        |d      r|j                  |_        t        |dd       ^t        |d      r|j                  |j                  _        n|j                  |_        |j                  |j                  j                         t        j                  d      |j                         D ]\  \  }}| j                  |v st        fd|j                         D              r8|j                  |j                  j                         ^ y )N
base_layerr   statemetac              3   <   K   | ]  }|j                   k(    y wr%   )device).0pr?   s     r!   	<genexpr>z,BoneModel._replace_module.<locals>.<genexpr>   s     I188t+Is   )setattrr,   r=   weightr   getattrr>   torA   torchnamed_modulesr   any
parameters)r    r6   
child_namer:   childnamemoduler?   s          @r!   r1   zBoneModel._replace_module   s   
J/
 5,'$$Ez<0 %Juf%"'**
5'4(4z<0.3kk
%%+#(;;
 MM%,,--.||F#&446 	3LD&{{d"IV5F5F5HIIIIell112	3r#   modelc                    |j                         D ]  \  }}| j                  |vsd|_         | j                  D ]  }| j                  |   j
                  }|dk(  r"|dk(  r%|j                         D ]  \  }}d|v sd|_         L|dk(  rU|j                         D ]A  \  }}t        |t              st        |d      s$|j
                  1d|j
                  _        C t        d| d       y )	NFr   allr   T	bone_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradr/   r   r   rJ   r-   r   r,   NotImplementedError)r    rQ   nrC   active_adapterr   rO   ms           r!    _mark_only_adapters_as_trainablez*BoneModel._mark_only_adapters_as_trainable   s   **, 	(DAq{{!#"'	( #22 	ZN##N388Dv~u}!224 /DAq{*./ $$224 4GD!!!Y/GAv4F166K]/3,4 *,<TFBW*XYY	Zr#   c                     t        |t              r|j                         }n|}t        |t        j                  j
                        rt        ||fi |}|S t        d| d      )NzTarget module zB is not supported. Currently, only `torch.nn.Linear` is supported.)r-   r	   get_base_layerrI   r   Linearr   r   )r&   r3   r4   r9   target_base_layerr:   s         r!   r.   zBoneModel._create_new_module   si    fn- & 5 5 7 &'9#FLCFCJ 	  (jk r#   rO   c                 z    	 t         |   |      S # t        $ r |dk(  r t        | j                  |      cY S w xY w)z1Forward missing attributes to the wrapped module.
base_model)super__getattr__AttributeErrorrG   rQ   )r    rO   r   s     r!   rc   zBoneModel.__getattr__   sC    	-7&t,, 	-|#4::t,,	-s    %::	inferencec           
         i }| j                   j                         D ]U  \  }}t        |      j                         D ci c]$  \  }}|t        |t              r|j
                  n|& }}}|sQd|d<   W |<   |S c c}}w )NTinference_mode)r   itemsr   r-   r   value)r    re   config_dictr'   ri   kvr   s           r!   get_peft_config_as_dictz!BoneModel.get_peft_config_as_dict   s    **002 	0JCKQRW=K^K^K`a41aaJq$$7Q>aFa+/'(	0 "C	 bs   )A<c                     | j                   j                         D ]*  }t        |t        t        f      s|j                  |       , y r%   )rQ   modulesr-   r	   r   enable_adapters)r    enabledrP   s      r!   _set_adapter_layerszBoneModel._set_adapter_layers   s<    jj((* 	0F&>3G"HI&&w/	0r#   c                 (    | j                  d       y )NTrq   )rr   r    s    r!   enable_adapter_layerszBoneModel.enable_adapter_layers   s       .r#   c                     | j                   D ]<  }| j                  |   j                  }|dk7  s"d| d}t        j                  |       > | j                  d       y )Nr   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Frt   )r/   r   r   warningswarnrr   )r    rY   valmsgs       r!   disable_adapter_layersz BoneModel.disable_adapter_layers   sp    "22 	#N"">277Cf}TUXTY ZG G  c"	# 	   /r#   c                     | j                   j                         D ]U  }t        |t              s|j                  r%t        j                  d       |j                          |j                  |       W || _	        y )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
rQ   ro   r-   r   mergedrx   ry   unmergeset_adapterrY   )r    r3   rP   s      r!   r   zBoneModel.set_adapter   s^    jj((* 	1F&),==MM"noNN$""<0	1 +r#   c                 ~    | j                   0|d   t        vrt        d      t        t        |d            | _         | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r   set)r   model_configs     r!   _prepare_adapter_configz!BoneModel._prepare_adapter_config   sK    %%-L)1cc !STT),B<P\C]^*K& r#   progressbar
safe_mergeadapter_namesc                 D   | j                  |       | j                  j                         D cg c]  \  }}| j                  |vs| }}}d|rdndz   dz   }t	        || |      D ]  }	 t        | j                  |      \  }	}
}t        |
d      r8|r|
j                  ||       | j                  |	||
j                         |
       bt        |
t              sst        |	||
j                  |
j                             | j                  S c c}}w # t        $ r Y w xY w)Nz
Unloading zand merging  rQ   )disabledescr=   )r   r   )_unloading_checksrQ   rJ   r   r   r   rd   r,   merger1   r]   r-   r   rE   modules_to_saverY   )r    r   r   r   r   r'   _key_listr   r6   r4   r5   s               r!   _unload_and_optionally_mergez&BoneModel._unload_and_optionally_merge  s    	}-&*jj&>&>&@[FCDKKWZDZC[[~B?'Ik/E 	\C.=djj#.N+ v|,LLJmLT$$V[&:O:O:QSYZF$89V-C-CFDYDY-Z[	\ zz! \
 " s   DD,D	DDr3   c                    |t        | j                  j                               vrt        d| d      | j                  |= | j                  j                         D cg c]  \  }}| j                  |vs| }}}d}|D ]P  }t        | j                  |      \  }}}t        |t              s.|j                  |       |B|j                  dd }R |xs g | _        | j                  ||       yc c}}w )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not existN)new_active_adapters)listr   keysr   rQ   rJ   r   r   r-   r   delete_adapterr/   rY   _delete_auxiliary_adapter)r    r3   r'   r   r   new_adapterr4   s          r!   r   zBoneModel.delete_adapter  s     tD$4$4$9$9$;<<x~_EFF\*&*jj&>&>&@[FCDKKWZDZC[[ 	<C*4::s;LAvq&),%%l3&"("8"8";K	< */R&&|&U \s   C36C3c                 *    | j                  |||      S )a  
        This method merges the Bone layers into the base model. This is needed if someone wants to use the base model
        as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.

        )r   r   r   r   )r    r   r   r   s       r!   merge_and_unloadzBoneModel.merge_and_unload6  s#    $ 00#
- 1 
 	
r#   c                 &    | j                  d      S )z
        Gets back the base model by removing all the bone modules without merging. This gives back the original base
        model.
        F)r   r   ru   s    r!   unloadzBoneModel.unloadL  s    
 00u0==r#   )F)T)TFFN)FFN)!r   
__module____qualname____doc__r   str__annotations__r   r"   staticmethodr(   r;   r1   r   Moduler[   r.   rc   boolrm   rr   rv   r|   r   r   r   r   r   r   rI   r   r   __classcell__)r   s   @r!   r   r   #   sY   /b FC
 t  < < D38Zbii ZD Z*  - - 0
/	0+   ! -1  	
  S	*4V3 V4 V0 im

59
RZ[_`c[dRe
	
,> >r#   r   )rx   dataclassesr   enumr   typingr   rI   r   r   peft.tuners.tuners_utilsr   r	   r
   
peft.utilsr   r   r   r   r   layerr   r   r    r#   r!   <module>r      sB           Z Z   (n>	 n>r#   