
    bi=                    n   d dl mZ d dlZd dlmZmZmZ d dlmZ d dl	m	Z	 d dl
mZmZmZmZmZmZ d dlmZmZmZ d dlmZmZmZmZmZ ej6                  ej8                  ej:                  ej<                  ej>                  ej@                  fZ!ejD                  jF                  ejH                  jF                  ejJ                  jF                  ejL                  jF                  ejN                  jF                  gZ(eejR                  ejT                  ejV                  ejX                  ejZ                  ej\                  f   Z/ej`                  jb                  ej`                  jd                  ej`                  jf                  ej`                  jh                  ejj                  ejl                  fZ7 G d	 d
e      Z8y)    )annotationsN)AnyOptionalUnion)nn)tqdm)adaloralohalokrloraoftshira)	BaseTunerBaseTunerLayercheck_target_module_exists)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapperPeftType_get_submodulesget_auto_gptq_quant_linearc                      e Zd ZdZd fdZddZedd       Z	 	 	 	 	 	 	 	 ddZddZ	ddZ
ed        Zd fd	Zdd
Zd Zd ZddZed        Z	 	 	 	 d	 	 	 	 	 ddZd dZddZ	 d!	 	 	 	 	 	 	 d"dZd#dZd$dZ xZS )%
MixedModela  
    A class that allows to mix different types of adapters in a single model.

    Note: This class should usually not be initialized directly. Instead, use `get_peft_model` with the argument
    `mixed=True`.

    Args:
        model (:obj:`nn.Module`):
            The model to be tuned.
        config (:obj:`PeftConfig`):
            The config of the model to be tuned. The adapter type must be compatible.
        adapter_name (:obj:`str`):
            The name of the first adapter.
    c                (    t         |   |||       y N)super__init__)selfmodelconfigadapter_name	__class__s       R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/mixed/model.pyr   zMixedModel.__init__G   s    5    c           	     Z   t        |t        j                        s5t        | j                  j
                   dt         dt        |       d      d | j                  D        }|D cg c]	  }|dvs| }}t        |      dkD  r"t        | j                  j
                   d      yc c}w )	z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        z only supports z configs, but got .c              3  6   K   | ]  }t        |d d        yw)biasN)getattr).0r   s     r"   	<genexpr>z7MixedModel._check_new_adapter_config.<locals>.<genexpr>V   s     OF'&&$/Os   )Nnone   zf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)

isinstanceConfigs__args__
ValueErrorr!   __name__COMPATIBLE_TUNER_TYPEStypepeft_configlen)r   r   biasesr'   s       r"   _check_new_adapter_configz$MixedModel._check_new_adapter_configJ   s     &'"2"23>>**+?;Q:RRdeijpeqdrrst  Pd>N>NO#)H4T-G$HHv;?>>**+ ,7 7   Is   '	B(1B(c                    t        | |      S r   )r   )r   keys     r"   _check_target_module_existsz&MixedModel._check_target_module_exists^   s    )&#66r#   c                4   t        |t        j                        r%t        j                  j                  | |g|i | y t        |t
        j                        r%t        j                  j                  | |g|i | y t        |t        j                        r%t        j                  j                  | |g|i | y t        |t        j                        r%t        j                  j                  | |g|i | y t        |t        j                        r%t        j                   j                  | |g|i | y t        |t"        j$                        r%t#        j&                  j                  | |g|i | y t)        dt+        |       dt,         d      )NzUnsupported config type , should be one of r%   )r-   r	   AdaLoraConfigAdaLoraModel_create_and_replacer   
LoraConfig	LoraModelr
   
LoHaConfig	LoHaModelr   
LoKrConfig	LoKrModelr   	OFTConfigOFTModelr   ShiraConfig
ShiraModelr0   r3   r2   )r   r   argskwargss       r"   r?   zMixedModel._create_and_replaceb   s7    fg334  44T6SDSFS0NN..tVMdMfM0NN..tVMdMfM0NN..tVMdMfM.LL,,T6KDKFK 1 1200vOOO7V~EXYoXppqrssr#   c                   t        |||       t        |d      r|j                         }nt        |d      r|j                  }t        |d      s.|j                  |_        t        |d      r|j
                  |_        t        |dd       ^t        |d      r|j                  |j                  _        n|j                  |_        |j                  |j                  j                         |j                         D ]l  \  }t        fdt        D              r%|j                  |j                  j                         dv sH|j                  |j                  j                         n y )N
base_layerquant_linear_moduler'   statec              3  &   K   | ]  }|v  
 y wr    )r)   prefixnames     r"   r*   z-MixedModel._replace_module.<locals>.<genexpr>   s     9f6T>9   ranknum)setattrhasattrget_base_layerrN   weightr'   r(   rO   rM   todevicenamed_modulesanyPREFIXES)r   parent
child_name
new_modulechildmodulerS   s         @r"   _replace_modulezMixedModel._replace_modulew   s   
J/
 5,'((*EU12--Ez<0 %Juf%"'**
5'4(4z<0.3kk
%%+#(;;
 MM%,,--. '446 	/LD&999		%,,--.D 		%,,--.		/r#   c                   |j                         D ]%  \  }t        fdt        D              rd|_        ' | j                  D ]  }t        | j                  |   dd      }|dk(  r#|dk(  r%|j                         D ]  \  }dv sd|_         M|dk(  rR|j                         D ]>  }t        |t              st        |d      s!|j                  .d|j                  _        @ t        d| d	       y )
Nc              3  &   K   | ]  }|v  
 y wr   rQ   )r)   rR   ns     r"   r*   z>MixedModel._mark_only_adapters_as_trainable.<locals>.<genexpr>   s     :vv{:rT   Fr'   r+   allT	lora_onlyzRequested bias: z, is not implemented.)named_parametersr]   r^   requires_gradactive_adaptersr(   r4   modulesr-   LayersrW   r'   r0   )r   r   pactive_adapterr'   mrg   s         @r"    _mark_only_adapters_as_trainablez+MixedModel._mark_only_adapters_as_trainable   s    **, 	(DAq:::"'	( #22 	QN4++N;VVLDv~u}!224 /DAq{*./ $ 4A!!V,F1CHZ/3,4 !#3D69N!OPP	Qr#   c                F   |j                  dd       }t        |      }||#t        d| j                  j                   d      |j                  dd      }|j                  dd      }|s|r#t        d| j                  j                   d      t        | t        j                        r$t        j                  j                  | ||fi |}|S t        | t        j                        r$t        j                  j                  | ||fi |}|S t        | t        j                        r$t        j                   j                  | ||fi |}|S t        | t"        j$                        r$t#        j&                  j                  | ||fi |}|S t        | t(        j*                        r$t)        j,                  j                  | ||fi |}|S t        | t.        j0                        r$t/        j2                  j                  | ||fi |}|S t        dt5        |        d	t6         d
      )Ngptq_quantization_configz$GPTQ quantization not supported for  (yet).loaded_in_8bitFloaded_in_4bitz-8bit and 4bit quantization not supported for zUnknown config type r<   r%   )getr   r0   	peft_typevaluepopr-   r	   r=   r>   _create_new_moduler   r@   rA   r
   rB   rC   r   rD   rE   r   rF   rG   r   rH   rI   r3   r2   )	r   r    targetrK   rt   AutoGPTQQuantLinearrv   rw   ra   s	            r"   r|   zMixedModel._create_new_module   s   #)::.H$#O 89QR$06I6UCFDTDTDZDZC[[bcdd$4e<$4e<^LVM]M]McMcLddklmmfg334 --@@W]haghJ  0::6<QWb[abJ  0::6<QWb[abJ  0::6<QWb[abJ  .88v`Y_`J
 	  1 12))<<V\SYd]cdJ  3DL>ATUkTllmnoor#   c                z    	 t         |   |      S # t        $ r |dk(  r t        | j                  |      cY S w xY w)z1Forward missing attributes to the wrapped module.r   )r   __getattr__AttributeErrorr(   r   )r   rS   r!   s     r"   r   zMixedModel.__getattr__   sB    	-7&t,, 	-w4::t,,	-s    %::c                    | j                   j                         D ]*  }t        |t        t        f      s|j                  |       , y r   )r   rm   r-   r   r   enable_adapters)r   enabledrc   s      r"   _set_adapter_layerszMixedModel._set_adapter_layers   s<    jj((* 	0F&>3G"HI&&w/	0r#   c                (    | j                  d       y )NTr   )r   r   s    r"   enable_adapter_layersz MixedModel.enable_adapter_layers   s       .r#   c                    | j                   D ]=  }t        | j                  |   dd      }|dk7  s#d| d}t        j                  |       ? | j                  d       y )Nr'   r+   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Fr   )rl   r(   r4   warningswarnr   )r   rp   valmsgs       r"   disable_adapter_layersz!MixedModel.disable_adapter_layers   ss    "22 	#N$**>:FFKCf}TUXTY ZG G  c"	# 	   /r#   c                    | j                   j                         D ]U  }t        |t              s|j                  r%t        j                  d       |j                          |j                  |       W || _	        y )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
r   rm   r-   rn   mergedr   r   unmergeset_adapterrp   )r   r    rc   s      r"   r   zMixedModel.set_adapter   s^    jj((* 	1F&&)==MM"noNN$""<0	1 +r#   c                ~    | j                   0|d   t        vrt        d      t        t        |d            | _         | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r0   set)r4   model_configs     r"   _prepare_adapter_configz"MixedModel._prepare_adapter_config   sK    %%-L)1cc !STT),B<P\C]^*K& r#   c                   |r%t        | j                  dd       dk(  rt        d      fd}| j                  j                         D cg c]   \  }t	        fdt
        D              r" }}}d|rdndz   d	z   }	t        || |	
      D ]  	 t        | j                        \  }
}}t        |d      r-|r ||       | j                  |
||j                         |       Wt        |t              sh|j                  |j                     }t        |d      r%|r|j!                         |j                         }t#        |
||        | j                  S c c}}w # t        $ r Y w xY w)Nquantization_methodgptqz4Cannot merge layers when the model is gptq quantizedc                   g }| }t        |d      r*|j                  |       |j                  }t        |d      r*t        |d d |dd        D ])  \  }}|j	                         |j                  |_        + | j	                         y )NrM   r,   
safe_mergeadapter_names)rW   appendrM   zipmerge)rc   pathlayerlayer_beforelayer_afterr   r   s        r"   merge_recursivelyzBMixedModel._unload_and_optionally_merge.<locals>.merge_recursively  s    DE%.E"(( %. .1crDH-E A)k!!Z}!U*5*@*@'A LLJmLLr#   c              3  &   K   | ]  }|v  
 y wr   rQ   r)   rR   r9   s     r"   r*   z:MixedModel._unload_and_optionally_merge.<locals>.<genexpr>  s     Kq^dFVYMKqrT   z
Unloading zand merging  r   )disabledescrM   r   )r(   r   r0   r\   r]   r^   r   r   r   rW   rd   rX   r-   r   modules_to_saverp   r   rV   )r   r   progressbarr   r   r   r9   _key_listr   r_   r}   target_namera   s      `` `       r"   _unload_and_optionally_mergez'MixedModel._unload_and_optionally_merge   sf    tzz#8$?6I !WXX
	M '+jj&>&>&@rrFCKqhpKqHqCrr~B?'Ik/E 	9C.=djj#.N+ v|,%f-$$V[&:O:O:QSYZF$89#33F4I4IJ
:|4"((Jm(\!+!:!:!<JZ8%	9( zz/ s " s    E0EE%%	E10E1c                H    t        d| j                  j                   d      )Nz(Weighted adapters are not supported for ru   )NotImplementedErrorr!   r1   r   rJ   rK   s      r"   add_weighted_adapterzMixedModel.add_weighted_adapter(  s#    !$LT^^MdMdLeel"mnnr#   c           	        t        |t              r|g}n|}t        |      t        | j                  j	                               z
  }|r;t        dt        |       dt        | j                  j	                                      |D ]  }| j                  |= | j                  j                         D cg c]   \  }t        fdt        D              r" }}}d}|D ]P  t        | j                        \  }}}t        |t              s.|j                  |       |B|j                  dd }R  xs g | _        | j!                  ||       yc c}}w )z
        Deletes an existing adapter.

        Args:
            adapter_name (Union[str, list[str]]): Name of the adapter(s) to delete.
        zAdapter(s) z  not found, available adapters: c              3  &   K   | ]  }|v  
 y wr   rQ   r   s     r"   r*   z,MixedModel.delete_adapter.<locals>.<genexpr>@  s     OubhPVZ]P]OurT   N)new_active_adapters)r-   strr   r4   keysr0   sortedr   r\   r]   r^   r   r   delete_adapterrl   rp   _delete_auxiliary_adapter)	r   r    r   
mismatchedr9   r   r   new_adapterr}   s	       `    r"   r   zMixedModel.delete_adapter+  sT    lC()NM(M'#d.>.>.C.C.E*FF
fZ011QRXY]YiYiYnYnYpRqQrs  * 
	@L  .*.***B*B*DvvQCOultOuLuvHvK @.tzz3?61fn5)),7"*&,&<&<Q&?@
	@ */R&&|&U ws   4 EEc                *    | j                  |||      S )a  
        This method merges the layers into the base model. This is needed if someone wants to use the base model as a
        standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        )r   r   r   r   )r   r   r   r   s       r"   merge_and_unloadzMixedModel.merge_and_unloadL  s#    " 00#
- 1 
 	
r#   c                &    | j                  d      S )z
        Gets back the base model by removing all the lora modules without merging. This gives back the original base
        model.
        F)r   r   r   s    r"   unloadzMixedModel.unloada  s    
 00u0==r#   c                :     | j                   j                  |i |S r   )r   generater   s      r"   r   zMixedModel.generateh  s    "tzz""D3F33r#   )r   	nn.Moduler   r.   r    r   returnNone)r   r.   r   r   )r   r.   r9   r   )r   r.   rJ   r   rK   r   r   r   )r   r   )r   r   r   r   )rS   r   )T)r    zUnion[str, list[str]]r   r   )TFFN)r   boolr   r   r   Optional[list[str]])rJ   r   rK   r   r   r   )FFN)r   r   r   r   r   r   r   r   )r   r   )rJ   r   rK   r   )r1   
__module____qualname____doc__r   r7   staticmethodr:   r?   rd   rr   r|   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__)r!   s   @r"   r   r   7   s   6( 7 7tt t 	t
 
t*/>Q,  6-0
/	0+   ! -1. . 	.
 +.`oVD im

59
Re
	
*>4r#   r   )9
__future__r   r   typingr   r   r   torchr   r   peft.tunersr	   r
   r   r   r   r   peft.tuners.tuners_utilsr   r   r   
peft.utilsr   r   r   r   r   LORALOHALOKRADALORAOFTSHIRAr2   rA   rR   rE   rC   rG   rI   r^   r@   rB   rD   r=   rF   rH   r.   r   	LoraLayer	LoHaLayer	LoKrLayerAdaLoraLayerOFTLayer
ShiraLayerrn   r   rQ   r#   r"   <module>r      sB   #  ' '   = = Z Z  #--xGWGWYaYeYegoguguv NNNNNNLL	 OOT__doow7L7Lcmm]b]n]nn 	JJJJJJMMLL	
r4 r4r#   