
    biE<                     *   d dl Z d dlmZ d dlZd dlmZ d dlmc mZ d dl	m
Z
  G d dej                  e
      Z G d de      Z G d d	e      Z G d
 dej                  j                         Z G d dej                  j                         Zd Zd Zy)    N)Any)LycorisLayerc                       e Zd ZdZdej
                  f fdZedee	   fd       Z
de	dedeed	f   fd
Zde	fdZde	fdZ	 dde	dededededededdfdZde	dej(                  fdZdej(                  dej(                  fdZ xZS )	LoHaLayer	hada_w1_a	hada_w1_b	hada_w2_a	hada_w2_bhada_t1hada_t2
base_layerc                    t         |           t        j                  | |       t        j                  i       | _        t        j                  i       | _        t        j                  i       | _        t        j                  i       | _        t        j                  i       | _	        t        j                  i       | _
        y N)super__init__r   nnParameterDictr   r	   r
   r   r   r   )selfr   	__class__s     Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/loha/layer.pyr   zLoHaLayer.__init__   s    dJ/ ))"-))"-))"-))"-''+''+    returnc                     h | j                   | j                  | j                  | j                  | j                  | j
                  S r   r   )r   s    r   _available_adapterszLoHaLayer._available_adapters*   sD    qq$..q4>>qDNNqUYUaUaqdhdpdpqqr   adapter_namershape.c           	         t        |      dk(  rat        j                  t        j                  |||d   |d               | j
                  |<   t        j                  t        j                  ||d               | j                  |<   t        j                  t        j                  ||d               | j                  |<   t        j                  t        j                  |||d   |d               | j                  |<   t        j                  t        j                  ||d               | j                  |<   t        j                  t        j                  ||d               | j                  |<   y t        j                  t        j                  |d   |            | j                  |<   t        j                  t        j                  ||d               | j                  |<   t        j                  t        j                  |d   |            | j                  |<   t        j                  t        j                  ||d               | j                  |<   y )N         r      )lenr   	Parametertorchemptyr   r   r	   r   r
   r   )r   r   r   r   s       r   create_adapter_parametersz#LoHaLayer.create_adapter_parameters.   s   u:?)+ekk!QaRWXYRZ6[)\DLL&+-<<AuQx8P+QDNN<(+-<<AuQx8P+QDNN<()+ekk!QaRWXYRZ6[)\DLL&+-<<AuQx8P+QDNN<(+-<<AuQx8P+QDNN<(+-<<E!Ha8P+QDNN<(+-<<AuQx8P+QDNN<(+-<<E!Ha8P+QDNN<(+-<<AuQx8P+QDNN<(r   c                 X   || j                   j                         v rt        j                  j	                  | j                   |   t        j                  d             t        j                  j	                  | j                  |   t        j                  d             t        j                  j	                  | j                  |   t        j                  d             t        j                  j                  | j                  |          || j                  j                         v rt        j                  j	                  | j                  |   t        j                  d             t        j                  j	                  | j                  |   t        j                  d             y y N   )a)r   keysr   initkaiming_uniform_mathsqrtr	   r
   zeros_r   r   r   r   r   s     r   reset_adapter_parametersz"LoHaLayer.reset_adapter_parameters?   s    4>>..00GG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGGNN4>>,784<<,,..GG$$T\\,%?499Q<$PGG$$T\\,%?499Q<$P /r   c                    || j                   j                         v rt        j                  j	                  | j                   |   t        j                  d             t        j                  j	                  | j                  |   t        j                  d             t        j                  j	                  | j                  |   t        j                  d             t        j                  j	                  | j                  |   t        j                  d             || j                  j                         v rt        j                  j	                  | j                  |   t        j                  d             t        j                  j	                  | j                  |   t        j                  d             y y r*   )r   r-   r   r.   r/   r0   r1   r	   r
   r   r   r   r3   s     r   reset_adapter_parameters_randomz)LoHaLayer.reset_adapter_parameters_randomN   s#    4>>..00GG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R4<<,,..GG$$T\\,%?499Q<$PGG$$T\\,%?499Q<$P /r   alpharank_dropoutmodule_dropoutinit_weightsuse_effective_conv2dNc                    |dk  rt        d|       || j                  |<   || j                  |<   ||z  | j                  |<   || j                  |<   || j
                  |<   | j                         }	t        |	t        j                        r t        |	j                  j                        }
nt        |	t        j                        rt|xr |	j                  dk7  }|r&|	j                  |	j                   g|	j                  }
nZ|	j                  |	j                   |	j                  d   z  |	j                  d   z  f}
n!t#        dt%        |	      j&                         | j)                  |||
       |r| j+                  |       n| j-                  |       | j/                  |       | j1                  | j2                         y)a  Internal function to create loha adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            alpha (`float`): Alpha for the added adapter.
            rank_dropout (`float`): The dropout probability for rank dimension during training.
            module_dropout (`float`): The dropout probability for disabling adapter during training.
            init_weights (`bool`): Whether to initialize weights.
            use_effective_conv2d (`bool`, *optional*, defaults to `False`):
                Use parameter effective decomposition for Conv2d with ksize > 1.
        r   z?`r` should be a positive integer value but the value passed is )r#   r#   r#   z0LoHa is not implemented for base layers of type N)
ValueErrorr   r7   scalingr8   r9   get_base_layer
isinstancer   Lineartupleweightr   Conv2dkernel_sizeout_channelsin_channels	TypeErrortype__name__r(   r4   r6   %_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r   r   r   r7   r8   r9   r:   r;   kwargsr   r   s              r   update_layerzLoHaLayer.update_layer]   s   . 6^_`^abcc |#(

< %*QY\"*6,',:L) ((*
j")),*++112E
BII.#7#\J<R<RV\<\ ##00*2H2Hb:KaKab ++**Z-C-CA-FFI_I_`aIbb
 NtT^O_OhOhNijkk 	&&|Q> )),700> 	22<@--.r   c                    || j                   j                         v rt        | j                   |   | j                  |   | j                  |   | j
                  |   | j                  |   | j                  |   t        j                  | j                  |               }ndt        | j                  |   | j                  |   | j                  |   | j                  |   t        j                  | j                  |               }| j                         }|j                  |j                  j                        }| j                   |   }| j"                  r|rt        j$                  |j'                  d            |kD  j)                  |j*                        } |j,                  dgdgt/        |j                  dd        z   j)                  |j0                        }||j3                         z  }||z  }|S )N)scaler   r#   )r   r-   make_weight_cpr   r	   r   r
   r   r&   tensorr>   make_weightr?   reshaperC   r   r8   trainingrandsizetodtypeviewr$   devicemean)r   r   rC   r   r8   drops         r   get_delta_weightzLoHaLayer.get_delta_weight   s   4<<,,..#\*|,|,\*|,|,ll4<<#=>F !|,|,|,|,ll4<<#=>F ((*

 1 1 7 78 ((6==\JJv{{1~.=AA&,,OD499R>1#FLL,<(="=>AA&--PD DIIKDdNFr   xc                 &   |j                   }| j                  r3| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}n | j                  |g|i |}| j
                  D ]l  }|| j                  vr| j                  |   }| j                  r&| j                  s:t        j                  d      |kD  sS| | j                  ||g|i |z   }n |j                  |      }|S )Nr#   )r[   disable_adaptersmergedunmerger   rM   r   r9   rW   r&   rX   _get_delta_activationsrZ   )r   ra   argsrN   previous_dtyperesultactive_adapterr9   s           r   forwardzLoHaLayer.forward   s     {{$T__Q888F[[$T__Q888F$T__Q888F #'"6"6 f!)A)AA!%!4!4^!D 4==UZZ]^=[#&Ad&A&A.RS&eVZ&e^d&eeFf >*r   )F)rJ   
__module____qualname__adapter_layer_namesr   Moduler   propertysetstrr   intrB   r(   r4   r6   floatboolrO   r&   Tensorr`   rk   __classcell__r   s   @r   r   r      s   d
,299 
, rSX r rRc Rc R%PSUXPX/ R"QS QQC Q. &+;/;/ ;/ 	;/
 ;/ ;/ ;/ #;/ 
;/z#S #U\\ #J 5<< r   r   c                        e Zd ZdZ	 	 	 	 	 	 ddej
                  dededededede	f fd	Z
ded
ej                  dededej                  f
dZdef fdZ xZS )rA   z LoHa implemented in Linear layerr   r   r   r7   r8   r9   r:   c                 b    t         	|   |       || _         | j                  ||||||fi | y r   r   r   _active_adapterrO   )
r   r   r   r   r7   r8   r9   r:   rN   r   s
            r   r   zLinear.__init__   s=     	$  ,,5,P\g`fgr   inputrg   rN   r   c                     | j                  |      }| j                  ||j                        }t        j                  ||      S r   )r`   _cast_input_dtyper[   Flinear)r   r   r}   rg   rN   delta_weights         r   rf   zLinear._get_delta_activations   s=     ,,\:&&ul.@.@Axx|,,r   c                 *    t         |          }d|z   S Nzloha.r   __repr__r   repr   s     r   r   zLinear.__repr__       g }r   )defaultr           r   r   TrJ   rl   rm   __doc__r   ro   rr   rs   rt   ru   r   r&   rv   r   rf   r   rw   rx   s   @r   rA   rA      s    *
 &! #!hIIh h 	h
 h h h h"--(--=@-LO-	-#  r   rA   c                        e Zd ZdZ	 	 	 	 	 	 	 ddej
                  dededededede	d	e	f fd
Z
dedej                  dededej                  f
dZdef fdZ xZS )rD   z LoHa implemented in Conv2d layerr   r   r   r7   r8   r9   r;   r:   c	           	      d    t         
|   |       || _         | j                  |||||||fi |	 y r   r{   )r   r   r   r   r7   r8   r9   r;   r:   rN   r   s             r   r   zConv2d.__init__   sD     	$  ,!UL.,Pd	
hn	
r   r}   rg   rN   r   c                    | j                  |      }| j                  ||j                        }| j                         }t	        j
                  |||j                  |j                  |j                  |j                        S )N)stridepaddingdilationgroups)
r`   r   r[   r?   r   conv2dr   r   r   r   )r   r   r}   rg   rN   r   r   s          r   rf   zConv2d._get_delta_activations  su     ,,\:&&ul.@.@A((*
xx$$&&(($$
 	
r   c                 *    t         |          }d|z   S r   r   r   s     r   r   zConv2d.__repr__"  r   r   )r   r   r   r   r   FTr   rx   s   @r   rD   rD      s    *
 &! #%*!
II
 
 	

 
 
 
 #
 
(

(-
=@
LO
	
 #  r   rD   c                   P    e Zd Ze ej
                  d      fd       Zed        Zy)
HadaWeightr#   c                 L    | j                  |||||       ||z  ||z  z  |z  }|S r   )save_for_backward)ctxw1aw1bw2aw2brQ   diff_weights          r   rk   zHadaWeight.forward+  s5    c3S%8c	cCi0E9r   c                     | j                   \  }}}}}||z  }|||z  z  }||j                  z  }|j                  |z  }	|||z  z  }||j                  z  }
|j                  |z  }~||	|
|d fS r   )saved_tensorsT)r   grad_outr   r   r   r   rQ   tempgrad_w1agrad_w1bgrad_w2agrad_w2bs               r   backwardzHadaWeight.backward1  s    &)&7&7#c3Ue#39%#%%<554<39%#%%<554<8Xt;;r   NrJ   rl   rm   staticmethodr&   rT   rk   r    r   r   r   r   *  s5    /;u||A  
 < <r   r   c                   P    e Zd Ze ej
                  d      fd       Zed        Zy)HadaWeightCPr#   c           	          | j                  |||||||       t        j                  d|||      }t        j                  d|||      }	||	z  |z  S )Nzi j k l, j r, i p -> p r k l)r   r&   einsum)
r   t1r   r   t2r   r   rQ   rebuild1rebuild2s
             r   rk   zHadaWeightCP.forwardB  sV    b#sBS%@<< >CM<< >CM("U**r   c                    | j                   \  }}}}}}}||z  }t        j                  d||      }	t        j                  d|	|      }
|
|z  }~
t        j                  d|	|      }t        j                  d||j                        }~~	t        j                  d||      }t        j                  d||j                        }~t        j                  d||      }	t        j                  d|	|      }
|
|z  }~
t        j                  d|	|      }t        j                  d||j                        }~~	t        j                  d||      }t        j                  d||j                        }~||||||d fS )Nzi j k l, j r -> i r k lzi j k l, i r -> r j k lzr j k l, i j k l -> r izi r k l, i j k l -> r j)r   r&   r   r   )r   r   r   r   r   r   r   r   rQ   r   rebuildgrad_wr   	grad_tempr   grad_t1r   r   grad_t2s                      r   r   zHadaWeightCP.backwardK  sY   .1.?.?+S#r3Ue#||5r3?,,8$D8#<< 94HLL!:FCEEJ	D<< 92yI,,8)SUUK||5r3?,,8$D8#<< 94HLL!:FCEEJ	D<< 92yI,,8)SUUK(GXxMMr   Nr   r   r   r   r   r   A  s7    7Cu||A + + N Nr   r   c                 4    t         j                  | ||||      S r   )r   apply)r   r   r   r   rQ   s        r   rU   rU   n  s    Cc366r   c           	      8    t         j                  | ||||||      S r   )r   r   )r   r   r   r   r   r   rQ   s          r   rS   rS   r  s    b#sBS%@@r   )r0   typingr   r&   torch.nnr   torch.nn.functional
functionalr   peft.tuners.lycoris_utilsr   ro   r   rA   rD   autogradFunctionr   r   rU   rS   r   r   r   <module>r      s          2~		< ~BY B)Y )^<(( <.*N5>>** *NZ7Ar   