
    bi*@              	           d dl Z d dlmZmZmZ d dlZd dlmZ d dlmc m	Z
 d dlmZ  G d dej                  e      Z G d de      Z G d d	e      Zdd
ededeeef   fdZd ZddZy)    N)AnyOptionalUnion)LycorisLayerc                   "    e Zd ZdZdej
                  ddf fdZedee	   fd       Z
de	ded	ed
edef
dZde	fdZde	fdZd Zde	dededededededededdfdZde	dej(                  fdZdej(                  dej(                  fdZ xZS )	LoKrLayerlokr_w1	lokr_w1_a	lokr_w1_blokr_w2	lokr_w2_a	lokr_w2_blokr_t2
base_layerreturnNc                    t         |           t        j                  | |       t        j                  i       | _        t        j                  i       | _        t        j                  i       | _        t        j                  i       | _        t        j                  i       | _	        t        j                  i       | _
        t        j                  i       | _        y N)super__init__r   nnParameterDictr
   r   r   r   r   r   r   )selfr   	__class__s     Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/lokr/layer.pyr   zLoKrLayer.__init__&   s    dJ/ ''+))"-))"-''+))"-))"-''+    c                     h | j                   | j                  | j                  | j                  | j                  | j
                  | j                  S r   r	   )r   s    r   _available_adapterszLoKrLayer._available_adapters3   sj    
\\
^^
 ^^
 \\	

 ^^
 ^^
 \\
 	
r   adapter_nameruse_w1use_w2use_effective_conv2dc           	      $   |rCt        j                  t        j                  |d   d   |d   d               | j                  |<   nxt        j                  t        j                  |d   d   |            | j
                  |<   t        j                  t        j                  ||d   d               | j                  |<   t        |      dk(  r|rGt        j                  t        j                  |d   d   |d   d   g|dd         | j                  |<   y |rt        j                  t        j                  |||d   |d               | j                  |<   t        j                  t        j                  ||d   d               | j                  |<   t        j                  t        j                  ||d   d               | j                  |<   y t        j                  t        j                  |d   d   |            | j                  |<   t        j                  t        j                  ||d   d   |d   z  |d   z              | j                  |<   y |rCt        j                  t        j                  |d   d   |d   d               | j                  |<   y t        j                  t        j                  |d   d   |            | j                  |<   t        j                  t        j                  ||d   d               | j                  |<   y )Nr               )r   	Parametertorchemptyr
   r   r   lenr   r   r   r   )r   r   r    shaper!   r"   r#   s          r   create_adapter_parametersz#LoKrLayer.create_adapter_parameters?   sv    )+ekk%(1+uUVxXY{6[)\DLL&+-<<E!HQKQR8S+TDNN<(+-<<AuQxPQ{8S+TDNN<(u:?-/\\%++eAhqkSXYZS[\]S^:kafghgiaj:k-l\*%-/\\%++aERSHV[\]V^:_-`\*/1||EKK5QR8TU;<W/X|,/1||EKK5QR8TU;<W/X|,/1||EKKaQRUV<W/X|,/1||EKK5QR8TU;Y^_`YaKadijkdlKl<m/n|, -/\\%++eAhqkSXYZS[\]S^:_-`\*/1||EKKaQRUV<W/X|,/1||EKK5QR8TU;<W/X|,r   c                    || j                   v r-t        j                  j                  | j                   |          nmt        j                  j                  | j                  |          t        j                  j                  | j                  |   t        j                  d             || j                  v rBt        j                  j                  | j                  |   t        j                  d             nt        j                  j                  | j                  |   t        j                  d             t        j                  j                  | j                  |   t        j                  d             || j                  v rBt        j                  j                  | j                  |   t        j                  d             y y N   )a)r
   r   initzeros_r   kaiming_uniform_r   mathsqrtr   r   r   r   r   r   s     r   reset_adapter_parametersz"LoKrLayer.reset_adapter_parametersa   s%   4<<'GGNN4<<56GGNN4>>,78GG$$T^^L%ATYYq\$R4<<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R4<<'GG$$T\\,%?499Q<$P (r   c                    || j                   v rBt        j                  j                  | j                   |   t	        j
                  d             nt        j                  j                  | j                  |   t	        j
                  d             t        j                  j                  | j                  |   t	        j
                  d             || j                  v rBt        j                  j                  | j                  |   t	        j
                  d             nt        j                  j                  | j                  |   t	        j
                  d             t        j                  j                  | j                  |   t	        j
                  d             || j                  v rBt        j                  j                  | j                  |   t	        j
                  d             y y r0   )r
   r   r3   r5   r6   r7   r   r   r   r   r   r   r8   s     r   reset_adapter_parameters_randomz)LoKrLayer.reset_adapter_parameters_randomq   sC   4<<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R4<<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R4<<'GG$$T\\,%?499Q<$P (r   c                    || j                   v rBt        j                  j                  | j                   |   t	        j
                  d             nt        j                  j                  | j                  |   t	        j
                  d             t        j                  j                  | j                  |   t	        j
                  d             || j                  v r-t        j                  j                  | j                  |          nmt        j                  j                  | j                  |          t        j                  j                  | j                  |   t	        j
                  d             || j                  v rBt        j                  j                  | j                  |   t	        j
                  d             y y r0   )r
   r   r3   r5   r6   r7   r   r   r   r4   r   r   r   r8   s     r   $reset_adapter_parameters_lycoris_wayz.LoKrLayer.reset_adapter_parameters_lycoris_way   s%   4<<'GG$$T\\,%?499Q<$PGG$$T^^L%ATYYq\$RGG$$T^^L%ATYYq\$R4<<'GGNN4<<56GGNN4>>,78GG$$T^^L%ATYYq\$R4<<'GG$$T\\,%?499Q<$P (r   alpharank_dropoutmodule_dropoutinit_weightsdecompose_bothdecompose_factorc
                    |dk  rt        d|       || j                  |<   || j                  |<   ||z  | j                  |<   || j                  |<   || j
                  |<   |
d   | j                  |<   | j                         }t        |t        j                        r|j                  |j                  }}t        ||	      \  }}t        ||	      \  }}||f||ff}|xr |t        |d   d   |d   d         dz  k   }|t        |d   d   |d   d         dz  k   }d}nt        |t        j                        r|j                   |j"                  }}|j$                  }t        ||	      \  }}t        ||	      \  }}||f||fg|}|xr |t        |d   d   |d   d         dz  k   }|t        |d   d   |d   d         dz  k\  }|xr |j$                  dk7  }n!t'        dt)        |      j*                         | j-                  ||||||       |r)|d	k(  r| j/                  |       n#| j1                  |       n| j3                  |       | j5                  |       | j7                  | j8                         y
)a  Internal function to create lokr adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            alpha (`float`): Alpha for the added adapter.
            rank_dropout (`float`): The dropout probability for rank dimension during training
            module_dropout (`float`): The dropout probability for disabling adapter during training.
            init_weights (`bool`): Whether to initialize adapter weights.
            use_effective_conv2d (`bool`): Use parameter effective decomposition for Conv2d with ksize > 1.
            decompose_both (`bool`): Perform rank decomposition of left kronecker product matrix.
            decompose_factor (`int`): Kronecker product decomposition factor.
        r   z?`r` should be a positive integer value but the value passed is rank_dropout_scaler%   r'   F)r%   r%   z0LoKr is not implemented for base layers of type lycorisN)
ValueErrorr    r>   scalingr?   r@   rE   get_base_layer
isinstancer   Linearin_featuresout_featuresfactorizationmaxConv2din_channelsout_channelskernel_size	TypeErrortype__name__r.   r=   r9   r;   %_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r   r   r    r>   r?   r@   rA   r#   rB   rC   kwargsr   in_dimout_dimin_min_nout_lout_kr-   r!   r"   k_sizes                         r   update_layerzLoKrLayer.update_layer   s   4 6^_`^abcc |#(

< %*QY\"*6,',:L)067K0L-((*
 j")),(44j6M6MGF&v/?@JD$(2BCLE5U^dD\2E(RQU1Xa[%(1+1NQR1R-RSFc%(1+uQx{;a??@F#( 
BII.(44j6M6MGF++F&v/?@JD$(2BCLE5U^dD\;F;E(RQU1Xa[%(1+1NQR1R-RSF#eAhqk58A;7!;;F#7#\J<R<RV\<\ NtT^O_OhOhNijkk 	&&|QvvOcd y(99,G--l;00> 	22<@--.r   c           	         || j                   v r| j                   |   }n| j                  |   | j                  |   z  }|| j                  v r| j                  |   }nb|| j                  v r5t        | j                  |   | j                  |   | j                  |         }n| j                  |   | j                  |   z  }t        ||| j                  |         }|j                  | j                         j                  j                        }| j                  |   }| j                  r|rt!        j"                  |j%                  d            |kD  j'                         } |j(                  dgdgt+        |j                  dd        z   j-                  |j.                        }| j0                  |   r||j3                         z  }||z  }|S )Nr   r%   )r
   r   r   r   r   make_weight_cpr   r   	make_kronrH   reshaperI   weightr-   r?   trainingr*   randsizefloatviewr,   todevicerE   mean)r   r   w1w2rh   r?   drops          r   get_delta_weightzLoKrLayer.get_delta_weight   s   4<<'l+B-|0LLB4<<'l+BT\\)\ :DNN<<XZ^ZhZhiuZvwB-|0LLB 2r4<<#=> 3 3 5 < < B BC ((6==\JJv{{1~.=DDFD499R>1#FLL,<(="=>AA&--PD&&|4		#dNFr   xc                 &   |j                   }| j                  r3| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}n | j                  |g|i |}| j
                  D ]l  }|| j                  vr| j                  |   }| j                  r&| j                  s:t        j                  d      |kD  sS| | j                  ||g|i |z   }n |j                  |      }|S )Nr%   )dtypedisable_adaptersmergedunmerger   rY   r   r@   ri   r*   rj   _get_delta_activationsrn   )r   ru   argsrZ   previous_dtyperesultactive_adapterr@   s           r   forwardzLoKrLayer.forward   s     {{$T__Q888F[[$T__Q888F$T__Q888F #'"6"6 f!)A)AA!%!4!4^!D 4==UZZ]^=[#&Ad&A&A.RS&eVZ&e^d&eeFf >*r   )rV   
__module____qualname__adapter_layer_namesr   Moduler   propertysetstrr   intboolr.   r9   r;   r=   rl   rb   r*   Tensorrt   r   __classcell__r   s   @r   r   r      sD   ,299 , , 	
SX 	
 	
 Y Y  Y
  Y  Y # YDQS Q QC Q"Q L/L/ L/ 	L/
 L/ L/ L/ #L/ L/ L/ 
L/\S U\\ : 5<< r   r   c                        e Zd ZdZ	 	 	 	 	 	 	 	 ddej
                  deeee	j                  f      dee	j                     dedededed	ed
ef fdZdede	j                   dedede	j                   f
dZdef fdZ xZS )rK   z LoKr implemented in Linear layerr   ro   rw   r   r    r>   r?   r@   rA   c
                 b    t         |   |       || _         | j                  ||||||	fi |
 y r   r   r   _active_adapterrb   )r   r   ro   rw   r   r    r>   r?   r@   rA   rZ   r   s              r   r   zLinear.__init__  s=     	$  ,,5,P\g`fgr   inputr|   rZ   r   c                     | j                  |      }| j                  ||j                        }t        j                  ||      S r   )rt   _cast_input_dtyperw   Flinear)r   r   r   r|   rZ   delta_weights         r   r{   zLinear._get_delta_activations.  s=     ,,\:&&ul.@.@Axx|,,r   c                 *    t         |          }d|z   S Nzlokr.r   __repr__r   repr   s     r   r   zLinear.__repr__6      g }r   )NNdefaultr           r   r   TrV   r   r   __doc__r   r   r   r   r   r*   ro   rw   r   rl   r   r   r   r   r{   r   r   r   s   @r   rK   rK     s    *
 6:'+%! #!hIIh sELL012h $	h
 h h h h h h&--(--=@-LO-	-#  r   rK   c                       e Zd ZdZ	 	 	 	 	 	 	 	 	 ddej
                  deeee	j                  f      dee	j                     dedededed	ed
edef fdZdede	j                   dedede	j                   f
dZdef fdZ xZS )rP   z LoKr implemented in Conv2d layerr   ro   rw   r   r    r>   r?   r@   r#   rA   c           	      d    t         |   |       || _         | j                  ||||||
|	fi | y r   r   )r   r   ro   rw   r   r    r>   r?   r@   r#   rA   rZ   r   s               r   r   zConv2d.__init__>  sD     	$  ,!UL.,Pd	
hn	
r   r   r|   rZ   r   c                    | j                  |      }| j                  ||j                        }| j                         }t	        j
                  |||j                  |j                  |j                  |j                        S )N)stridepaddingdilationgroups)
rt   r   rw   rI   r   conv2dr   r   r   r   )r   r   r   r|   rZ   r   r   s          r   r{   zConv2d._get_delta_activationsT  su     ,,\:&&ul.@.@A((*
xx$$&&(($$
 	
r   c                 *    t         |          }d|z   S r   r   r   s     r   r   zConv2d.__repr__d  r   r   )	NNr   r   r   r   r   FTr   r   s   @r   rP   rP   ;  s    *
 6:'+%! #%*!
II
 sELL012
 $	

 
 
 
 
 
 #
 
,

(-
=@
LO
	
 #  r   rP   	dimensionfactorr   c                     |dkD  r| |z  dk(  r|}| |z  }||fS |dk(  r| }d| }}||z   }||k  r8|dz   }| |z  dk7  r|dz  }| |z  dk7  r| |z  }||z   |kD  s||kD  rn
||}}||k  r8||kD  r||}}||fS )a  Factorizes the provided number into the product of two numbers

    Args:
        dimension (`int`): The number that needs to be factorized.
        factor (`int`, optional):
            Factorization divider. The algorithm will try to output two numbers, one of each will be as close to the
            factor as possible. If -1 is provided, the decomposition algorithm would try to search dividers near the
            square root of the dimension. Defaults to -1.

    Returns:
        Tuple[`int`, `int`]: A tuple of two numbers, whose product is equal to the provided number. The first number is
        always less than or equal to the second.

    Example:
        ```py
        >>> factorization(256, factor=-1)
        (16, 16)

        >>> factorization(128, factor=-1)
        (8, 16)

        >>> factorization(127, factor=-1)
        (1, 127)

        >>> factorization(128, factor=4)
        (4, 32)
        ```
    r   rd   r%    )r   r   mnlengthnew_mnew_ns          r   rN   rN   l  s    < zy6)a/!t|iqAUF
a%A%1$QJE %1$U"5=6!UV^%qA a% 	1u!1a4Kr   c                 6    t        j                  d| ||      }|S )Nzi j k l, i p, j r -> p r k l)r*   einsum)twawbrebuild2s       r   re   re     s    ||:Ar2FHOr   c                     t        |j                        dk(  r | j                  d      j                  d      } |j                         }t	        j
                  | |      }||z  S )Nr&   r'   )r,   r-   	unsqueeze
contiguousr*   kron)rq   rr   scalerebuilds       r   rf   rf     sO    
288}\\!_&&q)	BjjR GU?r   )rd   )g      ?)r6   typingr   r   r   r*   torch.nnr   torch.nn.functional
functionalr   peft.tuners.lycoris_utilsr   r   r   rK   rP   r   tuplerN   re   rf   r   r   r   <module>r      s{     ' '     2|		< |~ Y  F+Y +b1S 1# 1uS#X 1h
r   