
    bifJ                         d dl Z d dlZd dlmZmZmZ d dlZd dlmZ d dl	mc m
Z d dlmZmZ  G d de      Z G d dej                   e      Z G d d	ej                   e      Zy)
    N)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_mergec            
           e Zd ZdZdZdej                  ddfdZdede	d	e
d
e
ddf
dZdefdZdefdZdeddfdZdddZy)HRALayer)hra_u)hra_rhra_apply_GS
base_layerreturnNc                    || _         i | _        i | _        t        j                  i       | _        d| _        g | _        d| _        || _	        | j                         }t        |t        j                        r$|j                  |j                  c| _        | _        y t        |t        j                        r$|j                   |j"                  c| _        | _        y t%        dt'        |             )NFTzUnsupported layer type )r   r   r   nnParameterDictr
   _disable_adaptersmerged_adapterscast_input_dtype_enabledkwargsget_base_layer
isinstanceLinearin_featuresout_featuresConv2din_channelsout_channels
ValueErrortype)selfr   r   s      P/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/hra/layer.py__init__zHRALayer.__init__    s    $
%%b)
!&!(,%((*
j")),2<2H2H*JaJa/Dd/
BII.2<2H2H*JaJa/Dd/6tJ7G6HIJJ    adapter_namerapply_GSinit_weightsc                    |dk  rt        d|       || j                  |<   || j                  |<   | j                         }t	        |t
        j                        rCt        j                  t        j                  | j                  |      d      | j                  |<   nt	        |t
        j                        rct        j                  t        j                  | j                  |j                  d   z  |j                  d   z  |      d      | j                  |<   n!t        dt        |      j                          |r| j#                  |       n| j%                  |       | j'                  |       | j)                  | j*                         y)aJ  Internal function to create hra adapter

        Args:
            adapter_name (`str`): Name for the adapter to add.
            r (`int`): Rank for the added adapter.
            init_weights (`bool`): Whether to initialize weights.
            apply_GS (`bool`): Whether to apply Gram-Schmidt orthogonalization or not.
        r   z?`r` should be a positive integer value but the value passed is T)requires_gradz/HRA is not implemented for base layers of type N)r   r   r   r   r   r   r   	Parametertorchemptyr   r
   r   kernel_size	TypeErrorr   __name__reset_hra_parametersreset_hra_parameters_random%_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r    r$   r%   r&   r'   r   r   s          r!   update_layerzHRALayer.update_layer4   sM     6^_`^abcc#$

< *2,' ((*
j")),')||EKK@P@PRS4Tdh'iDJJ|$
BII.')||D,,z/E/Ea/HH:KaKabcKddfgh"(DJJ|$
 MdS]N^NgNgMhijj %%l3,,\: 	22<@--.r#   c                 $   | j                   |   dz  dk7  rWt        j                  d       t        j                  j                  | j                  |   t        j                  d             y | j                  |   j                  }t        j                  |d   |d   dz        }t        j                  j                  |t        j                  d             t        j                  t        j                  |dd            | j                  |<   y )N   r   z@The symmetric initialization can NOT be performed when r is odd!   a   dim)r   warningswarnr   initkaiming_uniform_r
   mathsqrtshaper+   zerosr*   repeat_interleave)r    r$   rD   half_us       r!   r0   zHRALayer.reset_hra_parameters`   s    ::l#a'1,MM\]GG$$TZZ%=1$NJJ|,22E[[q58q=9FGG$$Vtyy|$<')||E4K4KFTU[\4]'^DJJ|$r#   c                     t         j                  j                  | j                  |   t	        j
                  d             y )Nr8   r9   )r   r@   rA   r
   rB   rC   )r    r$   s     r!   r1   z$HRALayer.reset_hra_parameters_randomj   s*    
  L!9TYYq\ Jr#   scalec                     |dk(  ry | j                   D ]4  }|| j                  j                         vr t        j                  d       6 y )Nr;   zFScaling operation for HRA not supported! Automatically set scale to 1.r4   r
   keysr>   r?   r    rI   active_adapters      r!   scale_layerzHRALayer.scale_layerm   sF    A:"22 	dNTZZ__%66MMbc		dr#   c                     | j                   D ]4  }|| j                  j                         vr t        j                  d       6 y )Nz>Unscaling operation for HRA not supported! Keeping scale at 1.rK   rM   s      r!   unscale_layerzHRALayer.unscale_layerw   s<    "22 	\NTZZ__%66MMZ[		\r#   Nr   N)r/   
__module____qualname__adapter_layer_namesother_param_namesr   Moduler"   strintboolr5   r0   r1   floatrO   rQ    r#   r!   r	   r	      s    $1K299 K4 K(*/*/ */ 	*/
 */ 
*/X_ _K Kd d4 d\r#   r	   c                        e Zd ZdZ	 	 	 ddedededeeef   ddf
 fdZdd	ed
e	e
e      ddfdZddZddededej                  fdZdej                  dededej                  fdZdef fdZ xZS )	HRALinearz+
    HRA implemented in a dense layer.
    r$   r%   r&   r'   r   Nc                     t         |           t        j                  | |fi | || _         | j                  ||||fi | y rR   superr"   r	   _active_adapterr5   r    r   r$   r%   r&   r'   r   	__class__s          r!   r"   zHRALinear.__init__   sH     	$
5f5+,8\LVLr#   
safe_mergeadapter_namesc                 J   t        | |      }|sy|D ]  }|| j                  j                         v s!| j                         }|j                  j
                  }|r|j                  j                  j                         }| j                  |      }t        j                  |j                  |j
                        |      }t        j                  |      j                         st        d| d      |j                  |      |j                  _        nt| j                  |      }t        j                  |j                  j                  j                  |j
                        |      }|j                  |      |j                  _        | j                  j!                  |        y)b  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If `None`, all active adapters will be merged.
                Defaults to `None`.
        N1NaNs detected in the merged weights. The adapter  seems to be broken)r   r
   rL   r   weightdtypedatacloneget_delta_weightr+   mmtoisfiniteallr   r   append)	r    rf   rg   rN   r   
orig_dtypeorig_weightdelta_weight
new_weights	            r!   mergezHRALinear.merge   sX    0mD+ 	<N!22!002
'..44
 #-"3"3"8"8">">"@K#'#8#8#HL"'((;>>,:L:L+M|"\K >>+6::<(OP^O__rs  .9^^J-GJ%%*#'#8#8#HL!&**;*;*@*@*C*CLDVDV*WYe!fJ-7]]:-FJ%%*$$++N;+	<r#   c                 p   | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }| j                         }|j                  j                  }|| j                  j                         v r|j                  j                  j                         }| j                  |d      }t        j                  |j!                  |j                        |      }|j!                  |      |j                  _        t        | j                        dkD  ryyzW
        This method unmerges all merged adapter layers from the base weights.
        z Already unmerged. Nothing to do.Nr   T)reverse)mergedr>   r?   lenr   popr   rl   rm   r
   rL   rn   ro   rp   r+   rq   rr   )r    rN   r   rv   rw   rx   ry   s          r!   unmergezHRALinear.unmerge   s     {{MM<=$&&'!+!11557N,,.J#**00J!22(//44::<#44^T4R"XXknn\5G5G&H,W
)3z)B
!!& $&&'!+r#   r}   c                    | j                   |   }| j                  |   }| j                  |   }|j                  }|r|d d df   |d d df   j	                         z  j                  dd      g}t        d|      D ]|  }|d d |f   j                  dd      }	t        |      D ]!  }
|	||
   j                         |	z  ||
   z  z
  }	# |j                  |	|	j	                         z  j                  dd             ~ t        j                  |d      }t        j                  |d   |j                  |j                        d|z  |j                         z  z
  }|S ||j	                  d      z  }t        j                  |d   |j                  |j                        }|rt        |dz
  dd      }nt        |      }|D ]7  }|d d |f   j                  dd      }	|d|z  |	z  |	j                         z  z
  }9 |S Nr   r;   r<   )devicerm   r7   r   r   r
   rD   normviewrangetru   r+   cateyer   rm   r    r$   r}   rankr&   opt_urD   rl   iuijindicess               r!   rp   zHRALinear.get_delta_weight      zz,'$$\2

<(QT{U1a4[%5%5%77==b!DEF1d^ <1a4[%%b!,q ?Avay{{}r1VAY>>B?rBGGI~33B:;	<
 YYv1-FYYuQxEKKPSTW]S]`f`h`h`jSjjF  EJJ1J--EYYuQxEKKPFq"b1+ ;1a4[%%b!,!f*r/BDDF"::; r#   xargsr   c                 B   |j                   }| j                  r4| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}nt        j                  | j                  |j                        }| j                  D ]_  }|| j                  j                         vr | j                  |      }t        j                  |j                  |j                         |      }a | j                         j                   j"                  }	| j%                  |	|j                         }	t        j                  |	|      }| j%                  | j                  j&                  |j                         }
| j(                  r| j%                  ||j                         }n=|j                  | j                         j                   j"                  j                         }t+        j,                  |||
      }|j                  |      }|S )Nr   )inputrl   bias)rm   disable_adaptersr~   r   r   r+   r   r   r   r4   r
   rL   rp   rq   rr   r   rl   rn   _cast_input_dtyper   r   Flinearr    r   r   r   previous_dtyperesultry   rN   rx   rw   r   s              r!   forwardzHRALinear.forward   s     {{$T__Q888F[[$T__Q888F4#3#3AHHEJ"&"6"6 W!)::#44^D"XXjmmL4F4F&GV
	W --/66;;K00j>N>NOK+z:J))$//*>*>
@P@PQD,,**1j.>.>?DD,,.55::@@AXXAjtDF>*r#   c                 *    t         |          }d|z   S Nzhra.rb   __repr__r    repre   s     r!   r   zHRALinear.__repr__      g |r#   r   FTFNrS   Fr/   rT   rU   __doc__rY   rZ   r[   r   r"   r   listrz   r   r+   Tensorrp   r   r   r   __classcell__re   s   @r!   r_   r_      s     )-M M 	M
 M D#I&M 
M'< '<Xd3i=P '<\` '<RC$S 4 ELL < c S U\\ @#  r#   r_   c                        e Zd ZdZ	 	 	 ddedededeeef   f fdZdded	e	e
e      d
dfdZddZddeded
ej                  fdZdej                  deded
ej                  fdZd
ef fdZ xZS )	HRAConv2dzHRA implemented in Conv2d layerr$   r%   r&   r'   c                     t         |           t        j                  | |       || _         | j                  ||||fi | y rR   ra   rd   s          r!   r"   zHRAConv2d.__init__  sC     	$
++,8\LVLr#   Nrf   rg   r   c                    t        | |      }|sy|D ]  }|| j                  j                         v s!| j                         }|j                  j
                  }|rJ|j                  j                  j                         }|j                  | j                  | j                  |j                  d   z  | j                  j                  d   z        }| j                  |      }t        j                  |j!                  |j
                        |      }|j                  | j                  | j                  |j                  d   |j                  d         }t        j"                  |      j%                         st'        d| d      |j!                  |      |j                  _        n|j                  j                  }|j                  | j                  | j                  | j                  j                  d   z  | j                  j                  d   z        }| j                  |      }t        j                  |j!                  |j
                        |      }|j                  | j                  | j                  |j                  d   |j                  d         }|j!                  |      |j                  _        | j(                  j+                  |        y)ri   Nr   rj   rk   )r   r
   rL   r   rl   rm   rn   ro   r   r   r   r-   r   rp   r+   rq   rr   rs   rt   r   r   ru   )r    rf   rg   rN   r   rv   rw   rx   s           r!   rz   zHRAConv2d.merge!  sq    0mD+ +	<N!22!002
'..44
 #-"3"3"8"8">">"@K"-"2"2))((:+A+A!+DDtGbGbcdGee#K $(#8#8#HL"'((;>>,:L:L+M|"\K"-"2"2))(("..q1"..q1	#K !>>+6::<(OP^O__rs  .9^^J-GJ%%*","3"3"8"8K"-"2"2))((4??+F+Fq+IIDOOLgLghiLjj#K $(#8#8#HL"'((;>>,:L:L+M|"\K"-"2"2))(("..q1"..q1	#K .9^^J-GJ%%*$$++N;W+	<r#   c                    | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }| j                         }|j                  j                  }|| j                  j                         v r|j                  j                  j                         }|j                  | j                  | j                  |j                   d   z  |j                   d   z        }| j#                  |d      }t%        j&                  |j)                  |j                        |      }|j                  | j                  | j                  |j                   d   |j                   d         }|j)                  |      |j                  _        t        | j                        dkD  ryyr|   )r~   r>   r?   r   r   r   r   rl   rm   r
   rL   rn   ro   r   r   r   r-   rp   r+   rq   rr   )r    rN   r   rv   rw   rx   s         r!   r   zHRAConv2d.unmerge`  ss    {{MM<=$&&'!+!11557N,,.J#**00J!22(//44::<)..%%$$z'='=a'@@:CYCYZ[C\\  $44^T4R#hh{~~l6H6H'I<X)..%%t'7'79O9OPQ9RT^TjTjklTm *5
)C
!!&! $&&'!+r#   r}   c                    | j                   |   }| j                  |   }| j                  |   }|j                  }|r|d d df   |d d df   j	                         z  j                  dd      g}t        d|      D ]|  }|d d |f   j                  dd      }	t        |      D ]!  }
|	||
   j                         |	z  ||
   z  z
  }	# |j                  |	|	j	                         z  j                  dd             ~ t        j                  |d      }t        j                  |d   |j                  |j                        d|z  |j                         z  z
  }|S ||j	                  d      z  }t        j                  |d   |j                  |j                        }|rt        |dz
  dd      }nt        |      }|D ]7  }|d d |f   j                  dd      }	|d|z  |	z  |	j                         z  z
  }9 |S r   r   r   s               r!   rp   zHRAConv2d.get_delta_weighty  r   r#   r   r   r   c                 b   |j                   }| j                  r4| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}nt        j                  | j                  | j                  j                  d   z  | j                  j                  d   z  |j                        }| j                  D ]_  }|| j                  j                         vr | j                  |      }t        j                  |j                  |j                         |      }a | j                  j                   j"                  }	|	j%                  | j&                  | j                  | j                  j                  d   z  | j                  j                  d   z        }	| j)                  |	|j                         }	| j)                  | j                  j*                  |j                         }
t        j                  |	|      }|j%                  | j&                  | j                  | j                  j                  d   | j                  j                  d         }| j,                  r| j)                  ||j                         }n=|j                  | j/                         j                   j"                  j                         }t1        j2                  |||
| j                  j4                  d   | j                  j6                  d         }|j                  |      }|S )Nr   r   )r   rl   r   paddingstride)rm   r   r~   r   r   r+   r   r   r-   r   r4   r
   rL   rp   rq   rr   rl   rn   r   r   r   r   r   r   r   conv2dr   r   r   s              r!   r   zHRAConv2d.forward  s     {{$T__Q888F[[$T__Q888F  4??#>#>q#AADOOD_D_`aDbbxxJ #'"6"6 W!)::#44^D"XXjmmL4F4F&GV
	W //0055K%**!!  4??#>#>q#AADOOD_D_`aDbbK 00j>N>NOK))$//*>*>
@P@PQD+z:J#!!  ++A.++A.	J ,,**1j.>.>?DD,,.55::@@AXX!//2--a0F >*r#   c                 *    t         |          }d|z   S r   r   r   s     r!   r   zHRAConv2d.__repr__  r   r#   r   r   rS   r   r   r   s   @r!   r   r     s    ) )-M M 	M
 M D#I&M=< =<Xd3i=P =<\` =<~D2S 4 ELL <1 1c 1S 1U\\ 1f#  r#   r   )rB   r>   typingr   r   r   r+   torch.nnr   torch.nn.functional
functionalr   peft.tuners.tuners_utilsr   r   r	   rX   r_   r   r]   r#   r!   <module>r      s\      ' '     Lb\~ b\JN		8 Nb|		8 |r#   