
    bi                    H   d dl mZ d dlZd dlmZmZmZ d dlZd dlm	Z	 d dl
m	c mZ d dlmZmZ ddlmZ  G d de	j$                        Z G d	 d
e	j$                        Z G d de      Z G d de	j$                  e      Z G d de	j$                  e      Z	 	 	 	 	 	 	 	 ddZy)    )annotationsN)AnyOptionalUnion)BaseTunerLayercheck_adapters_to_merge   )	OFTConfigc                  *     e Zd ZdZd fd	Zd Z xZS )MultiplicativeDropoutLayerz>
    Implements the multiplicative dropout layer for OFT.
    c                0    t         |           || _        y)z
        Initializes the multiplicative dropout layer.

        Parameters:
        p (float): The probability of dropping out a block. Defaults to 0.0.
        N)super__init__p)selfr   	__class__s     P/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/oft/layer.pyr   z#MultiplicativeDropoutLayer.__init__!   s     	    c                x   | j                   r,| j                  dkD  r|j                  d   |j                  d   k7  rt        d      |j                  \  }}}|dk(  r|S t	        | j                  |z        }||z
  }t        j                  t        j                  ||j                        t        j                  ||j                        g      }|t        j                  |         j                  |dd      }t        j                  ||j                        j                  |dd      }d|z
  |z  ||z  z   }|S )aI  
        Applies multiplicative dropout to the input tensor.

        Parameters:
        x (Tensor): The input tensor of shape (D, H, H), where `D` represents
                    the number of OFT blocks, and `H` is the size of the square blocks along the last two dimensions,
                    the block size in OFT.
        r   z4The last two dimensions of input should be the same!r	   device)trainingr   shape
ValueErrorinttorchcatonesr   zerosrandpermvieweyerepeat)	r   xDH_num_to_replace	num_zerosmask
eye_matrixs	            r   forwardz"MultiplicativeDropoutLayer.forward+   s    ==TVVaZwwr{aggbk) !WXXggGAq! Av !_NN*I99ejjI5;;W`ijiqiqKrstDq)*//1a8D1QXX6==aAFJTQ
!22Ar   )        )__name__
__module____qualname____doc__r   r.   __classcell__r   s   @r   r   r      s    r   r   c                  ~     e Zd Z	 	 	 	 	 	 d fd	Zd Zd Z	 d	 	 	 	 	 	 	 	 	 ddZddZddZd Z	d Z
d	 Zd
 Z xZS )OFTRotationModulec                R   t         |           || _        || _        || _        || _        t        j                  t        j                  ||            | _
        || _        || _        || _        || _        |	| _        |
| _        t        j"                  ||d      \  | _        | _        y )Nr	   )r   r   r
n_elements
block_sizein_featuresnn	Parameterr   emptyweightcoftepsblock_sharekernel_sizeuse_cayley_neumannnum_cayley_neumann_termstriu_indicesrowscols)r   r9   r:   r;   r<   rA   rB   rC   rD   rE   rF   r   s              r   r   zOFTRotationModule.__init__I   s     	$$&ll5;;q*#=>	&&"4(@%$11*j!L	49r   c                    |j                   d   }t        j                  ||||j                  |j                        }||d d | j
                  | j                  f<   ||j                  dd      z
  }|S )Nr   r   dtyper   r   )r   r   r!   r   rL   rH   rI   	transpose)r   vecr;   
batch_sizematrixs        r   _pytorch_skew_symmetricz)OFTRotationModule._pytorch_skew_symmetricf   se    YYq\
ZZ

Z]ZcZcd*-q$))TYY&'&**2r22r   c                `    |j                   d   }|d d | j                  | j                  f   }|S )Nr   )r   rH   rI   )r   rP   r;   rO   rN   s        r   _pytorch_skew_symmetric_invz-OFTRotationModule._pytorch_skew_symmetric_invn   s/    \\!_
 Q		499,-
r   c                $   |j                   \  }}|j                  }| j                  ||      }|rt        j                  ||j
                  |j                        j                  |dd      }	|dkD  r|	j                  |d       |dkD  rt        j                  ||      }
|	j                  |
d       |
}t        d|      D ]+  }t        j                  ||      }|	j                  |d       - nt        j                  |j                   d   |j
                        j                  d	      j                  ||j                   d   |j                   d         }t        j                  j                  ||z   ||z
  d
      }	|	j                  |      S )z
        Perform the Cayley parametrization on a batch of skew-symmetric matrices.

        Args:
            data: A batch of skew-symmetric matrices of shape (b, r, c).
        rK   r	   g       @)alpha      r   r   r   F)left)r   rL   rQ   r   r$   r   r%   add_bmmrange	unsqueezeexpandlinalgsolveto)r   Qr;   rE   num_neumann_termsbr)   previous_dtypeQ_skewR	Q_squaredQ_poweriid_mats                 r   _cayley_batchzOFTRotationModule._cayley_batchu   sW    ww1 --a<		*QXXQWWELLQPQSTUA 1$vS)$q( %		&& 9IFF9CF0'G"1&78 3"'))GV"<wc23
 		&,,r*6==A16<<+V\\"-=> 
 ""6F?FVO%"PAttN##r   c                b   | j                  || j                        }|dz  t        j                  t        j                  |j
                  d               z  }t        j                  |j                  d      |j                  d      f|j                  |j                        j                  d      j                  |      }||z
  }t        j                  ||z
  dd      }||k  j                         }t        j                  ||||||z  z  z         }| j                  || j                        S )Nr	   r   rK   )r	   rV   T)dimkeepdim)rQ   r;   r   sqrttensorr   r!   sizer   rL   r\   	expand_asnormboolwhererS   )	r   ra   rB   oft_RIdiff	norm_diffr,   outs	            r   _project_batchz OFTRotationModule._project_batch   s    ,,Q@Ag

5<<A#?@@KKA

16u||SXS^S^_Yq\Yu 	

 qyJJuqyfdC	S &&(kk$q3$2B+C'CD//T__EEr   c                    |j                   d   dk(  rt        |      D cg c]  }|d   	 }}nt        |      D cg c]	  }||df    }}t        j                  | }|S c c}w c c}w )Nr   r	   )r   ..)r   r[   r   
block_diag)r   rv   rankri   blocksAs         r   _block_diagonalz!OFTRotationModule._block_diagonal   sk    ;;q>Q-24[9eFm9F9-24[9eAsFm9F9 f% :9s
   A A%c                   |j                   \  }}}}t        | j                  t              r| j                  | j                  }}n| j                  \  }}dx}}	dx}
}|d|
z  z   |z
  |z  dz   }|d|z  z   |z
  |	z  dz   }|j	                  d||      j	                  d||	      }|j                  dddddd      j                         }|j                  ||z  |z  d      }|S )z
        Unfold with stride=1, padding=0 to preserve spatial dimensions. Only use kernel_size from base layer to define
        patch size.
        r	   r   rV   rW         r   )r   
isinstancerD   r   unfoldpermute
contiguousr#   )r   r&   rO   in_channels	in_heightin_widthkernel_heightkernel_widthstride_hstride_wpad_hpad_w
out_height	out_width
x_unfoldeds                  r   _unfoldzOFTRotationModule._unfold   s   
 89ww4
KHd&&,*.*:*:D<L<L<M*.*:*:'M<8  !e)+m;H1L
E	)L8XEI	 XXa9@@LRZ[
''1aAq9DDF
__Z*%<y%H"M
r   c                   |\  }}}}t        | j                  t              r| j                  | j                  }}n| j                  \  }}||z
  dz   }	||z
  dz   }
|j                  ||	|
|||      }|j	                  dddddd      j                         }t        j                  |j                  |||z  |z  |	|
z        ||f||fd      }|S )	z;
        Fold back to preserve spatial dimensions.
        r	   r   rW   rV   r   r   )r	   r	   )output_sizerD   stride)r   rD   r   r#   r   r   Ffold)r   r   
orig_shaperO   r   r   r   r   r   r   r   
x_reshapedx_foldeds                r   _foldzOFTRotationModule._fold   s     8B4
KHd&&,*.*:*:D<L<L<M*.*:*:'M< .2
|+a/	  __ZYUbdpq
  ''1aAq9DDF
 66OOJm(Cl(RT^ajTjk"H-&5	
 r   c                D   |j                   }|| j                  j                   k7  r%|j                  | j                  j                         }|j                  }| j                  r]t        j                         5  | j                  j                  | j                  | j                  | j                               d d d        | j                  | j                  | j                  | j                  | j                        }t        |      dk(  r| j                  |      }|j                  }| j                   r| j"                  | j                  z  n| j$                  }|j                  d d } |j&                  g ||| j                   }| j                   r+|j)                  |dd      }t        j*                  d||      }	nt        j*                  d||      }	 |	j&                  | }
t        |      dk(  r| j-                  |
|      }
|
j                  |      S # 1 sw Y   _xY w)NrB   r   r   r	   z...rk,rkc->...rc)rL   r@   r`   r   rA   r   no_gradcopy_r{   rB   rk   r;   rE   rF   lenr   rC   r<   r9   reshaper%   einsumr   )r   r&   required_dtyper   orth_rotatefolded_shaper~   
batch_dimsr   x_rotated_reshaped	x_rotateds              r   r.   zOFTRotationModule.forward   s   
 T[[...T[[&&'AWW
99 R!!$"5"5dkktxx"5"PQR ((KK$*A*A4C`C`

 z?aQAww6:6F6Ft4??2DFFWWSb\
QYYB
BDB$//B
%,,T1a8K!&.@*k!Z!&.@*k!Z.&..=	z?a

9j9I||N++7R Rs   7AHHc                   | j                   }| j                  rUt        j                         5  | j	                  || j
                        }| j                   j                  |       ddd       | j                  || j                  | j                  | j                        }| j                  s| j                  n| j                  | j                  z  }| j                  ||      S # 1 sw Y   ~xY w)
        Compute the delta weight for the given adapter.

        Args:
            adapter (str):
                The name of the adapter for which the delta weight should be computed.
        r   N)r@   rA   r   r   r{   rB   r   rk   r;   rE   rF   rC   r9   r<   r   )r   r@   r   r~   s       r   
get_weightzOFTRotationModule.get_weight  s     99 *,,V,B!!&)* ((DOOT%<%<d>[>[
 "--tvv43C3Ct3V##K66* *s   9C##C,)FiUMu?F)r   r   Tr   )Tr   )
ra   torch.Tensorr;   r   rE   rt   rb   r   returnr   )gh㈵>)rv   r   r~   r   r   r   )r0   r1   r2   r   rQ   rS   rk   r{   r   r   r   r.   r   r4   r5   s   @r   r7   r7   H   s|     !"M: kl$$$$+.$$DH$$dg$$	$$NF"
4>',R7r   r7   c                  r    e Zd ZU dZdZded<   dZded<   ddZedd       Z	d	 Z
dd
ZdddZd Zd Zd Zy)OFTLayerz#
    Implements the OFT layer.
    rv   ztuple[str, ...]adapter_layer_names)r9   oft_block_sizeoft_dropoutother_param_namesc                D   || _         t        j                  i       | _        i | _        i | _        i | _        t        j                  i       | _        d| _        g | _        d| _	        || _
        | j                         }t        |t        j                        r|j                  |j                  }}nt        |t        j                         r|j"                  |j$                  }}nt'        |d      r&t'        |d      r|j(                  |j*                  }}n|t'        |d      r&t'        |d      r|j,                  |j.                  }}nJt'        |d      r3|j0                  j2                  dk(  r|j                  |j                  }}nt'        |d	      r2|j0                  j2                  d
k(  r|j                  |j                  }}n|j0                  j2                  dk(  r|j                  |j                  }}nt'        |d      r2|j0                  j2                  dk(  r|j                  |j                  }}n]t'        |d      r%t'        |d      r|j                  |j                  }}nd\  }}t5        j6                  dt9        |       dt:               || _        || _        y)z
        Initializes the OFT layer.

        Note, currently only support linear layer and convolutional layer, with further support for other layers to be
        added soon.

        Parameters:
        base_layer: the pretrained model layer
        FT
infeaturesoutfeatures
input_sizer   	codebooksQuantizedLinearw_bitWQLinear_GEMM
EetqLinearW_q	HQQLinearr<   out_features)NNzUnsupported layer type 'z(' encountered, proceed at your own risk.N)
base_layerr=   
ModuleDictrv   r   r9   r   _disable_adaptersmerged_adapterscast_input_dtype_enabledkwargsget_base_layerr   Linearr<   r   Conv2dr   out_channelshasattrr   r   r   r   r   r0   warningswarntypeUserWarning)r   r   r   r<   r   s        r   r   zOFTLayer.__init__<  s.    %]]2&
  ==,!&!(,%((*
j")),(2(>(>
@W@WK
BII.(2(>(>
@W@WKZ.7:}3U(2(=(=z?U?UKZ.7:}3U(2(=(=z?U?UKZ-*2F2F2O2OSd2d(2(>(>
@W@WKZ)j.B.B.K.K.^(2(>(>
@W@WK!!**l:(2(>(>
@W@WKZ'J,@,@,I,I[,X(2(>(>
@W@WK z=1gj.6Y,6,B,BJD[D[\,6)\MM*4
+;*<<degr '(r   c                    h | j                   S Nr   )r   s    r   _available_adapterszOFTLayer._available_adaptersw  s    }r   c                L    || j                   vry t        j                  d       y )NFScaling operation for OFT not supported! Automatically set scale to 1.)scalingr   r   )r   adapterscales      r   	set_scalezOFTLayer.set_scale{  s    $,,&^_r   c                    |dk(  ry | j                   D ]4  }|| j                  j                         vr t        j                  d       6 y )Nr	   r   active_adaptersrv   keysr   r   r   r   active_adapters      r   scale_layerzOFTLayer.scale_layer  sF    A:"22 	dNTZZ__%66MMbc		dr   Nc                    | j                   D ]4  }|| j                  j                         vr t        j                  d       6 y )Nz>Unscaling operation for OFT not supported! Keeping scale to 1.r   r   s      r   unscale_layerzOFTLayer.unscale_layer  s<    "22 	\NTZZ__%66MMZ[		\r   c                   	 |dkD  rt        |      }nt        j                         }| j                  j	                  t        j
                  ||i             |dk(  ry|dk7  rt| j                  |z  dk7  s|| j                  kD  r:|}| j                  | j                  |      }t        j                  d| d| d       t        | j                  |z        }n|dk7  ry|dk(  rt| j                  |z  dk7  s|| j                  kD  r:|}| j                  | j                  |      }t        j                  d| d| d       t        | j                  |z        }nt        d	      ||d
z
  z  dz  }t        |s|nd
||| j                  ||||	|
	      | j                  |<   | j                  ||       || j                  |<   || j                   |<   | j#                  |       | j%                  | j&                         y)ze
        Update the linear layer with trainable OFT weights. Override for other layer types.
        r/   r   r   Invalid `oft_block_size` (!)! Adjusted `oft_block_size` to ().Invalid `r` ()! Adjusted `r` to (ZSomething went wrong, please report this error: https://github.com/huggingface/peft/issuesr	   rV   )rA   rB   rC   rE   rF   N)r   r=   Identityr   updater   r<   adjust_oft_parametersr   r   r   r   r7   rv   reset_oft_parametersr9   r   %_move_adapter_to_device_of_base_layerset_adapterr   )r   adapter_namer9   r   module_dropoutrA   rB   rC   init_weightsrE   rF   oft_dropout_layerold_oft_block_sizeold_rr:   s                  r   update_layerzOFTLayer.update_layer  s    	 C :^ L "|=N.O PQ6n).0A5$JZJZ9Z%3"!%!;!;D<L<Ln!]01C0DDefteuuwx D$$67A!V!+!#q(A0@0@,@..t/?/?CeW4H2NO !1!1Q!67Nl 
 $~'9:a?
#4 Aa#1%=
$


<  	!!,=  !|,:L) 	22<@--.r   c                J   |du r:t         j                  j                  | j                  |   j                  dd       y|| j                  j                         v rI|du r7t         j                  j                  | j                  |   j                         yt        d|      y)z+
        Reset the OFT parameters.
        Fr/   g?)meanstdNTz$Unknown initialization init_weights=)r=   initnormal_rv   r@   r   zeros_r   )r   r   r   s      r   r   zOFTLayer.reset_oft_parameters  s     5 GGOODJJ|4;;#3OO4::??,,t#tzz,7>>? #H</!JKK -r   c                    ||k  r#|}||k  r||z  dk7  r|dz  }||k  r||z  dk7  rn|S |}|dkD  r||z  dk7  r|dz  }|dkD  r	||z  dk7  r||z
  ||z
  k  r|S |S )zY
        Adjust the OFT parameters to be divisible by the in_features dimension.
        r   r	    )r   r<   paramshigher_paramslower_paramss        r   r   zOFTLayer.adjust_oft_parameters  s     K"M;.;3NRS3S"  ;.;3NRS3S Q;#=#BAL Q;#=#B \!}v'=>  r   )r   	nn.Moduler   None)r   zset[str])r   floatr   r  r   r   r  )r0   r1   r2   r3   r   __annotations__r   r   propertyr   r   r   r   r   r   r   r  r   r   r   r   2  s]    
 ,65)OO9)v  `d\O/bL!r   r   c                       e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 fdZd
ddZddZddZddZd fdZ	 xZ
S )r   zOFT implemented in Linear layerc                    t         |           t        j                  | |fi | || _        || _        | j                  |||||||||	|

       || _        y N)r   r   rA   rB   rC   r   rE   rF   )r   r   r   fan_in_fan_out_active_adapterr   is_target_conv_1d_layer)r   r   r   r9   r   r   rA   rB   rC   rE   rF   r  r   r  r   r   s                  r   r   zLinear.__init__  sp    " 	$
5f5,+))#%1%= 	 	
 (?$r   c                   t        | |      }|sy|D ]  }|| j                  v s| j                         }|j                  j                  }|r|j                  j
                  }| j                  |      }t        j                  |dd      }t        j                  ||j                  |j                              }t        j                  |dd      }t        j                  |      j                         st        d| d      |j                         j                  |      |j                  _        n|j                  j
                  }| j                  |      }t        j                  |dd      }t        j                  ||j                  |j                              }t        j                  |dd      }|j                         j                  |      |j                  _        | j                  j!                  |        y)ab  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If `None`, all active adapters will be merged.
                Defaults to `None`.
        Nr   r	   z1NaNs detected in the merged weights. The adapter z seems to be broken)r   r   r   r@   rL   dataget_delta_weightr   rM   mmr`   isfiniteallr   r   r   appendr   
safe_mergeadapter_namesr   r   
orig_dtypeorig_weightsoft_mats           r   mergezLinear.merge0  s    0mD+ 	<N!9!99!002
'..44
#-#4#4#9#9L"33NCG#(??<A#FL#(88G\__W]]5S#TL#(??<A#FL >>,7;;=(OP^O__rs  .:-D-D-F-I-I*-UJ%%*#-#4#4#9#9L"33NCG#(??<A#FL#(88G\__W]]5S#TL#(??<A#FL-9-D-D-F-I-I*-UJ%%*$$++N;7	<r   c                   | j                   st        j                  d       y| j                         }|j                  j
                  }t        | j                        dkD  r| j                  j                         }|| j                  j                         v r| j                  |      }| j                         j                  j                  }t        j                  |dd      }t        j                  |j!                         |j#                  |j
                              }t        j                  |dd      }|j#                  |      |j                  _        t        | j                        dkD  ryyzW
        This method unmerges all merged adapter layers from the base weights.
        z Already unmerged. Nothing to do.Nr   r	   )mergedr   r   r   r@   rL   r   r   poprv   r   r  r  r   rM   r  tr`   r   r   r  r   r  r  s         r   unmergezLinear.unmerge_  s    {{MM<=((*
&&,,
$&&'!+!11557N!22//?#224;;@@$|QB$xx		\__W]]5ST$|QB)5)D
!!& $&&'!+r   c                <    | j                   |   j                         S r   rv   r   r   r   s     r   r  zLinear.get_delta_weightu       zz,'2244r   c                $   |j                   }| j                  r3| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}n| j
                  D ]\  }|| j                  j                         vr | j                  |   }| j                  ||j                  j                         } ||      }^  | j                  |j                  |      g|i |}|j                  |      }|S r   rL   disable_adaptersr"  r&  r   r   rv   r   _cast_input_dtyper@   r`   r   r&   argsr   rd   resultr   rv   s           r   r.   zLinear.forward  s      {{$T__Q888F[[$T__Q888F"&"6"6 !)::

>2**1ell.@.@A!H %T__QTT.%9KDKFKF>*r   c                *    t         |          }d|z   S Nzoft.r   __repr__r   repr   s     r   r6  zLinear.__repr__      g |r   )   r   r/   Fr   FFr   FTF)r   strr9   r   r   r   r   r  rA   rt   rB   r  rC   rt   rE   rt   rF   r   r  rt   r   Union[bool, str]r  rt   r   r  FNr  rt   r  zOptional[list[str]]r   r  r	  r   z!tuple[torch.Tensor, torch.Tensor])r&   r   r   r   r   r;  )r0   r1   r2   r3   r   r  r&  r  r.   r6  r4   r5   s   @r   r   r     s    )  #!#(()$)-(-#? #? 	#?
 #? #? #? #? #? !#? #&#? #? '#? "&#?  
!#?J-<^E,	5. r   r   c                       e Zd ZdZ	 	 	 	 	 	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
 fdZd ZdddZddZddZddZ	d fdZ
 xZS )r   zOFT implemented in Conv2d layerc                    t         |           t        j                  | |       || _        || _        | j                  |||||||	|
||
       y r  )r   r   r   r  r  r   )r   r   r   r9   r   r  r   rA   rB   rC   r   rE   rF   r   r   s                 r   r   zConv2d.__init__  sc      	$
+,+ 	))#%1%= 	 	
r   c                   |dkD  rt        |      }nt        j                         }| j                  j	                  t        j
                  ||i             | j                         }|j                  d   dkD  rt        d      | j                  |j                  d   z  |j                  d   z  }|dk(  rQ|dk7  rL||z  dk7  s||kD  r0|}| j                  ||      }t        j                  d| d| d       t        ||z        }na|dk7  rQ|dk(  rL||z  dk7  s||kD  r0|}| j                  ||      }t        j                  d	| d
| d       t        ||z        }nt        d      ||dz
  z  dz  }t        |s|nd|||||||j                  |	|

      | j                   |<   | j#                  ||       || j$                  |<   || j&                  |<   | j)                  |       | j+                  | j,                         y)zE
        Update the conv2d layer with trainable OFT weights.
        r/   r   r   r	   z1Conv2d with dilation > 1 is not supported by OFT.r   r   r   r   r   r   rV   )rA   rB   rC   rD   rE   rF   N)r   r=   r   r   r   r   r   dilationr   r<   rD   r   r   r   r   r7   rv   r   r9   r   r   r   r   )r   r   r9   r   r   rA   rB   rC   r   rE   rF   r   r   conv_filter_dimr   r   r:   s                    r   r   zConv2d.update_layer  s+   " C :^ L "|=N.O PQ ((*
q!A%PQQ**Z-C-CA-FFI_I_`aIbb6n)/148X%3"!%!;!;O^!\01C0DDefteuuwx O~56A!V!+"a'1+>..BeW4H2NO A!56Nl 
 $~'9:a?
#4 Aa#"..1%=$


<  	!!,=  !|,:L) 	22<@--.r   c                   t        | |      }|sy|D ]  }|| j                  j                         v s!| j                         }|j                  j
                  }|rJ|j                  j                  j                         }| j                  |      }|j                  | j                  | j                  |j                  d   z  |j                  d   z        }t        j                  |dd      }t        j                  ||j!                  |j
                              }t        j                  |dd      }|j                  | j                  | j                  |j                  d   |j                  d         }|j#                         j!                  |      |j                  _        nH| j                  |      }|j                  j                  j                         }|j                  | j                  | j                  |j                  d   z  |j                  d   z        }t        j                  |dd      }t        j                  ||j!                  |j
                              }t        j                  |dd      }|j                  | j                  | j                  |j                  d   |j                  d         }|j#                         j!                  |      |j                  _        | j$                  j'                  |        y)a^  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        Nr   r	   )r   rv   r   r   r@   rL   r  cloner  r#   r   r<   rD   r   rM   r  r`   r   r   r  r  s           r   r  zConv2d.merge  s    0mD+ %	<N!22!002
'..44
 $.#4#4#9#9#?#?#AL"33NCG#/#4#4))4+;+;j>T>TUV>W+WZdZpZpqrZs+s$L $)??<A#FL#(88G\__W]]5S#TL#(??<A#FL#/#4#4))4+;+;Z=S=STU=VXbXnXnopXq$L .:-D-D-F-I-I*-UJ%%*"33NCG#-#4#4#9#9#?#?#AL#/#4#4))4+;+;j>T>TUV>W+WZdZpZpqrZs+s$L $)??<A#FL#(88G\__W]]5S#TL#(??<A#FL#/#4#4))4+;+;Z=S=STU=VXbXnXnopXq$L .:-D-D-F-I-I*-UJ%%*$$++N;K%	<r   c                   | j                   st        j                  d       y| j                         }|j                  j
                  }t        | j                        dkD  r| j                  j                         }|| j                  j                         v r| j                  |      }| j                         j                  j                  j                         }|j                  | j                  | j                   | j                         j"                  d   z  | j                         j"                  d   z        }t%        j&                  |dd      }t%        j(                  |j+                         |j-                  |j
                              }t%        j&                  |dd      }|j                  | j                  | j                   | j                         j"                  d   | j                         j"                  d         }|j-                  |      |j                  _        t        | j                        dkD  ryyr!  )r"  r   r   r   r@   rL   r   r   r#  rv   r   r  r  rG  r#   r   r<   rD   r   rM   r  r$  r`   r%  s         r   r&  zConv2d.unmergeG  s    {{MM<=((*
&&,,
$&&'!+!11557N!22//?#224;;@@FFH+00%%$$t':':'<'H'H'KKdNaNaNcNoNopqNrr   %|QB$xx		\__W]]5ST$|QB+00%%$$'')55a8'')55a8	  *6)D
!!&) $&&'!+r   c                <    | j                   |   j                         S r(  r)  r*  s     r   r  zConv2d.get_delta_weightg  r+  r   c                $   |j                   }| j                  r3| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}n| j
                  D ]\  }|| j                  j                         vr | j                  |   }| j                  ||j                  j                         } ||      }^  | j                  |j                  |      g|i |}|j                  |      }|S r   r-  r0  s           r   r.   zConv2d.forwardr  s      {{$T__Q888F[[$T__Q888F"&"6"6 !)::

>2**1ell.@.@A!H %T__QTT.%9KDKFKF>*r   c                *    t         |          }d|z   S r4  r5  r7  s     r   r6  zConv2d.__repr__  r9  r   )
r:  r   Fr/   Fr   FTFr   )r   r  r   r;  r9   r   r   r   r  rt   r   r  rA   rt   rB   r  rC   rt   r   r<  rE   rt   rF   r   r   r  r=  r>  r	  r?  )r&   r   r1  r   r   r   r   r   r@  )r0   r1   r2   r3   r   r   r  r&  r  r.   r6  r4   r5   s   @r   r   r     s    ) $ #!)-#(()"
"
 "
 	"

 "
 "
 "
 "
 "
 "
 '"
 !"
 #&"
 
"
HI/V7<rE@	5. r   r   c                d   d }t        | t              r| j                         }n| }t        |t        j                  j
                        rt        | |fi |}|S t        |t        j                  j                        r3|d   r!t        j                  d       dx|d<   |_	        t        | |fi |}|S )Nr  zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.F)
r   r   r   r   r=   r   r   r   r   r  )targetr   
oft_configr   
new_moduletarget_base_layers         r   dispatch_defaultrQ    s     J&.)"113"#UXX__5FL;F;
  
%uxx	7"#MM3 DIHF#$z'@FL;F;
r   )rM  ztorch.nn.Moduler   r;  rN  r
   r   zOptional[torch.nn.Module])
__future__r   r   typingr   r   r   r   torch.nnr=   torch.nn.functional
functionalr   peft.tuners.tuners_utilsr   r   configr
   Moduler   r7   r   r   r   rQ  r  r   r   <module>rZ     s    #  ' '     L ) )Xg7		 g7TS!~ S!lQRYY QhoRYY od 
 r   