
    bi?:                         d dl Z d dlmZmZ d dlZd dlmZ d dlmZ d dl	m
Z
mZ d dlmZ  G d de
      Z G d d	ej                  e      Z G d
 dej                  e      Z G d de      Z G d de      Zy)    N)AnyOptional)Conv1D)BaseTunerLayercheck_adapters_to_merge)	transposec                   D    e Zd ZdZdej
                  deddfdZd Zd Z	y)	IA3Layer)ia3_l
base_layeris_feedforwardreturnNc                    || _         t        j                  i       | _        d| _        g | _        || _        | j                         }t        |t        j                        r|j                  |j                  }}nt        |t        j                  t        j                  f      r|j                  |j                  }}nt        |t        j                         r|j"                  |j$                  }}nmt        |t&              rFt)        |j*                  d      r|j*                  j,                  n|j*                  j.                  \  }}nt1        dt3        |             || _
        || _        y )NFds_shapezUnsupported layer type )r   nnParameterDictr   _disable_adaptersmerged_adaptersr   get_base_layer
isinstanceLinearin_featuresout_featuresConv2dConv3din_channelsout_channels	Embeddingnum_embeddingsembedding_dimr   hasattrweightr   shape
ValueErrortype)selfr   r   kwargsr   r   s         P/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/ia3/layer.py__init__zIA3Layer.__init__   s   $%%b)
!&!,((*
j")),(2(>(>
@W@WK
RYY		$:;(2(>(>
@W@WK
BLL1(2(A(A:C[C[K
F+.5j6G6G.T
!!**ZdZkZkZqZq &K 6tJ7G6HIJJ&(    c                 d   | j                   r"t        j                  d| j                  f      }n!t        j                  | j                  df      }t        j                  |      | j                  |<   |r| j                  |       | j                  |       | j                  | j                         y N   )r   torchrandnr   r   r   	Parameterr   reset_ia3_parameters%_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r&   adapter_nameinit_ia3_weightsr"   s       r(   update_layerzIA3Layer.update_layer6   s     [[!T%5%5!67F[[$"3"3Q!78F#%<<#7

< %%l322<@--.r*   c                     || j                   j                         v r.t        j                  j	                  | j                   |   d       y y )Ng      ?)r   keysr   init	constant_)r&   r5   s     r(   r1   zIA3Layer.reset_ia3_parametersC   s7    4::??,,GGdjj6< -r*   )
__name__
__module____qualname__adapter_layer_namesr   Moduleboolr)   r7   r1    r*   r(   r
   r
      s1    $)299 )d )QU )0/=r*   r
   c                        e Zd Z	 	 	 	 ddej                  dedededededdf fd	Zdd
edee	e      ddfdZ
ddZdej                  dededej                  fdZ xZS )r   r   r5   fan_in_fan_outr   is_target_conv_1d_layerr6   r   Nc                     t         |           t        j                  | ||       || _        || _        || _        | j                  ||       y N)r   )superr)   r
   rD   rE   _active_adapterr7   )	r&   r   r5   rD   r   rE   r6   r'   	__class__s	           r(   r)   zLinear.__init__K   sN     	$
>J,'>$+,(89r*   
safe_mergeadapter_namesc                 h   t        | |      }|sy|D ]  }|| j                  j                         v s!| j                         }t	        | j                  |   j
                  | j                        }|j                  j
                  j                  }|r|j                  j
                  }t        j                  ||      }t        j                  |      j                         st        d| d      |j                  |      |j                  _        nHt        j                  |j                  j
                  |      j                  |      |j                  _        | j                  s|j                   | j                  |   j#                  |j                   j$                        }|j                   j
                  j                  }t        j                  |j                   j
                  |j
                        j                  |      |j                   _        | j&                  j)                  |         y)^  
        Merge the active adapter weights into the base weights

        Args:
            safe_merge (`bool`, *optional*):
                If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                before merging the weights. This is useful if you want to check if the merge operation will produce
                NaNs. Defaults to `False`.
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        N1NaNs detected in the merged weights. The adapter  seems to be broken)r   r   r9   r   r   datarD   r"   dtyper.   mulisfiniteallr$   tor   biasreshaper#   r   append)	r&   rK   rL   active_adapterr   r   
orig_dtypeorig_weightsscalings	            r(   mergezLinear.merge\   s    0mD+ 	<N!22!002
!$**^"<"A"A4CVCVW'..3399
#-#4#4#9#9L#(99\5#AL >>,7;;=(OP^O__rs  .:__Z-HJ%%*-2YYz7H7H7M7Mu-U-X-XYc-dJ%%***
0K"jj8@@AVAVWG!+!5!5!;!;J+099Z__5I5I7<<+X+[+[\f+gJOO($$++N;-	<r*   c                    | j                   st        j                  d       yt        j                  d       t        | j                        dkD  r| j                  j                         }|| j                  j                         v rg| j                         }t        | j                  |   j                  | j                        dz   }|j                  j                  j                  }t        j                  |j                  j                  |      j!                  |      |j                  _
        | j"                  s|j$                  | j                  |   j'                  |j$                  j(                        }|j$                  j                  j                  }t        j                  |j$                  j                  |j                  dz         j!                  |      |j$                  _
        t        | j                        dkD  ryy)W
        This method unmerges all merged adapter layers from the base weights.
         Already unmerged. Nothing to do.N,Unmerge result can be inaccurate for (IA)^3.r   :0yE>)mergedwarningswarnlenr   popr   r9   r   r   rQ   rD   r"   rR   r.   divrV   r   rW   rX   r#   )r&   rZ   r   r   r[   r]   s         r(   unmergezLinear.unmerge   su    {{MM<=DE$&&'!+!11557N!22!002
!$**^"<"A"A4CVCVWZ^^'..3399
).:3D3D3I3I5)Q)T)TU_)`
!!&**
0K"jj8@@AVAVWG!+!5!5!;!;J+099Z__5I5I7<<Z^K^+_+b+bcm+nJOO( $&&'!+r*   xargsr'   c                    |j                   x}}| j                  r4| j                  r| j                           | j                  |g|i |}|S | j                  r | j                  |g|i |}|S d}| j
                  D ]X  }|| j                  j                         vr | j                  |   j                   }|| j                  |   j                         z  }Z | j                  r=|j                  |      }||z  j                  |      }	 | j                  |	g|i |}|S  | j                  |g|i |}|j                   }
||z  j                  |
      }|S r,   )rR   disable_adaptersrd   rj   r   r4   r   r9   flattenr   rV   )r&   rk   rl   r'   rR   previous_dtyperesultia3_scalingrZ   intermresult_dtypes              r(   forwardzLinear.forward   sf   !"(  {{$T__Q888F, + [[$T__Q888F( % K"&"6"6 D!)::

>288tzz.9AACC	D ""DDK k/--n=(A$A&A 	 )<T<V<%|| ;.22<@r*   )FFFTFNr   N)r<   r=   r>   r   r@   strrA   r)   r   listr^   rj   r.   Tensorr   ru   __classcell__rJ   s   @r(   r   r   I   s      %$(-!%:II: : 	:
 : "&: : 
:"(< (<Xd3i=P (<\` (<To. c S U\\ r*   r   c                        e Zd Z	 	 	 ddej                  dededededdf fdZd	 Zdd
ede	e
e      ddfdZddZdej                  dededej                  fdZ xZS )_ConvNdr   r5   rD   r   r6   r   Nc                     t         |           t        j                  | ||       || _        || _        |j
                  j                         | _        | j                  ||       y rG   )	rH   r)   r
   rD   rI   r"   dim_kernel_dimr7   )r&   r   r5   rD   r   r6   r'   rJ   s          r(   r)   z_ConvNd.__init__   s]     	$
>J,+%,,002,(89r*   c                 f   | j                   r| j                  n| j                  }d|fd| j                  dz
  z  z   }t	        j
                  |      }t        j                  |      | j                  |<   |r| j                  |       | j                  |       | j                  | j                         y )Nr-   )r-      )r   r   r   r   r.   r/   r   r0   r   r1   r2   r3   r4   )r&   r5   r6   num_featuresweights_sizer"   s         r(   r7   z_ConvNd.update_layer   s    +/+>+>t''DDUDU<(443C3Ca3G+HH\*#%<<#7

< %%l322<@--.r*   rK   rL   c                 T   t        | |      }|sy|D ]  }|| j                  j                         v s!| j                         }|j                  j
                  j                  }| j                  |   j
                  }| j                  s|j                  dd      }|rt        j                  |j                  j
                  |      j                         }t        j                  |      j                         st        d| d      |j                  |      |j                  _        nHt        j                  |j                  j
                  |      j                  |      |j                  _        | j                  s|j                   | j                  |   j#                  |j                   j$                        }t        j                  |j                   j
                  |j
                        j                  |      |j                   _        | j&                  j)                  |        y)rN   Nr   r-   rO   rP   )r   r   r9   r   r"   rQ   rR   r   r   r.   rS   clonerT   rU   r$   rV   rW   rX   r#   r   rY   )	r&   rK   rL   rZ   r   r[   rr   output_weightr]   s	            r(   r^   z_ConvNd.merge   s    0mD+ 	<N!22!002
'..3399
"jj8==**"-"7"71"=K$)IIj.?.?.D.Dk$R$X$X$ZM >>-8<<>(OP^O__rs  .;-=-=j-IJ%%*-2YYz7H7H7M7M{-[-^-^_i-jJ%%***
0K"jj8@@AVAVWG+099Z__5I5I7<<+X+[+[\f+gJOO($$++N;1	<r*   c                 $   | j                   st        j                  d       yt        j                  d       t        | j                        dkD  r| j                  j                         }|| j                  j                         v rn| j                         }|j                  j                  j                  }| j                  |   j                  }| j                  s|j                  dd      }t        j                  |j                  j                  |dz         j!                  |      |j                  _
        | j                  s|j"                  | j                  |   j%                  |j"                  j&                        }|j"                  j                  j                  }t        j(                  |j"                  j                  |j                        j!                  |      |j"                  _
        t        | j                        dkD  ryy)r`   ra   Nrb   r   r-   rc   )rd   re   rf   rg   r   rh   r   r9   r   r"   rQ   rR   r   r   r.   ri   rV   rW   rX   r#   rS   )r&   rZ   r   r[   rr   r]   s         r(   rj   z_ConvNd.unmerge  s}    {{MM<=DE$&&'!+!11557N!22!002
'..3399
"jj8==**"-"7"71"=K).:3D3D3I3I;Y]K])^)a)abl)m
!!&**
0K"jj8@@AVAVWG!+!5!5!;!;J+099Z__5I5I7<<+X+[+[\f+gJOO( $&&'!+r*   rk   rl   r'   c                    |j                   x}}| j                  r4| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}nd}| j
                  D ]J  }|| j                  j                         vr | j                  |   j                   }|| j                  |   z  }L | j                  r^|j                  |      }||z  j                  | j                         j                  j                         }	 | j                  |	g|i |}n* | j                  |g|i |}|j                  |      |z  }|j                  |      }|S r,   )rR   rn   rd   rj   r   r4   r   r9   r   rV   r   r"   )
r&   rk   rl   r'   rR   rp   rq   rr   rZ   rs   s
             r(   ru   z_ConvNd.forward  sX   !"(  {{$T__Q888F[[$T__Q888FK"&"6"6 :!)::

>288tzz.99	: ""DDK k/--d.A.A.C.J.J.P.PQ(A$A&A(<T<V<5)K7>*r*   )FFTrv   rw   )r<   r=   r>   r   r@   rx   rA   r)   r7   r   ry   r^   rj   r.   rz   r   ru   r{   r|   s   @r(   r~   r~      s    
  %$!%:II: : 	:
 : : 
:"	/*< *<Xd3i=P *<\` *<Xh2 c S U\\ r*   r~   c                        e Zd Z fdZ xZS )r   c                 r    t        |   |i | | j                  dk(  st        d| j                         y )N   z0Conv2d layer kernel must have 4 dimensions, not rH   r)   r   r$   r&   rl   r'   rJ   s      r(   r)   zConv2d.__init__?  C    $)&)1$OPTP`P`Oabcc %r*   r<   r=   r>   r)   r{   r|   s   @r(   r   r   <  s    d dr*   r   c                        e Zd Z fdZ xZS )r   c                 r    t        |   |i | | j                  dk(  st        d| j                         y )N   z0Conv2d layer kernel must have 5 dimensions, not r   r   s      r(   r)   zConv3d.__init__G  r   r*   r   r|   s   @r(   r   r   E  s    d dr*   r   )re   typingr   r   r.   torch.nnr   transformers.pytorch_utilsr   peft.tuners.tuners_utilsr   r   
peft.utilsr   r
   r@   r   r~   r   r   rB   r*   r(   <module>r      so         - L  ,=~ ,=^oRYY od~bii ~BdW ddW dr*   