
    bi?                       d dl mZ d dlZd dlmZ d dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ dd	lmZ  e
       r  G d
 dej&                  j(                  e      Z e	       r! G d dej&                  j(                  e      Zyy)    )annotationsN)Optional)is_bnb_4bit_availableis_bnb_available)check_adapters_to_merge)dequantize_bnb_weight)	transpose   )	VeraLayerc                  x     e Zd Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZd	d
dZddZddZddZd fdZ xZ	S )Linear8bitLtc
           	         t         |           t        j                  | |       || _        || _        | j                  |||||||	       y N)vera_dropoutinit_weights	d_initialsuper__init__r   fan_in_fan_out_active_adapterupdate_layerself
base_layeradapter_namevera_Avera_Brr   r   r   r   kwargs	__class__s              O/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/vera/bnb.pyr   zLinear8bitLt.__init__!   X     GtZ0"0D#/D ))#      c                "   | j                   rNt        j                  ddj                  | j                         ddj                  | j
                         d       t        | |      }|sy |D ]  }|| j                  j                         vr!t        j                  d       | j                  |      }| j                         j                  }| j                         j                  }|j                  |j                  |_        t        ||      }|j                  |j                         j                  |j"                        |z   }|r2t%        j&                  |      j)                         st+        d| d      t,        j.                  j1                  |j                  d      d	|j2                  
      j                  |j"                        | j                         _        |j5                          | j                  j7                  |        y )N'Already following adapters were merged ,#. You are now additionally merging .zWMerge vera module to 8-bit linear may get different generations due to rounding errors.1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)mergedwarningswarnjoinmerged_adaptersactive_adaptersr   vera_lambda_dkeysget_delta_weightget_base_layerweightstateSCBr   todtypedevicetorchisfiniteall
ValueErrorbnbnn
Int8Paramsr/   reset_gradsappend)	r   
safe_mergeadapter_namesactive_adapter	vera_datar:   r;   outputw_datas	            r"   mergezLinear8bitLt.merge=   s   {{=chhtG[G[>\=] ^88;AUAU8V7WWXZ
 4D-HM "/ <!););)@)@)BBm !11.A	,,.55++-3399$ &

EI.vu=9??366y7G7GH9TennV&<&@&@&B$KNK[[no  03vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#$$++N;5<r$   c                   | j                   st        j                  d       y t        | j                        dkD  r| j                  j                         }|| j                  j                         vrPt        j                  d       | j                  |      }| j                         j                  }| j                         j                  }|j                  |j                  |_        t        ||      }|j                  |j                        j                  |j                         |z
  }t"        j$                  j'                  |j                  d      d|j(                        j                  |j                         | j                         _
        |j+                          t        | j                        dkD  ry y )NAlready unmerged. Nothing to dor   zYUnmerge vera module to 8-bit linear may get different generations due to rounding errors.)r;   r,   Fr-   )r0   r1   r2   lenr4   popr6   r7   r8   r9   r:   r;   r<   r   r=   r>   r?   rD   rE   rF   r/   rG   )r   rK   rL   r:   r;   rM   rN   s          r"   unmergezLinear8bitLt.unmerged   s\   ;;?@d**+a/!%!5!5!9!9!;!););)@)@)BBo !11.A	,,.55++-3399$ &

EI.vUC9??366y7G7GH9T/2vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#) d**+a/r$   c                   | j                   |   }| j                  |   }|j                  }|j                  }|j                  dk(  xr( |t
        j                  k(  xs |t
        j                  k(  }| j                  |   }| j                  |   }|r@|j                         }|j                         }|j                         }|j                         }|ddd| j                  f   j                  |j                        }	|d| j                  ddf   j                  |j                        }
|j                  d      }|j                  d      }t        ||
z  ||	z  z  | j                         }|r|j                  |      }|S )a=  
            Compute the delta weight for the given adapter.

            Args:
                adapter (str): The name of the adapter for which the delta weight should be computed.

            Returns:
                torch.Tensor: The computed delta weight for the VeRA adapter.

            Note:
                This method implements the VeRA-specific weight update. Unlike LoRA, VeRA uses shared projection
                matrices (vera_A and vera_B) across all layers, along with per-layer trainable parameters (lambda_d and
                lambda_b).
            r,   Nr>   r   r   r?   r>   typer@   float16bfloat16r6   vera_lambda_bfloatin_featuresr=   out_features	unsqueezer	   r   r   adapterr   r   r?   r>   cast_to_fp32lambda_dlambda_bsliced_Asliced_Boutput_tensors               r"   r8   zLinear8bitLt.get_delta_weight   sh     [[)F[[)F ]]FLLE
 ";;%/gUemm5K5fuX]XfXfOfL))'2H))'2H#>>+#>>+a!34#3#3!33477HH1 1 1114588IH))"-H))"-H &x(':x(?R&SUYUhUhiM - 0 0u 0 =  r$   c           	        | j                   r4| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}n | j                  |g|i |}| j                  D ]  }|| j
                  j                         vr!| j
                  |   }| j                  |   }| j                  |   }| j                  |   }	| j                  |   }
t        j                          }|r8|j                  }|j                  }|j                  |k7  r|j                  |      }|ddd| j                  f   j                  |j                         }|	d| j"                  ddf   j                  |j                         } |
|j                  |j                              }|t        j$                  j&                  j)                  |t        j$                  j&                  j)                  ||      z  |      z  }|r|j                        }||z   } |j                  |j                        S )a  
            Perform the forward pass using the VeRA adapter.

            Args:
                x (torch.Tensor): Input tensor.

            Returns:
                torch.Tensor: Output tensor after applying the VeRA adaptation.

            Note:
                This method implements the VeRA-specific forward pass. It applies the shared projections (vera_A and
                vera_B) along with the per-layer trainable parameters (lambda_d and lambda_b) to compute the adapter
                output.
            N)disable_adaptersr0   rT   r   r5   r6   r7   r\   r   r   r   r@   is_autocast_enabledr>   r=   r^   r?   r_   rE   
functionallinearr   xargsr    resultrK   rd   re   r   r   dropoutrequires_conversionexpected_dtypecompute_dtyperf   rg   x_tempadapter_outputs                     r"   forwardzLinear8bitLt.forward   s    $$;;LLN(<T<V<(<T<V<(<T<V<&*&:&: 5N%T-?-?-D-D-FF #11.AH#11.AH![[8F![[8F"//?G.3.G.G.I*I'*)/(077m3 !] 3A%a);4+;+;);&;<??IH%&9(9(9&91&<=@@JH$QTT(..%9:F%-0C0C0J0J 588#6#6#=#=fh#OOQY1 &N +)7):):>)J#n4F?5D 99QWW%%r$   c                *    t         |          }d|z   S Nzvera.r   __repr__r   repr!   s     r"   r|   zLinear8bitLt.__repr__       '"$CS= r$   r   g        FTg?r   ztorch.nn.Moduler   strr   intr   r]   r   boolr   r   r   r]   returnNoneFNrI   r   rJ   zOptional[list[str]]r   r   r   r   r   torch.Tensorro   r   r   r   r   r   
__name__
__module____qualname__r   rO   rT   r8   rx   r|   __classcell__r!   s   @r"   r   r       s     "%#(!%"	'	 	 	  	 !	 	 	 	8%	<N	$63	!j9	&v	! 	!r$   r   c                  x     e Zd Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZd	d
dZddZddZddZd fdZ xZ	S )
Linear4bitc
           	         t         |           t        j                  | |       || _        || _        | j                  |||||||	       y r   r   r   s              r"   r   zLinear4bit.__init__   r#   r$   c                   | j                   rNt        j                  ddj                  | j                         ddj                  | j
                         d       t        | |      }|sy |D ]  }|| j                  j                         vr!t        j                  d       | j                  |      }| j                         j                  }|j                  }|j                         D ci c]  \  }}|j                  d      r|| }}}t        j                   j#                  |j$                  |j&                        |z   }	|r2t)        j*                  |	      j-                         st/        d| d      t        j0                  j2                  |	j5                  d	      fd
di|j5                  |j6                        | j                         _        | j                  j9                  |        y c c}}w )Nr&   r'   r(   r)   zWMerge vera module to 4-bit linear may get different generations due to rounding errors._r*   r+   r,   r.   F)r0   r1   r2   r3   r4   r5   r   r6   r7   r8   r9   r:   __dict__items
startswithrD   rl   dequantize_4bitdataquant_stater@   rA   rB   rC   rE   
Params4bitr=   r?   rH   )
r   rI   rJ   rK   rL   r:   r    kvrN   s
             r"   rO   zLinear4bit.merge  s   {{=chhtG[G[>\=] ^88;AUAU8V7WWXZ
 4D-HM "/ <!););)@)@)BBm !11.A	,,.55+1<<>S41acAR!Q$SS77VEWEWX[ddennV&<&@&@&B$KNK[[no  03vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, $$++N;/< Ts   ,G5G5c                   | j                   st        j                  d       y t        | j                        dkD  r2| j                  j                         }|| j                  j                         vrPt        j                  d       | j                  |      }| j                         j                  }|j                  }t        j                  j                  |j                  |j                         |z
  }t        j"                  j$                  |j'                  d      fddi|j'                  |j(                        | j                         _
        t        | j                        dkD  r1y y )NrQ   r   zYUnmerge vera module to 4-bit linear may get different generations due to rounding errors.r,   r.   F)r0   r1   r2   rR   r4   rS   r6   r7   r8   r9   r:   r   rD   rl   r   r   r   rE   r   r=   r?   )r   rK   rL   r:   r    rN   s         r"   rT   zLinear4bit.unmerge7  s   ;;?@d**+a/!%!5!5!9!9!;!););)@)@)BBo !11.A	,,.5577VEWEWX[dd/2vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, d**+a/r$   c                   | j                   |   }| j                  |   }|j                  }|j                  }|j                  dk(  xr( |t
        j                  k(  xs |t
        j                  k(  }| j                  |   }| j                  |   }|r@|j                         }|j                         }|j                         }|j                         }|d d d | j                  f   j                  |j                        }	|d | j                  d d f   j                  |j                        }
|j                  d      }|j                  d      }t        ||
z  ||	z  z  | j                         }|r|j                  |      }|S )Nr,   rV   rW   rX   ra   s               r"   r8   zLinear4bit.get_delta_weightM  s`   [[)F[[)F]]FLLE!;;%/gUemm5K5fuX]XfXfOfL))'2H))'2H#>>+#>>+a!34#3#3!33477HH1 1 1114588IH))"-H))"-H%x(':x(?R&SUYUhUhiM - 0 0u 0 =  r$   c           	        | j                   r4| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}n | j                  |g|i |}|j	                         }| j
                  D ]  }|| j                  j                         vr!| j                  |   }| j                  |   }| j                  |   }| j                  |   }	| j                  |   }
t        j                          }|r8|j                  }|j                  }|j                  |k7  r|j                  |      }|d d d | j                   f   j                  |j"                        }|	d | j$                  d d f   j                  |j"                        } |
|j                  |j                              }|t        j&                  j(                  j+                  |t        j&                  j(                  j+                  ||      z  |      z  }|r|j                        }||z   } |j                  |j                        S )N)rj   r0   rT   r   cloner5   r6   r7   r\   r   r   r   r@   rk   r>   r=   r^   r?   r_   rE   rl   rm   rn   s                     r"   rx   zLinear4bit.forwardk  s   $$;;LLN(<T<V<(<T<V<(<T<V<&*&:&: 5N%T-?-?-D-D-FF #11.AH#11.AH![[8F![[8F"//?G.3.G.G.I*I'*)/(077m3 !] 3A%a);4+;+;);&;<??IH%&9(9(9&91&<=@@JH$QTT(..%9:F%-0C0C0J0J 588#6#6#=#=fh#OOQY1 &N +)7):):>)J#n4F?5D 99QWW%%r$   c                *    t         |          }d|z   S rz   r{   r}   s     r"   r|   zLinear4bit.__repr__  r   r$   r   r   r   r   r   r   r   r   r   r   s   @r"   r   r      s     "%#(!%"	'	 	 	  	 !	 	 	 	8"	<H	,	!<,	&\	! 	!r$   r   )
__future__r   r1   typingr   bitsandbytesrD   r@   peft.import_utilsr   r   peft.tuners.tuners_utilsr   peft.utils.integrationsr   peft.utils.otherr	   layerr   rE   Moduler   r    r$   r"   <module>r      sk    #     E < 9 &  Q!uxx	 Q!h e!UXX__i e! r$   