
    bi:              (          d dl mZ d dlmZ d dlmZ d dlZdZ eej                  d      r/dZej                  j                  Z
ej                  j                  Zn,ej                  j                  Z
ej                  j                  Zej                  j                  dd	        e
d      	 	 djd
ej                  dej                  dej                  dej                  dej                  deej                     deej                     deej                  eej                     f   fd       Zej                  j                  dd        e
d      	 	 djd
ej                  dej                  dej                  dej                  deej                     deej$                     dej                  fd       Zej                  j                  dd        e
d      d
ej                  dej                  fd       Zej                  j                  dd        e
d      d
ej                  dej                  dej                  fd        Zej                  j                  d!d"        e
d!      dkd
ej                  fd#       Zej                  j                  d$d%        e
d$      d
ej                  d&ej                  dej                  fd'       Z ed$d(      d
ej                  d&ej                  fd)       Zej                  j                  d*d+        e
d*      	 	 djd
ej                  dej                  dej                  deej$                     deej                     dej                  fd,       Zej                  j                  d-d.        e
d-      	 dkd
ej                  deej                  ej                  ej                  ej                  eej                     f   fd/       Zej                  j                  d0d1        e
d0      d
ej                  d2ej                  d3ed4ed5ee   dej$                  dej                  fd6       Zej                  j                  d7d8        e
d7      d
ej                  d2ej                  d3ed4ed5ee   dej$                  dej                  ddfd9       Zej                  j                  d:d;        e
d:      d
ej                  d3ed4ed<ej$                  deej                  ej                  f   f
d=       Zej                  j                  d>d?        e
d>      d
ej                  d2ej                  d@ej                  d3edej$                  dej                  fdA       Zej                  j                  dBdC        e
dB      d
ej                  d2ej                  d@ej                  d3edej$                  dej                  fdD       Zej                  j                  dEdF        e
dE      d
ej                  d@ej                  d3edeej                  ej                  f   fdG       Zej                  j                  dHdI        e
dH      d
ej                  dej                  dJee   d2ej                  d@ej                  d3edej                  fdK       Zej                  j                  dLdM        e
dL      d
ej                  dej                  dJee   d2ej                  d@ej                  d3edej                  ddfdN       Zej                  j                  dOdP        e
dO      	 dldQedRej                  dSej                  dTej                  dUeej                     dVeej                     dWedXedYedZed[ed\ed]ed^ed_ed`edaeddf$db       Zej                  j                  dcdd        e
dc      	 dldQedRej                  dSej                  dTej                  dUeej                     dYedZed[ed\ed]ed_ed`edeej                  dfeej                     dgej                  dheej                     d^edaeddf&di       Zy)m    )Sequence)prod)OptionalNFregister_fakeTz"bitsandbytes::int8_mixed_scaled_mmz{(Tensor A, Tensor CA, Tensor CB, Tensor SCA, Tensor SCB, Tensor? outlier_cols=None, Tensor? bias=None) -> (Tensor, Tensor?)ACACBSCASCBoutlier_colsbiasreturnc                 >   g |j                   d d |j                   d   }t        j                  || j                  | j                        }t        j
                  j                         j                         }| j                  |t        j                        }	||	fS )Nr   devicedtyper   )
shapetorchemptyr   r   libraryget_ctxnew_dynamic_size	new_emptyint64)
r   r   r	   r
   r   r   r   shapeCoutsubAs
             L/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/bitsandbytes/_ops.py_r!      sz     +rxx}*bhhqk*F
++fQXXQWW
=C==((*;;=L;;|5;;;7D9    zbitsandbytes::int8_scaled_mmzm(Tensor A, Tensor B, Tensor row_stats, Tensor col_stats, Tensor? bias=None, ScalarType? dtype=None) -> TensorB	row_stats	col_statsr   c                     g | j                   d d |j                   d   }t        j                  || j                  |xs t        j                        S )Nr   r   r   )r   r   r   r   float16)r   r#   r$   r%   r   r   r   s          r    r!   r!   4   sE     )qwws|(QWWQZ(F;;vahhe6Lu}}MMr"   z bitsandbytes::int8_linear_matmulz(Tensor A, Tensor B) -> Tensorc                 l   t        j                  | j                  t         j                  k(  d        t        j                  |j                  t         j                  k(  d        g | j                  d d |j                  d   }t        j
                  || j                  t         j                        S )Nc                       yNzA must be int8 r+   r"   r    <lambda>z_.<locals>.<lambda>I       r"   c                       yNzB must be int8r+   r+   r"   r    r,   z_.<locals>.<lambda>J   r-   r"   r   r   r   )r   _checkr   int8r   r   r   int32)r   r#   r   s      r    r!   r!   G   sv    	LLEJJ&(@A	LLEJJ&(@A(qwws|(QWWQZ(F;;vahhekkBBr"   z$bitsandbytes::int8_linear_matmul.outz'(Tensor A, Tensor B, Tensor! out) -> ()r   c                 ,    g  j                   d d |j                   d   t        j                   j                  t        j                  k(  d        t        j                  |j                  t        j                  k(  d        t        j                  j                   k(  fd       t        j                  j
                   j
                  k(   fd       t        j                  j                  t        j                  k(  fd       y )Nr   r   c                       yr*   r+   r+   r"   r    r,   z_.<locals>.<lambda>[   r-   r"   c                       yr/   r+   r+   r"   r    r,   z_.<locals>.<lambda>\   r-   r"   c                  (    d d j                    S NExpected out.shape == , got r   )r   r   s   r    r,   z_.<locals>.<lambda>]   s    0FvhfUXU^U^T_.` r"   c                  <    d j                    dj                    S NzExpected out.device == r9   r   r   r   s   r    r,   z_.<locals>.<lambda>^   "    3J188*TZ[^[e[eZf1g r"   c                  "    d j                    S )Nz!Expected out.dtype == int32, got r   )r   s   r    r,   z_.<locals>.<lambda>_   s    5VWZW`W`Va3b r"   )r   r   r0   r   r1   r   r2   )r   r#   r   r   s   ` `@r    r!   r!   W   s    (qwws|(QWWQZ(F	LLEJJ&(@A	LLEJJ&(@A	LLf$&`a	LLqxx')gh	LLekk)+bcr"   z#bitsandbytes::int8_vectorwise_quantz<(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor?)c                    t        j                  | j                  | j                  t         j                        }t        j                  t        | j                  d d       | j                  t         j                        }|dk(  r||d fS t         j                  j                         j                         }||| j                  |t         j                        fS )Nr   r           r   )r   r   r   r   r1   r   float32r   r   r   r   r   )r   	thresholdout_rowr$   r   s        r    r!   r!   h   s    kk!''!((%**EGD".qxxu}}UIC	4''==((*;;=LIq{{<u{{{KKKr"   z%bitsandbytes::int8_vectorwise_dequantz"(Tensor A, Tensor stats) -> Tensorstatsc                     t        j                  | j                  t         j                  k(  d        t        j                  | t         j
                        S )Nc                       yr*   r+   r+   r"   r    r,   z_.<locals>.<lambda>z   r-   r"   r   )r   r0   r   r1   
empty_likerC   r   rF   s     r    r!   r!   x   s5    	LLEJJ&(@AAU]]33r"   defaultc                 2    | |j                  dd      z  dz  S )Nr      g   @ ?)viewrJ   s     r    r!   r!      s      uzz"a  #777r"   zbitsandbytes::int8_mm_dequantzc(Tensor A, Tensor row_stats, Tensor col_stats, ScalarType? dtype=None, Tensor? bias=None) -> Tensorc                     t        j                  | j                  t         j                  k(  d        t        j                  | |xs t         j
                        S )Nc                       y)NzA must be int32r+   r+   r"   r    r,   z_.<locals>.<lambda>   r-   r"   r   )r   r0   r   r2   rI   r'   )r   r$   r%   r   r   s        r    r!   r!      s<     
LLEKK')BCAU%;emm<<r"   zbitsandbytes::int8_double_quantzL(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor, Tensor, Tensor?)c                 D   t        j                  | t         j                        }t        j                  | t         j                        }t        j                  t	        | j
                  d d       | j                  t         j                        }t        j                  | j
                  d   | j                  t         j                        }t         j                  j                         j                         }| j                  |t         j                        }|||||fS )Nr   r   r   )r   rI   r1   r   r   r   r   rC   r   r   r   r   r   )r   rD   rE   out_colr$   r%   	outlier_nr   s           r    r!   r!      s    
 q

3Gq

3GD".qxxu}}UIAGGBKNI%%'88:I;;y;<LGY	<??r"   zbitsandbytes::dequantize_4bitza(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype) -> Tensorabsmax	blocksize
quant_typer   c                 p    t        j                  |       t        j                  ||| j                        S )N)r   r   )r   _check_is_sizer   r   )r   rT   rU   rV   r   r   s         r    r!   r!      s)     
#;;uE!((;;r"   z!bitsandbytes::dequantize_4bit.outzj(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype, Tensor! out) -> ()c                 4    t        j                  |       t        j                  j                  k(  fd       t        j                  j                   j                  k(   fd       t        j                  j
                  k(  fd       y )Nc                  (    d d j                    S r7   r:   )r   r   s   r    r,   z_.<locals>.<lambda>       /EeWFSVS\S\R]-^ r"   c                  <    d j                    dj                    S r<   r=   r>   s   r    r,   z_.<locals>.<lambda>   r?   r"   c                  (    d  dj                    S NzExpected out.dtype == r9   r   r   r   s   r    r,   z_.<locals>.<lambda>   r[   r"   )r   rX   r0   r   r   r   )r   rT   rU   rV   r   r   r   s   `   ```r    r!   r!      s_     
#	LLe#%^_	LLqxx')gh	LLe#%^_r"   zbitsandbytes::quantize_4bitzW(Tensor A, int blocksize, str quant_type, ScalarType quant_storage) -> (Tensor, Tensor)quant_storagec                 2   t        j                  |       | j                         }|| z   }t        j                  |f| j                  t         j
                        }t        j                  |dz   |j                  dz  z  df| j                  |      }||fS )Nr   rM      )r   rX   numelr   r   rC   itemsize)r   rU   rV   r`   nblocksrT   r   s           r    r!   r!      s     
#		AYJF[[&1885==IF
++A=#9#9A#=>B188[h
iC;r"   z"bitsandbytes::dequantize_blockwisezQ(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype) -> Tensorcodec                      t        j                  |       t        j                   j                  t         j                  k(   fd       t        j
                   |      S )Nc                  "    d j                    S NzA must be uint8, got r   r   s   r    r,   z_.<locals>.<lambda>       3H	1R r"   r   )r   rX   r0   r   uint8rI   )r   rT   rg   rU   r   s   `    r    r!   r!      s@    	#	LLEKK')RSAU++r"   z&bitsandbytes::dequantize_blockwise.outzZ(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype, Tensor! out) -> ()c                     t        j                  |       t        j                   j                  t         j                  k(   fd       t        j                  j
                   j
                  k(   fd       t        j                  j                   j                  k(   fd       t        j                  j                  k(  fd       y )Nc                  "    d j                    S rj   r   rk   s   r    r,   z_.<locals>.<lambda>   rl   r"   c                  <    d j                    dj                    S r7   r:   r>   s   r    r,   z_.<locals>.<lambda>   "    1GyPVWZW`W`Va/b r"   c                  <    d j                    dj                    S r<   r=   r>   s   r    r,   z_.<locals>.<lambda>  r?   r"   c                  (    d  dj                    S r^   r   r_   s   r    r,   z_.<locals>.<lambda>  r[   r"   )r   rX   r0   r   rm   r   r   )r   rT   rg   rU   r   r   s   `   ``r    r!   r!      s     
#	LLEKK')RS	LLagg%'bc	LLqxx')gh	LLe#%^_r"   z bitsandbytes::quantize_blockwisez:(Tensor A, Tensor code, int blocksize) -> (Tensor, Tensor)c                    t        j                  |       | j                         }|| z   }t        j                  |f| j                  t         j
                        }t        j                  | t         j                        }||fS )Nr   r   )r   rX   rc   r   r   rC   rI   rm   )r   rg   rU   re   rf   rT   r   s          r    r!   r!     se    	#		AYJF[[&1885==IF


1EKK
0C;r"   zbitsandbytes::gemv_4bitzW(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize) -> TensorshapeBc                     t        j                  |       t        j                   j                          j	                  d      k(   fd       t        j                   j
                  t         j                  t         j                  t         j                  fv  fd       t        j                  j
                  t         j                  t         j                  t         j                  t         j                  fv fd       g  j                  d d |d   }t        j                  | j                   j
                        S )Nr   c                  "    d j                    S Nz5A must be a vector with leading dimensions of 1, got r:   rk   s   r    r,   z_.<locals>.<lambda>      4ijkjqjqir2s r"   c                  "    d j                    S Nz-A must be float16, bfloat16, or float32, got r   rk   s   r    r,   z_.<locals>.<lambda>       ?yI r"   c                  "    d j                    S NzNB must be backed by storage of type uint8, bfloat16, float16, or float32, got r   r#   s   r    r,   z_.<locals>.<lambda>$      `abahah`ij r"   r   r   )r   rX   r0   rc   sizer   r'   bfloat16rC   rm   r   r   r   )r   r#   ru   rT   rg   rU   r   s   ``     r    r!   r!     s     
#	LLaffRj(*st	LL	EMM5>>5==AAI 
LL	EKKNNj 'aggcrl&F1I&E;;uQXXQWW==r"   zbitsandbytes::gemv_4bit.outz`(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize, Tensor! out) -> ()c                 L    t        j                  |       t        j                   j                          j	                  d      k(   fd       t        j                   j
                  t         j                  t         j                  t         j                  fv  fd       t        j                  j
                  t         j                  t         j                  t         j                  t         j                  fv fd       t        j                  j                  g  j                  d d d   k(   fd       t        j                  j                   j                  k(   fd       t        j                  j
                   j
                  k(   fd       y )	Nr   c                  "    d j                    S rx   r:   rk   s   r    r,   z_.<locals>.<lambda>;  ry   r"   c                  "    d j                    S r{   r   rk   s   r    r,   z_.<locals>.<lambda>>  r|   r"   c                  "    d j                    S r~   r   r   s   r    r,   z_.<locals>.<lambda>B  r   r"   r   c                  R    dg  j                   d d d    dj                    S )Nr8   r   r   r9   r:   )r   r   ru   s   r    r,   z_.<locals>.<lambda>F  s3    ()C1773B<)C)C(DF399+V r"   c                  <    d j                    dj                    S r<   r=   r>   s   r    r,   z_.<locals>.<lambda>H  r?   r"   c                  <    d j                    dj                    S r^   r   r>   s   r    r,   z_.<locals>.<lambda>I  rq   r"   )r   rX   r0   rc   r   r   r'   r   rC   rm   r   r   )r   r#   ru   rT   rg   rU   r   s   ```   `r    r!   r!   0  s	    
#	LLaffRj(*st	LL	EMM5>>5==AAI 
LL	EKKNNj 
LL		/qwws|/VAY//V 
LLqxx')gh	LLagg%'bcr"   z$bitsandbytes::optimizer_update_32bita-  (str optimizer_name, Tensor(a0!) g, Tensor(a1!) p, Tensor(a2!) state1, Tensor(a3!)? state2, Tensor(a4!)? unorm_vec, float max_unorm, float param_norm, float beta1, float beta2, float beta3, float alpha, float eps, float weight_decay, int step, float lr, float gnorm_scale, bool skip_zeros=False) -> ()optimizer_namegpstate1state2	unorm_vec	max_unorm
param_normbeta1beta2beta3alphaepsweight_decaysteplrgnorm_scalec                    t        j                  j                         j                         k(  fd       t         j                  t         j                  t         j
                  g}t        j                  j                  |v fd       t        j                  j                  j                  k(  fd       y )Nc                  L    d j                          dj                          S Nz3g and p must have the same number of elements, got z and rc   r   r   s   r    r,   z_.<locals>.<lambda>i  '    Eaggi[PUVWV]V]V_U`a r"   c                  "    d j                    S Nz-g must be bfloat16, float16, or float32, got r   r   s   r    r,   z_.<locals>.<lambda>o  r|   r"   c                  <    d j                    dj                    S Nz9Expected all tensors to have the same dtype, got g.dtype=z
, p.dtype=r   r   s   r    r,   z_.<locals>.<lambda>s  "    KAGG9T^_`_f_f^gh r"   )r   r0   rc   r'   r   rC   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
skip_zeroscompute_dtypess    ``                r    r!   r!   R  s~    * 
LL		QWWYa mmU^^U]]CN	LL	>!I 
LL	177hr"   z-bitsandbytes::optimizer_update_8bit_blockwiseaD  (str optimizer_name, Tensor(a0!) g, Tensor(a1!) p, Tensor(a2!) state1, Tensor(a3!)? state2, float beta1, float beta2, float beta3, float alpha, float eps, int step, float lr, Tensor(a4!) qmap1, Tensor(a5!)? qmap2, Tensor(a6!) absmax1, Tensor(a7!)? absmax2, float weight_decay, float gnorm_scale, bool skip_zeros=False) -> ()qmap1qmap2absmax1absmax2c                    t        j                  j                         j                         k(  fd       t         j                  t         j                  t         j
                  g}t        j                  j                  |v fd       t        j                  j                  j                  k(  fd       t        j                  j                  t         j                  k(  fd       t        j                  j                  j                  cxk(  xr t         j
                  k(  nc fd       t        j                  j                  t         j                  k(  fd       t        j                  j                  j                  cxk(  xr t         j
                  k(  nc fd       y y )Nc                  L    d j                          dj                          S r   r   r   s   r    r,   z_.<locals>.<lambda>  r   r"   c                  "    d j                    S r   r   r   s   r    r,   z_.<locals>.<lambda>  r|   r"   c                  <    d j                    dj                    S r   r   r   s   r    r,   z_.<locals>.<lambda>  r   r"   c                  "    d j                    S )Nzstate1 must be uint8, got r   )r   s   r    r,   z_.<locals>.<lambda>  s    ,V\\N; r"   c                  <    dj                    d j                    S )Nz:Expected qmap1 and absmax1 to be float32, got qmap1.dtype=z, absmax1.dtype=r   )r   r   s   r    r,   z_.<locals>.<lambda>  s"    LU[[MYijqjwjwixy r"   c                  "    d j                    S )Nzstate2 must be uint8, got r   )r   s   r    r,   z_.<locals>.<lambda>  s    0? r"   c                  <    dj                    d j                    S )Nz:Expected qmap2 and absmax2 to be float32, got qmap2.dtype=z, absmax2.dtype=r   )r   r   s   r    r,   z_.<locals>.<lambda>  s&    PQVQ\Q\P]]mnun{n{m|} r"   )r   r0   rc   r'   r   rC   r   rm   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   s    ````       ````    r    r!   r!   }  s   , 
LL		QWWYa mmU^^U]]CN	LL	>!I 
LL	177h 
LL#; 
LLw}}55y LLEKK'?	
 	KK7==9EMM9}	
 r"   )NN)rB   )F)collections.abcr   mathr   typingr   r   _IS_TORCH_GTE_24hasattrr   r   register_kernelimpl_abstractimpldefineTensortupler!   r   intstrfloatr+   r"   r    <module>r      s
   $    
5==/*MM//Mmm33O MM//Mmm((O   ( B 34 ,0#'|| 	 
	
 
 5<<( 5<<
  5<<%,,//0 5(   "s -. $(#'	N||	N||	N ||	N ||		N
 5<<
 	N EKK 	N \\	N /	N   &$ 12C C%,, C 3C   *- 56d d%,, dU\\ d 7d   )B 45	L 	L 6	L   <>b c 674 4ell 4u|| 4 84 8)D8 8ell 8 E8
   #i ./
 $(#'=||=||= ||= EKK 	=
 5<<
 = \\= 0=   %R 01 
@||
@ 5<<u||U\\8ELLCYYZ
@ 2
@   #g ./	<||	<LL	< 	< 		<
 C=	< ;;	< \\	< 0	<   'p 23`||`LL` ` 	`
 C=` ;;` 
` 
` 4`   !] ,-	||	 #	14	EJ[[	
5<<%&	 .	   (W 34, ,u|| ,5<< ,C ,X]XcXc ,hmhtht , 5,   ,` 78`||`"\\`16`JM`V[VaVa`hmhtht` 9`   79u v 12 U\\ c eELLRWR^R^D^>_  3   ] ()>||>>.6sm>EJ\\>Y^YeYe>ru>
\\> *>"   !f ,-d||d||d SMd LL	d
 ,,d d 
d 
d .d6   * t 56& %!!||! ||! LL	!
 U\\"! %! ! ! ! ! ! ! 
! ! !  	!!" #!& 
'! 7!H   3 K >?( '3
3
||3
 ||3
 LL	3

 U\\"3
 3
 3
 3
 3
 
3
 3
 	3
 <<3
 ELL!3
 \\3
  ell#!3
" #3
$ %3
( 
)3
 @3
r"   