bi>,DddlmZddlZddlZddlmZddlmZm Z m Z ddl Z ddl Z ddl mZddlmZddlmZmZdd lmZmZiZ ej0ej2fej4ej6fej8ej:fej<ej>fej0ej2fej4ej6fd Z Gd d Z!Gd dZ"GddZ#e jHddZ%e jLjOdkDrde jfdZ(nddl)Z)de jfdZ(e jTe%ddZ+dde jfdZ,ddZ-ddZ.ddZ/ddZ0ddZ1dd Z2dd!Z3d"ee e jfd#Z4d$ed%ejjfd&Z6de ed%e ejjfd'Z7Gd(d)Z8 dde jd*e e jd+e e jd,e e jd%e9e je8ff d-Z: dde jd.e e8d+e e jd*e e jd,e e jd/e;d%e jfd0ZZHed9eE:dded*ed,e e jd%efd?ZI dd@eJdAedBedCedDeKdEeKdFe;dGeKdHe e jdIeKdJeKdKeKdLeKdMeKdNe e jdOeKd%df"dPZLedQeE: dd@eJdAedBedCedHe e jdDeKdIeKdEeKdFe;dGeKdRedSe e jdTedUe e jdVedWe e jdLeKdMeKdNe e jdOeKd%df*dXZM dd@eJdAedBedCedHe e jdDeKdIeKdJeKdKeKdEeKdFe;dGeKdRedSe e jdYedZe e jdLeKdMeKd%df&d[ZNed9eE:dd\ed]edFe;d^e;fd_ZOe jfd`ZQ ddedaed,e e jfdbZR ddedaed,e e jfdcZS ddedaed,e e jfddZTde jfde jdae jd,e e jfdeZV dde jdfe jdge jd,e e jdhe e jf diZWed9eE: dde jdfe e jdge e jdje e jd%e9e je je e jff dkZXed9eE:dde jfdlZYGdmdnZZGdodpZ[GdqdrZ\dsZ]dtZ^e jfduZ` dde jdge e jdfe e jdve e jdwe e jf dxZade jdye jfdzZbdde jfd{Zc dd|e eZe jfdae jd,e e jfd}Zddd~ZedZfy))IterableN)prod)AnyOptionalUnion)Tensor) deprecated)pack_dict_to_tensorunpack_tensor_to_dict)HIP_ENVIRONMENTlib)adammomentumrmsproplionlamblarsc4eZdZdZdZdZedZddZy)GlobalPageManagerNctdNzCall get_instance() instead RuntimeErrorselfs R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/bitsandbytes/functional.py__init__zGlobalPageManager.__init__6899cg|_yN) paged_tensorsrs r initializezGlobalPageManager.initialize9s r c|j0|j||_|jj|jSr" _instance__new__r$clss r get_instancezGlobalPageManager.get_instance<6 == KK,CM MM $ $ &}}r cJ|jdddD]}t||y)N)r#prefetch_tensor)rto_cputs r prefetch_allzGlobalPageManager.prefetch_allCs,##DbD) 'A Av & 'r F) __name__ __module__ __qualname__r'rr$ classmethodr+r2r rrr3s*I:  'r rc2eZdZdZdZdZedZdZy)CUBLAS_ContextNctdrrrs rrzCUBLAS_Context.__init__Nrr ci|_yr")contextrs rr$zCUBLAS_Context.initializeQs  r c|j0|j||_|jj|jSr"r&r)s rr+zCUBLAS_Context.get_instanceTr,r c|j|jvrtjj }tjj |t jtj|j|j<tjj ||j|jSr") indexr=torchcudacurrent_device set_devicectc_void_pr get_context)rdevice prev_devices rrGzCUBLAS_Context.get_context[s} <>>: ??MM'!*bkk!((3RZZ5IJr c d}|jtjk(r+ttd|dd}t j |}nG|jtjk(r*ttd|dd}t j|}|td|t|dd}|r|rt|| t||t|t|t j|j|js |jrtjj!yy)Nc_fp32_uint8zFunction not implemented: is_managedF)rYrAfloat32getattrrrEc_floatuint8c_uint8NotImplementedErrorr/rsc_int64numelrhrB synchronize) func_namerpBvalueprefetchfunccvaluer{s relementwise_funcrs Dww%--sa {%0$7E" EKK sa {&148E" |!$>yk"JKKL%0Jh = A WQZAGGI)>?zzQZZ    r c td|d|y)Nfillr)rprrHrs rrrsVQe,r c td||dy)N_mulrr)rprrHs rrrsVQ1%r cN|rdnd}d|z}|s|dkr|sd|znd|zdz }tj|d|}d|jz }|dk(r|S|jdz}tj|d|j dg|zz||dj zS) Nr ?r)rAlinspacerrtolist)signed total_bitsadd_zerosign total_valuesvaluesgapls rcreate_linear_maprs4sDj=L:> -3q*}: 8I ^^D#| 4F   C ax LLNa ||F2AJ--/1#);fQRj>O>O>QQRRr c ddlm}|rv|jt j |ddddj }dgdz}|jt j |dddd j }nu|jt j |ddddj }dgd z}|jt j |dddd j }||z|z}t j|}|jj}||jz}|jd k(sJ|S#t$r}td|d}~wwxYw) Nr)normzZScipy is required for `create_normal_map`. Install `bitsandbytes` with the `[test]` extra.g? r.rr) scipy.statsr ImportErrorppfrArrrsortrmaxr) offsetuse_extra_valueriev1v2v3vrs rcreate_normal_maprsR$  XXennVS!4Sb9 : A A CSH xxvsA6s;<< D D F XXennVS!4Sb9 : A A CSH xxvsA6s;<< D D F R" A \\!_F [[] ! !F fjjlF <<>S  M/  h  sE E0 E++E0cP|}|}|rdnd}||z||z k(sJg}ttd||z z d||z zdD]\}} |jd| zg} tt j ddg|} d|dz z} td|zD]} | D]z}| dk7rdnd}tt|D]\}}||d|dz zzz }| dk(r |d| zz}n|d| | z dz zz}| j||si| j| |t | d|zk(sJ| j|dkr/dt | z }t|D]}| jd| jtj| }||jz}|S)Nr rr)repeatrr) enumeraterangeappendlist itertoolsproductlenrrAtensorr)r exponent_bitsprecision_bitsrephas_signevaluesivalrlstbiasevalue bit_patternrpvalrcodes rcreate_fp8_maprsAAqH q5J) )) )GEA-(*B$C"DaM\dLdFeghij3q#vF y  !Q? @C " #Dm,- & &K1A!E$T+%67 04Ah// 0{T7 *v}q'8%9 99 MM%  uf% & & v;!Z- '' ' KKMA~CKs A MM!   KKM << DDHHJD Kr cg}|dz }d||z zdz }t|D]}t|rd||z|z zdznd||z|z dzzdz}tjdd|tj}|dd|ddzdz } |d|dz |zz| zj z }|s|d|dz |zz | zj z }|d kDr{tjdd|dztj}|dd|ddzdz } |d|dz zz| zj z }|r!|d|dz |zz | zj z }|j d |j d t|d|zk(sJd t|z } t| D]}|j d |jtj|tjS) a+ Creates the dynamic quantiztion map. The dynamic data type is made up of a dynamic exponent and fraction. As the exponent increase from 0 to -7 the number of bits available for the fraction shrinks. This is a generalization of the dynamic type where a certain number of the bits and be reserved for the linear quantization region (the fraction). n determines the maximum number of exponent bits. For more details see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561] r rg?rYNr.@ rrr) rintrArr|rrrrr) rmax_exponent_bitsrdata non_sign_bitsadditional_itemsrfraction_items boundariesmeansrs rcreate_dynamic_maprs-" DNM]->>?!C $ % O !m#&77 81 <q=(+<1S8 ",q01A56%?GGII  r 1A 56:;1S8 ",q01A56%?GGII  r 1A 56:; ahhmmu, ,F KK7 8 9 iIPkQDElmlslsuvu}u}k~kQjR S   7|arRYtZMNuvu|u|~GGuHtZs[ \   MkQ tZs C2 C7 rreturncL|jjdk(rFtjtj j |jjStjtj j|jjS)Nxpu) rHrrErFrA_C_xpu_getCurrentRawStreamr@_cuda_getCurrentRawStream)rs r_get_tensor_streamrssd }}U"{{588<"++C,=,=>>> g %\\%(@"ABEEfMF/226:!"45/033F;eW^%<= F(NFF|,8$''/k*%((0%!1229'2B2N%**WW-.TX  KgDs H7H7H79H=c|j|j|j|jt |j j dt|jd}|jr|j|jj|jj|jjjt |jj j d|jjd|s|S|jDcic]#\}}t!|t"j$s!||%}}}|jDcic]#\}}t!|t"j$r!||%}}}t'||d|jz<|Scc}}wcc}}w)z returns dict of tensors and strings to use in serialization via _save_to_state_dict() param: packed -- returns dict[str, torch.Tensor] for state_dict fit for safetensors saving ztorch.)rrrrrYr[)rrrrrzquant_state.bitsandbytes__)rrrrstrrYstriptupler[rr rcloneritemrrrArr )rpackedrrrqs_packed_dictnon_tensor_dicts ras_dictzQuantState.as_dictsT //kk_**844::&   ;; NN%)[[%7%7(, (=(=(, (8(8(>(>(@$' (9(9$:$@$@$J%)[[%5%5%7  N,3==?Z41ajELL>Y!Q$ZZ,3MMO_DAq:aQVQ]Q]C^1a4__NabqNr84??JK[_s/"F5F5-"F;F;c|jj||_|jj||_|jr|jj||_|j jj||j _|j jj||j _yyr")rrrrrr)rrHs rrz QuantState.tosIILL( kknnV, ;;++..0DK!%!3!3!6!6v!>DKK #{{//226:DKK  r ct|tsytj|j|jdxr/|j |j k(xrtj|j |j dxr|j|jk(xr|j|jk(xr|j|jk(xr|j%|j|j|jk(n|j|juxrI|j%|j|j|jk(S|j|juS)NFgư>)atol) rrrAallcloserr[rrYrrrr)rothers r__eq__zQuantState.__eq__"s&%, NN4;; 4 @  ekk) tyy%**4@  ekk) %//1   5#3#33  ;;*u||/G u||+[[ELL0 ;;*u||/G u||+ [[ELL0 r )NNNNNNNr3)r4r5r6__doc__valid_quant_typesr rrrr7dictrrrArHrr!rr').0xs00rrrsa&8IJ1N1#.J M$ )*(0S#X0 000d@; kKs Arrrrnc|;dtvr*tj|jtd<td}tj j jj||j|j|\}}|r]|j}||z}t||d\} } t| |j|jd||j|| } n4t||j|jd||j} ||j|n|}| |j| j| _ || fS)aWQuantize a tensor in blocks of values. The input tensor is quantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is calculated for scaling the non-linear quantization. Args: A (`torch.Tensor`): The input tensor. Supports `float16`, `bfloat16`, or `float32` datatypes. code (`torch.Tensor`, *optional*): A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type. For more details, see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561]. absmax (`torch.Tensor`, *optional*): A tensor to use to store the absmax values. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 4096. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. nested (`bool`, *optional*): Whether to additionally quantize the absmax values. Defaults to False. Raises: ValueError: Raised when the input data type is not supported. Returns: `Tuple[torch.Tensor, QuantState]`: A tuple containing the quantization results. - `torch.Tensor`: The quantized tensor. - [`QuantState`]: The state object used to undo the quantization. dynamicF)rrT)copy)rrrrYrrrrrrY) name2qmaprrrHrAops bitsandbytesquantize_blockwisedefaultmeanrrYcopy_r) rprrrnrr_out_absmaxrqabsmaxrrs rr4r4:s:F | I %#5#7#:#:188#DIi #II**==EE  MD' 6,W RWX -''  !dggahhTg6R^gopovovw !_#))D/$C#\\+*<*<=   r rrcv||J|=|;dtvr*tj|jtd<td}|t |||t j }|j}|jr\t|j|j}||jz }|jt j k7r|j}|rt jjjj!|||j"j|j|j$|j||St jjjj'|||j"j|j|j$|jS)aDequantize a tensor in blocks of values. The input tensor is dequantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is used for scaling the non-linear dequantization. Args: A (`torch.Tensor`): The quantized input tensor. quant_state ([`QuantState`], *optional*): The quantization state as returned by [`quantize_blockwise`]. Required if `absmax` is not provided. absmax (`torch.Tensor`, *optional*): A tensor containing the scaling values. Required if `quant_state` is not provided and ignored otherwise. code (`torch.Tensor`, *optional*): A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type. For more details, see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561]. Ignored when `quant_state` is provided. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 4096. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. Ignored when `quant_state` is provided. Raises: ValueError: Raised when the input data type is not supported. Returns: `torch.Tensor`: The dequantized tensor. The datatype is indicated by `quant_state.dtype` and defaults to `torch.float32`. r.r0rn)r1rrrHrrAr|rrdequantize_blockwiserrrYrr2r3rnrrr5)rprrrrnrrs rr=r=swR  "f&88 8 | + I %#5#7#:#:188#DIi # TYV[VcVcd   F%k&8&8+:L:LM+$$$ <<5== (\\^F  3377       )  ! !    8  99 ! ! 6 6 > > AHH%  r cd|d}d}|dk(r gd}n4|dk(rgd}n*|dk(rgd}n |dk(r|d k(r gd ddd }n td |td |dtj||}|j|j j |j dk(sJ|S)NrBr)rg 6GgfgTFٿgI4ҿg০ǿg Org__?g`\?g?g@g?g4?g`?g`v"?rr)rg?g @g(@g@g@rg@rgg g(ggggint4)rr rrr.iiaf4@)rg|8geg:Kڞ׿gH2퓊cпg}Yu-ÿgQ #(DrgF?g`_?g 0E?gL_߹E?gƶ=?ga@?gкv -?rr.z94-bit AbnormalFloats currently only support blocksize 64.z Typename z not supportedrH)rrArdiv_absrr)typenamerHrrs r get_4bit_typerPs ~ D5  $ U l V G U  ?"d#D&&&ab b |!IhZ~"FGG <<V ,DIIdhhjnn ::<2   Kr c <| tsdnd}t|||||d|SNrJrr quantize_4bitrprrnrcompress_statistics quant_storages r quantize_fp4rY ,-B3 FC4GP] ^^r c <| tsdnd}t|||||d|SNrJrSrrTrVs r quantize_nf4r]-rZr rc | tsdnd}|j}tjjj j ||||\}} t||j} |rB| j} t| | z d\} } ~ t| ||j|| || | }nt| ||j|| |}||j|n|}| |j|j|_||fS)aQuantize tensor A in blocks of 4-bit values. Quantizes tensor A by dividing it into blocks which are independently quantized. Args: A (`torch.Tensor`): The input tensor. Supports `float16`, `bfloat16`, or `float32` datatypes. absmax (`torch.Tensor`, *optional*): A tensor to use to store the absmax values. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 128 on ROCm and 64 otherwise. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. compress_statistics (`bool`, *optional*): Whether to additionally quantize the absmax values. Defaults to False. quant_type (`str`, *optional*): The data type to use: `nf4` or `fp4`. Defaults to `fp4`. quant_storage (`torch.dtype`, *optional*): The dtype of the tensor used to store the result. Defaults to `torch.uint8`. Raises: ValueError: Raised when the input data type is not supported. Returns: Tuple[`torch.Tensor`, `QuantState`]: A tuple containing the quantization results. - `torch.Tensor`: The quantized tensor with packed 4-bit values. - [`QuantState`]: The state object used to undo the quantization. rJrSrKr)r)rr[rYrrrrr)rr[rYrrr)r r[rAr2r3rUr5rPrHr6r4rrYr7r)rprrnrrWrrX input_shaper8r9rrr:rstates rrUrU:sB-B3 ''KII**88@@  MD' AHH 5D,Wv-=M ''!  ''!  !_#))D/$C||ELL1 :r c:| tsdnd}t|||||dSrRr dequantize_4bitrprrrnrs rdequantize_fp4re(-B3 1k63 5 IIr c:| tsdnd}t|||||dSr\rbrds rdequantize_nf4rhrfr c .| tsdnd}|+||Jt||j|j||}n |j}|j r\t |j|j}||jz }|jtjk7r|j}|ctjjjj|||j |j"|j|j|n`tjjjj%|||j |j"|j|j}|jddk(r|j'S|S)aDequantizes a packed 4-bit quantized tensor. The input tensor is dequantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is used for scaling the non-linear dequantization. Args: A (`torch.Tensor`): The quantized input tensor. quant_state ([`QuantState`], *optional*): The quantization state as returned by [`quantize_4bit`]. Required if `absmax` is not provided. absmax (`torch.Tensor`, *optional*): A tensor containing the scaling values. Required if `quant_state` is not provided and ignored otherwise. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 128 on ROCm and 64 otherwise. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. quant_type (`str`, *optional*): The data type to use: `nf4` or `fp4`. Defaults to `fp4`. Raises: ValueError: Raised when the input data type or blocksize is not supported. Returns: `torch.Tensor`: The dequantized tensor. rJrS)rr[rYrrr<rr )r rr[rYrrr=rrrAr|rr2r3rcrnrrr5r1)rprrrnrrs rrcrcsbF-B3 !co55 ))))!  ##%k&8&8+:L:LM+$$$ <<5== (\\^F  ..22 v{,,k.D.DkFWFWYdYjYjps 3 ii$$44<<   ! !  " "         wwqzQuuw Jr zDThis function is deprecated and will be removed in a future release.)categoryc|Vdtvr*tj|jtd<td}|j|j}t j |j }|jtjk7r|j}||z }t|||}|||ffS)Nr.) r1rrrHrArNrrYr|rquantize_no_absmax)rprrnrinps rquantizerns  | I %#5#7#:#:188#DIi #wwqxx  YYq\   F ||u}}$ f*C S$ ,C  r r`c||J|X|Vdtvr*tj|jtd<td}|j|j}|||f}t ||d|}||dzS)Nr.r r)r1rrrHdequantize_no_absmax)rpr`rrrns r dequantizerqs   22 2 | I %#5#7#:#:188#DIi #wwqxx  } q%(C 0C q>r c Xt|5|%tj|tj}t ||gt j t|t|t|tj|jddd|S#1swY|SxYw)a Quantizes input tensor to 8-bit. Quantizes the 32-bit input tensor `A` to the 8-bit output tensor `out` using the quantization map `code`. Parameters ---------- A : torch.Tensor The input tensor. code : torch.Tensor The quantization map. out : torch.Tensor, optional The output tensor. Needs to be of type byte. Returns ------- torch.Tensor: Quantized 8-bit tensor. Nr) rTrA zeros_likerrr cquantizersrErbr)rprrns rrlrls,  T ;""1EKK8C1c( gdmWQZrxx ?RS T J T Js B BB)c rt|5|%tj|tj}t |||gt |}t jt|t|t|tj|j|ddd|S#1swY|SxYw)a Dequantizes the 8-bit tensor to 32-bit. Dequantizes the 8-bit tensor `A` to the 32-bit tensor `out` via the quantization map `code`. Parameters ---------- A : torch.Tensor The 8-bit input tensor. code : torch.Tensor The quantization map. out : torch.Tensor The 32-bit output tensor. Returns ------- torch.Tensor: 32-bit output tensor. Nr) rTrArsr|rrr cdequantizersrErbr)rprrnstreams rrprp8s,  ^ ;""1EMM:C4C.!#A&  wqz73<!'')ATV\] ^ J^ Js BB,,B6optimizer_namegrstate1beta1epssteplrrbeta2beta3alpha weight_decay gnorm_scale unorm_vec max_unormcd}|dkDr-tj|jj}t |||||gtj j j|||||||||| | | || ||| |y)az Performs an inplace optimizer update with one or two optimizer states. Universal optimizer update for 32-bit state and 32/16-bit gradients/weights. Parameters ---------- optimizer_name : str The name of the optimizer: {adam}. g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Optimizer state 1. beta1 : float Optimizer beta1. eps : float Optimizer epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. state2 : torch.Tensor Optimizer state 2. beta2 : float Optimizer beta2. beta3 : float Optimizer beta3. alpha : float Optimizer alpha. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. skip_zeros : bool Whether to skip zero-valued gradients or not (default: False). rN)rArrrrr2r3optimizer_update_32bit)rxryrrzr{r|r}r~rrrrrrrr skip_zeros param_norms rrrXs|J3ZZ /  q!VVY/0 II11         %r zyThis function is deprecated and will be removed in a future release. Please use optimizer_update_8bit_blockwise instead. qmap1qmap2max1max2new_max1new_max2chd}|dkDr-tj|jj}t |5t |||||| | | | ||g |j tjk(rt|j tjk(rVt|dt|t|t|t|t|tj|tj|tj|tj|tj|tj|tj| t| t| t| t| t|t|tj|tj|tj|jn|j tjk(rs|j tjk(rUt|dt|t|t|t|t|tj|tj|tj|tj|tj|tj|tj| t| t| t| t| t|t|tj|tj|tj|jn%t!d|j d|j dddy#1swYyxYw)a Performs an inplace Adam update. Universal Adam update for 32/8-bit state and 32/16-bit gradients/weights. Uses AdamW formulation if weight decay > 0.0. Parameters ---------- optimizer_name : str The name of the optimizer. Choices {adam, momentum} g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Adam state 1. state2 : torch.Tensor Adam state 2. beta1 : float Adam beta1. beta2 : float Adam beta2. eps : float Adam epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. qmap1 : torch.Tensor Quantization map for first Adam state. qmap2 : torch.Tensor Quantization map for second Adam state. max1 : torch.Tensor Max value for first Adam state update. max2 : torch.Tensor Max value for second Adam state update. new_max1 : torch.Tensor Max value for the next Adam update of the first state. new_max2 : torch.Tensor Max value for the next Adam update of the second state. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. rrr zAGradient+optimizer bit data type combination not supported: grad z , optimizer N)rArrrrTrrYr|rstr2optimizer8bitrsrEr~rurfloat16r )rxryrrzrr{rr|r}r~rrrrrrrrrrrs roptimizer_update_8bitrsZJ3ZZ /  51aE5$hX`ab 77emm #  (C n -a 0   " 9% :& 5! 5! 3 4  2  !! <( ;' 1779%+ .WW %&,,%++*E n -a 0   " 9% :& 5! 5! 3 4  2  !! <( ;' 1779%+ 0STUT[T[S\\hioiuiuhvw g555s MN((N1absmax1absmax2ct||||| | ||gtjjj |||||||||| | | | | |||||yr")rrAr2r3optimizer_update_8bit_blockwise)rxryrrzrr{rrrr|r}r~rrrrrrrs rrr:sn*q!VVUE7GDE II::           'r grad gnorm_vec percentilec 6t|5t||g|jtjk(r_t j t|t|tj|tj|jn|jtjk(r_t jt|t|tj|tj|jntd|jddddtj||dz}tj|\}}tj||}d}||kDr||z }|||fS#1swYexYw)aApplies percentile clipping grad: torch.Tensor The gradient tensor. gnorm_vec: torch.Tensor Vector of gradient norms. 100 elements expected. step: int The current optimization steps (number of past gradient norms). zGradient type z not supported!Ndr)rTrrYrAr|rcpercentile_clipping_g32rsrErurrcpercentile_clipping_g16r sqrtr) rrr}r current_gnormvalsr clip_valuers rpercentile_clippingrhsF  K4#$ :: &  ( (  " 4  4::<(   ZZ5== (  ( (  " 4  4::<(  ~djj\IJ J#K&JJy45M 9%ID#D,-JKz! =0 *k 117KKs DFFc tjjstjj|j|k7s|j|k7r%t d|jd|j|j }|j }|}|} d} t|dk(rt|dk(r|s%| s#|j d|j dk7rd} n|r%| s#|j d|j dk7rd} n|r%| r#|j d|j dk7rd} n|s| r|j d|j dk7rld} nht|dk(rt|dk(r|s%| s#|j d|j dk7rd} n%|r$| s"|j d|j dk7rd} n|r$| r"|j d|j dk7rd} n|s| r|j d|j dk7rd} nt|dk(rt|dk(r|s$| s"|j d|j dk7rd} nq|r$| s"|j d|j dk7rd} nK|r$| r"|j d|j dk7rd} n%|s#| r!|j d|j dk7rd} |a|j } | syt|dk(rjt|dk(r[| d|dk(rO| d|dk(rC|d|dk(r7|d|dk(r+d} n't|dk(rJt|dk(r<|s | s |d|df} n|r | r |d|df} n|r | s |d|df} n|s| r|d|df} nt|dk(rZt|dk(rL|s| s|d|d|df} n|r| r|d|d|df} n|r| s|d|d|df} nz|sx| rv|d|d|df} ngt|dk(rYt|dk(rK|s| s|d|d|df} n8|r| r|d|d|df} n%|r| s|d|d|df} n|s| r|d|d|df} | std |d |d |d | d  S) Nz3Expected torch.int8 input tensors A and B, but got  and Trr rFrDz?Tensor dimensions incorrect for matrix mulitiplication: A x B:  x z with transpose for A x B: r) rArBis_initializedinitrY TypeErrorr[rr ) rprrn transposed_A transposed_B expected_typesAsBtAtBcorrectsouts r check_matmulrsr :: $ $ & ww-177m#;MaggYV[\]\c\c[deff B B B BG 2w!|B1 "qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G RA#b'Q,"qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G RA#b'Q,"qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G yy3r7aio supported for tensor contractions, but dims for A x B were: r)rrAzerosint32rHrr[ batched_igemmstrider r:r+rGrrcigemmrEc_boolrurs)rprrnrrrrrnldbmrldaldcptrs rigemmr s 1c< >D {kkt5;;qxxH 177|qS\Q. 771: # aggaj(@ As+ + B BB1 eRU^ #b'Q,eRUBqE "B1 eRU^ #b'Q,eRUBqE " 2w!| 88:a=AGGAJ & L XXZ]aggaj (L qww<1 xxz!} *$ A!''!*,# xxz!} *$ A!''!*,# r7a<1A((*,QA6C W\c"gl11 AQ%C qE qEhhj|!4e RA2w!||1A2a5BqE>_`b_ccfgifjk   qE qE qEBqEMe  % % ' 3 3AHH =Cq!SkJJ  , , 1  1  1     3 3 3  Jr ct|jdk(rt|jdk(s%td|jd|jt|||||}|0t j |tj |j}|jr|jd}d}n|j}|d|jdk7r$|j}|jd}n|d|jdk(rd }|jd}n{|ddk(r$|j}|jd}nO|ddk(r$|j}|jd}n#|j}|jd}|jr|jd}d}n|j}|d|jdk7r&|j}|jd}d}nP|d|jdk(r|jd}d }n%|j}|jd}d}|jd} |jd} |jd} |jd} | } |jd|jdz}|jd|jdz}|jd|jdz}tjj|j}t|||gtj |t#j$|t#j$|t#j&| t#j&| t#j&| t)|t)|t)|t#j&|t#j&|t#j&| t#j*|t#j*|t#j*|t#j,| |S) NrDz@Expected 3-dimensional tensors for bmm, but got shapes A and B: rrr FrrT)rr[r rrArrrH is_contiguousr contiguousr:r+rGrrcbatched_igemmrErrursc_longc_uint32)rprrnrrrrsr num_batchrrrrstrideAstrideBstrideCrs rrrpso qww<1 CLA$5[\]\c\c[ddijkjqjqirstt 1c< >D {kkt5;;qxxHhhjm HHJ Q41771:  A((*Q-C qTQWWQZ L((*Q-CtqyLLNhhjm1LLNhhjmLLNhhjmhhjm HHJ Q41771:  A((*Q-C L qTQWWQZ ((*Q-CL A((*Q-C L I  A  A  A Cggaj1771:%Gggaj1771:%Gggaj1771:%G  % % ' 3 3AHH =C q!Sk  , , 1  1  1     3 3 3 ' ' ' I!$ Jr c|7tjjjj ||||Stjjjj ||S)aLPerforms an 8-bit integer matrix multiplication. A linear transformation is applied such that `out = A @ B.T`. When possible, integer tensor core hardware is utilized to accelerate the operation. Args: A (`torch.Tensor`): The first matrix operand with the data type `torch.int8`. B (`torch.Tensor`): The second matrix operand with the data type `torch.int8`. out (`torch.Tensor`, *optional*): A pre-allocated tensor used to store the result. dtype (`torch.dtype`, *optional*): The expected data type of the output. Defaults to `torch.int32`. Raises: `NotImplementedError`: The operation is not supported in the current environment. `RuntimeError`: Raised when the cannot be completed for any other reason. Returns: `torch.Tensor`: The result of the operation. )rAr2r3int8_linear_matmulrnr5)rprrnrYs rrrsS&  1155aC@ 99 ! ! 4 4 < This function is useful for training, but for inference it is advised to use [`get_row_absmax`] instead. The column-wise quantization scales are not typically needed in inference scenarios. Args: A (`torch.Tensor` with dtype `torch.float16`): Input tensor. row_stats (`torch.Tensor`, *optional*): If provided, calculation of row statistics is skipped. col_stats (`torch.Tensor`, *optional*): If provided, calculation of column statistics is skipped. nnz_block_ptr (`torch.Tensor`, *optional*): Not used. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing quantization statistics. - `torch.Tensor` with dtype `torch.float32`: The row-wise quantization statistics. - `torch.Tensor` with dtype `torch.float32`: The column-wise quantization statistics. - `torch.Tensor` with dtype `torch.bool`, *optional*: A mask indicating the locations of outliers in the input tensor. Nr.rrF)dimkeepdim)is_floating_pointrNrgr[ masked_fill_get_row_absmaxamaxr)rprrr threshold outlier_maskabsAs rget_colrow_absmaxrsD    LI-uuw||B , s?9,L   lC 0  'q)4I   a 7==?I i --r c *|jtjk(sJt|jdd}|jd}tj |ftj |j}t|gt|5tjt|t|tj|tj|tj|t!|ddd|S#1swY|SxYw)aTDetermine the quantization statistics for input matrix `A` in accordance to the `LLM.int8()` algorithm. For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). Args: A (`torch.Tensor` with dtype `torch.float16`): The input matrix. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `torch.Tensor` with dtype `torch.float32`: The absolute maximum value for each row, with outliers ignored. Nr.rX)rYrArrr[emptyr|rHrrTrcget_row_statsrsrEr~rur)rprrowscolsrs rrr@s 77emm ## #  D 772;D TG5==JI qcN     AJ I  JJy ! JJt  JJt  q !      s A/DDc heZdZdedededej dej dej f dZy) COOSparseTensorrrnnzrowidxcolidxrc|jtjk(sJ|jtjk(sJ|jtjk(sJ|j |k(sJ|j |k(sJ|j |k(sJ||_||_||_||_||_ ||_ yr") rYrArrrrrrrrr)rrrrrrrs rrzCOOSparseTensor.__init__gs||u{{***||u{{***||u}},,,||~$$$||~$$$||~$$$     r N)r4r5r6rrArrr8r rrrfsE"),6;llLQLLbgbnbnr rceZdZdZy)CSRSparseTensorc|jtjk(sJ|jtjk(sJ|jtjk(sJ|j |k(sJ|j |k(sJ|j |dzk(sJ||_||_||_||_||_ ||_ yNr ) rYrArrrrrrrowptrrr)rrrrrrrs rrzCSRSparseTensor.__init__z||u{{***||u{{***||u}},,,||~$$$||~$$$||~)))     r Nr4r5r6rr8r rrry r rceZdZdZy)CSCSparseTensorc|jtjk(sJ|jtjk(sJ|jtjk(sJ|j |k(sJ|j |k(sJ|j |dzk(sJ||_||_||_||_||_ ||_ yr) rYrArrrrrrcolptrrr)rrrrrrrs rrzCSCSparseTensor.__init__rr Nrr8r rrrrr rctj|jd\}}|jdtj|j dzftj |jj}|j|j|jd|jdt|j |j|j||j|j SNT return_countsr rXr)r@srcr)rAuniqueradd_rrrrHscatter_longrcumsum_rrrrr)cooArcountsrs rcoo2csrrs\\$++TBNFF KKN [[$))a-)T[[EWEW XF OO&++-VZZ\qOA NN1 499dii64;;PTP[P[ \\r cFtj|j\}}|j|}|j|}tj |d\}}|j dtj|jdzftj|jj}|j|j|jd|jdt|j |j|j"|||Sr)rArrrrrrrrrrHrrrrrrr)rr col2rowidxrr colvaluesrrs rcoo2cscr sjj-OC [[ $F [[ $F S=Iv NN1 [[$))a-)T[[EWEW XF OO)..* !OD NN1 499dii666 RRr ctj|ftj|}tj|ftj|}tj|f||}t||||||S)NrX)rArrr)rrrrHrYrrrs r coo_zerosr s[ [[#u{{6 BF [[#u{{6 BF [[#uV This function is useful for training, but for inference it is advised to use [`int8_vectorwise_quant`] instead. This implementation performs additional column-wise transposed calculations which are not optimized. Args: A (`torch.Tensor` with dtype `torch.float16`): The input matrix. col_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantization scales. row_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantization scales. out_col (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantized data. out_row (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantized data. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing the quantized tensor and relevant statistics. - `torch.Tensor` with dtype `torch.int8`: The row-wise quantized data. - `torch.Tensor` with dtype `torch.int8`: The column-wise quantized data. - `torch.Tensor` with dtype `torch.float32`: The row-wise quantization scales. - `torch.Tensor` with dtype `torch.float32`: The column-wise quantization scales. - `torch.Tensor` with dtype `torch.int32`, *optional*: A list of column indices which contain outlier features. zUrow_stats must be None. int8_double_quant() does not support pre-allocated row_stats.zUcol_stats must be None. int8_double_quant() does not support pre-allocated col_stats.zQout_col must be None. int8_double_quant() does not support pre-allocated out_col.zQout_row must be None. int8_double_quant() does not support pre-allocated out_row.)r)r rAr2r3int8_double_quantr5)rprrrrrs rrrssNpqqpqqlmmlmm 99 ! ! 3 3 ; ;A ; SSr statscjtjjjj ||S)aYDequantizes a tensor with dtype `torch.int8` to `torch.float32`. Args: A (`torch.Tensor` with dtype `torch.int8`): The quantized int8 tensor. stats (`torch.Tensor` with dtype `torch.float32`): The row-wise quantization statistics. Returns: `torch.Tensor` with dtype `torch.float32`: The dequantized tensor. )rAr2r3int8_vectorwise_dequantr5)rprs rrrs' 99 ! ! 9 9 A A!U KKr cjtjjjj ||S)awQuantizes a tensor with dtype `torch.float16` to `torch.int8` in accordance to the `LLM.int8()` algorithm. For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). Args: A (`torch.Tensor` with dtype `torch.float16`): The input tensor. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing the quantized tensor and relevant statistics. - `torch.Tensor` with dtype `torch.int8`: The quantized data. - `torch.Tensor` with dtype `torch.float32`: The quantization scales. - `torch.Tensor` with dtype `torch.int32`, *optional*: A list of column indices which contain outlier features. )rAr2r3int8_vectorwise_quantr5)rprs rrrs'$ 99 ! ! 7 7 ? ?9 MMr rct|ts|jr|jtj k(sJdt|j d|j d|j|jdj|jdj|j}|Et j|j|j df|j|j}|j}|j j#|k(sJ|j$j#|k(sJ|jj#|k(sJ|j&|j dk(sJ|j) }|j+|rdnd}|j d}t,j/j0}t3|j }t3|j$} t3|j} t3|} t3|} t5j6|j} t5j6|j}t5j6|j&}t5j6|j d}t5j6|}t5j6|}t9|j |j$|j||gt;j<||| | | ||||| || t5j>| |S)Nz8Tensor must be `COOSparseTensor or a PyTorch COO tensor.rr )rrrrrrrHrY) rr is_sparselayoutrA sparse_coor[_nnzindicesrrrrrHrYrrrrrrrrKr+r=rsrErurr cspmm_coor)rrrnrrrrr ptrRowidx ptrColidx ptrValuesptrBptrCcnnzcrowsAccolsAccolsBcldbcldcs rspmm_coor* s dO ,~~$++1A1A"A F A AA <<>!$((*<<>!$((*;;=   {kk499aggaj1!((!''R ((C ;;   # %% % ;;   # %% % ;;   # %% % 99 "" "((L ((*MM       , Jr c^|Otj|j|jdf|j|j j }|j}|jj|k(sJ|jj|k(sJ|j j|k(sJ|j|jdk(s J|jd|jtj|jd\}}|jdj}tj|d\}} | j} |j}|ddksJd |dd |j tj tj"fvsJt%|} t%|} t%| } t%|j} t%|j}t%|j }t%|}t%|}t%|}t'j(|j}t'j(|j}t'j(|j}t'j(|jd}t'j(|jd}t+|5t-|j|j|j |||g|j tj k(r#t/j0| | | | ||||||||||n?|j tj"k(r"t/j2| | | | ||||||||||ddd|S#1swY|SxYw) Nr rrz vs Tr) descending z)Current max count per row is 8 but found r)rArrr[rHrrYrrrrrrcumsumrrrint8rsrErurTrr cspmm_coo_very_sparse_naive_fp16 cspmm_coo_very_sparse_naive_int8)rr dequant_statsrnr_rr max_countmax_idx ptrOffset ptrMaxCount ptrMaxIdxrr r!r"r#ptrDequantStats cnnz_rowsr$r%crowsBr's rspmm_coo_very_sparser<Ps {kk499aggaj1!(($++J[J[\ ((C ;;   # %% % ;;   # %% % ;;   # %% % 99 "?tyykaggY$?? " T[[=IAv ]]1  ! ! #FFt<IwkkmG I Q<2 Z!J9UV<.XYZZ  77u}}ejj1 11 1I)$K I $I $I $I 1:D 3*I ::dhh D ZZ "F ZZ #F ZZ #F  #4;; T[[!S-PQ 77emm #  0 0  WW "  0 0 )#L JM#L Js )B/N""N,g_@r3)T)NTr")TrT)g+ew?T)TrBrr)Tr@r)NNNF)NNNNr=F)NrJ)NNNN)NNNNrr) NrrrrrNrF)rrNr)rrF)rB)NFFN)NFF)NNNr)r)NNNNr)gcollections.abcrctypesrErmathrtypingrrrnumpyrcrArtyping_extensionsr bitsandbytes.utilsr r cextensionr rr1cadam_static_8bit_grad_32cadam_static_8bit_grad_16cmomentum_static_8bit_grad_32cmomentum_static_8bit_grad_16crmsprop_static_8bit_grad_32crmsprop_static_8bit_grad_16clion_static_8bit_grad_32clion_static_8bit_grad_16rrr:rKrHFIRST_CUDA_DEVICErB device_countrTrVr|ror/rrrrrrrrrFrrsrrr4rr=rPrrYr]rUrerhrc FutureWarningrnrqrlrprrrrrrr/rrrrrrrrrrrrrr halfr rrrr*r<Cr8r rrSsw  %'' (I,   %% %%  )) )) (( (( %% %%  %% %%  )) )) +8''0**2"!ELLq1  ::q '5<<'(5<<("MM2CKu||K!6-&S&:'T43n!x 67!HPvP"++P %x %HR[[$9 %n n f$(%)"&  D ||D 5<< D U\\ "D %,,  D 5<< #$DR)-%)#'"& J ||J*%J U\\ "J 5<< J %,,  J  J \\JZOh&*"&++ _ || _ U\\ " _ %,,  _&*"&++ _ || _ U\\ " _ %,,  _&*"&++N ||N U\\ "N %,, N 5<< #$Nf)-%)"&# J || J*% J U\\ " J %,,  J } J  \\ J)-%)"&# J || J*% J U\\ " J %,,  J } J  \\ J)-%)"&# J ||J*%J U\\ "J %,,  J } J \\JZ R]jk$("&  5<<  %,,  65( () l& R]jk.2%)#'"&   E&&.) * U\\ " 5<<  %,,    l( R]jk&Xell5KW]l< R]jkF&x 7MY_lP&*(,#VV VV  V  V  V V V U\\ "V V V VVV %V !V$ %Vr ; ,(,)AA AA  A U\\ " A  A A A A A A ELL !A A 5<< AA u||$!A"#A$%A& %'A()A* +A  Aj'++ ++  + U\\ " +  + + + + + + + + ELL !++ ell #!+"#+$%+( )+\ R]jk&2f&2&2s&2PS&2l&2RGLjjQn#'  " " " %,, "P#' b b b %,, bP#' ] ] ] %,, ]@X\chcncnC%,,C5<<Chu||>TC<#'#'  |||||| %,,   5<< 8 R]jk)-(,,0 6. ||6. %6. %6.ELL) 6.  5<<x '= => 6.l6.r R]jk"ell"l"J&""] S.3ZZD)-(,&*&* 0T ||0T %0T %0Tell # 0T ell # 0Tf Lu|| LELL LNU\\N0#'@ - .@ ||@ %,, @FGT r