bifddlZddlmZmZmZmZmZddlZddlZddlm Z m Z m Z m Z ddl m cmZddlZddlmZddlmZddlmZddlmZmZedd ZGd d ejj8ZGd dejj8ZGddejj<ZdedfdZ Gdde jBZ"Gdde"Z#Gdde"Z$Gddejj<Z%dZ&Gdde j8Z'Gdd e j8Z(Gd!d"e(Z)Gd#d$e(Z*Gd%d&e jBZ+Gd'd(e jBZ,Gd)d*e jBZ-y)+N)AnyOptionalTypeVarUnionoverload)Tensordevicedtypenn)HIP_ENVIRONMENT) QuantState)GlobalOptimManager)*INVERSE_LINEAR_8BIT_WEIGHTS_FORMAT_MAPPING OutlierTracerTztorch.nn.Module)boundceZdZdZ ddededeedeededed ed eed dffd Z dd Z ddZ ded efdZ xZ S)StableEmbeddinga Custom embedding layer designed to improve stability during training for NLP tasks by using 32-bit optimizer states. It is designed to reduce gradient variations that can result from quantization. This embedding layer is initialized with Xavier uniform initialization followed by layer normalization. Example: ``` # Initialize StableEmbedding layer with vocabulary size 1000, embedding dimension 300 embedding_layer = StableEmbedding(num_embeddings=1000, embedding_dim=300) # Reset embedding parameters embedding_layer.reset_parameters() # Perform a forward pass with input tensor input_tensor = torch.tensor([1, 2, 3]) output_embedding = embedding_layer(input_tensor) ``` Attributes: norm (`torch.nn.LayerNorm`): Layer normalization applied after the embedding. Methods: reset_parameters(): Reset embedding parameters using Xavier uniform initialization. forward(input: Tensor) -> Tensor: Forward pass through the stable embedding layer. Nnum_embeddings embedding_dim padding_idxmax_norm norm_typescale_grad_by_freqsparse_weightreturnc t |||||||||| | tjj || |_t jj|dddiya Args: num_embeddings (`int`): The number of unique embeddings (vocabulary size). embedding_dim (`int`): The dimensionality of the embedding. padding_idx (`Optional[int]`): Pads the output with zeros at the given index. max_norm (`Optional[float]`): Renormalizes embeddings to have a maximum L2 norm. norm_type (`float`, defaults to `2.0`): The p-norm to compute for the `max_norm` option. scale_grad_by_freq (`bool`, defaults to `False`): Scale gradient by frequency during backpropagation. sparse (`bool`, defaults to `False`): Computes dense gradients. Set to `True` to compute sparse gradients instead. _weight (`Optional[Tensor]`): Pretrained embeddings. r weight optim_bits N) super__init__torchr LayerNormnormr get_instanceregister_module_override) selfrrrrrrrrr r __class__s R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/bitsandbytes/nn/modules.pyr%zStableEmbedding.__init__0sp@            HH&&}V&D '')BB4T`bdSefctjjj|j|j yNr&r initxavier_uniform_r!_fill_padding_idx_with_zeror+s r-reset_parametersz StableEmbedding.reset_parameters_(  %%dkk2 ((*r.c|jFtj5|j|jj ddddyy#1swYyxYwNrrr&no_gradr!fill_r5s r-r4z+StableEmbedding._fill_padding_idx_with_zerojS    ' 7 D,,-33A6 7 7 ( 7 7 )AAinputc ^tj||j|j|j|j |j |j}|jtj}|j|j|jjSr0) F embeddingr!rrrrrtor&get_default_dtyper(r r+r?embs r-forwardzStableEmbedding.forwardos}kk  KK    MM NN  # # KK ffU,,./yy~  !2!233r.)NN@FFNNNrN)__name__ __module__ __qualname____doc__intrfloatboolrr%r6r4rG __classcell__r,s@r-rrs:&*$(#($(-g-g-gc] -g 5/ -g  -g!-g-g&!-g -g^+7 4V44r.rceZdZdZ ddededeedeededed ed eed ee d dffd Z ddZ ddZ ded efdZ xZS) EmbeddingzS Embedding class to store and retrieve word embeddings from their indices. Nrrrrrrrrr rc t ||||||||||  tjj |dddiyr)r$r%rr)r*) r+rrrrrrrrr r,s r-r%zEmbedding.__init__sV>            '')BB4T`bdSefr.ctjjj|j|j yr0r1r5s r-r6zEmbedding.reset_parametersr7r.c|jFtj5|j|jj ddddyy#1swYyxYwr9r:r5s r-r4z%Embedding._fill_padding_idx_with_zeror=r>r?c tj||j|j|j|j |j |j}|Sr0)rArBr!rrrrrrEs r-rGzEmbedding.forwardsHkk  KK    MM NN  # # KK  r.)NNrHFFNNrI)rJrKrLrMrNrrOrPrr r%r6r4rGrQrRs@r-rTrTs&*$(#($(#'*g*g*gc] *g 5/ *g  *g!*g*g&!*g *g *gX+7 V  r.rTc&eZdZddddddejddf deej deedeede d e d ejd ed d e ddfdZ dZ dZdZdZe d%dej dee efde d ed ddf dZdZdZd&deeeee fde fdZd&deeeee fde fdZe d'dedeeeefdeee e fde def dZed(dedee e fde defd Zed(ded!ede defd"Zfd#Zed)fd$ ZxZS)* Params4bitNFTfp4data quant_state blocksizecompress_statistics quant_type quant_storagemodule Linear4bit bnb_quantizedrc |tjd}| tsdnd}tjj |||} || _|| _|| _|| _|| _ | | _ || _ || _ | S)Nr@) r&emptyr r_make_subclassr^r_r`r]rardr\rb) clsr\ requires_gradr]r^r_r`rarbrdr+s r-__new__zParams4bit.__new__s~ <;;q>D  "1sI||**3mD"#6 $&**   r.cv|jj}|j|d<|j|d<|S)Nr\rk)__dict__copyr\rkr+states r- __getstate__zParams4bit.__getstate__s6 ""$ f !%!3!3o r.c|d|_|d|_|d|_|d|_|d|_|d|_|d|_|d|_|d |_y) Nrkr^r_r`r]r\rardrb) rkr^r_r`r]r\rardrbrps r- __setstate__zParams4bit.__setstate__sr"?3{+#()>#?  - /&M "?3"?3Ho r.ct|jt|}|j}|j|t j |d|_t j |d|_|S)Nr]r\)typerlrrrtrodeepcopyr]r\)r+memo new_instancerqs r- __deepcopy__zParams4bit.__deepcopy__sgDz))$t*5 !!#!!%(#'==}1E#F   MM%-8 r.ct|jt|}|j}|j||Sr0)rvrlrrrt)r+ryrqs r-__copy__zParams4bit.__copy__s<Dz))$t*5 !!#!!%(r.quantized_statsrkc tjj||j|}||_t j |||_|jj|_|jj|_ |jj|_ d|_ |j|_||_|j|j|j_|S)N)qs_dictr T)r&rrirCrkr from_dictr]r^nestedr_r`rdr rarb)rjr\r}rkr rbkwargsr+s r-from_prequantizedzParams4bit.from_prequantized s||**3@*%//PVW))33#'#3#3#:#: **55!!ZZ ;; "&*&6&6DKK # r.cZ|jjj|}tjj ||j |j|j|j\}}||_||_ |j||j_ d|_ |S)N)r^r_r`raT) r\ contiguousrCbnb functional quantize_4bitr^r_r`rar]rbrd)r+r ww_4bitr]s r- _quantizezParams4bit._quantize&s II " % %f -!nn:: nn $ 8 8,, ;   & ;; "&1DKK #! r.c&|jdSNcpur rCr5s r-rzParams4bit.cpu6wwew$$r.r non_blockingc<|j|d|S||SNcudar rrr+r rs r-rzParams4bit.cuda9'wwfQ]w^^FQ]w^^r.c<|j|d|S||SNxpurrrs r-rzParams4bit.xpu<'wwv~eP\w]]6P\w]]r.r+r cyr0r+r r rs r-rCz Params4bit.to? r.cyr0rr+r rs r-rCz Params4bit.toGNQr.tensorcyr0rr+rrs r-rCz Params4bit.toJDGr.c tjjj|i|\}}}}|,|jdk7r|j s|j |S|j|jj|tt|!||||j|j|j|j|j|j|j }|S)Nmetar r r)rkr]r^r_r`rard)r&_C_nn _parse_torvrdrr]rCrZr$rkr^r_r`ra) r+argsrr r rconvert_to_format new_paramr,s r-rCz Params4bit.toMs9>9O9OQU9`Y_9`6|%6  &++"7@R@R>>&) )+  ##F+" &L Q"00 ,,..$($<$<??"00"00 I r.c |i}|tjtjfvr|dt||||}t |t rt fd|DS|jjjjjjjj St||||S)Nrc 3K|]e}|jjjjjj j j gyw) r\rkr]r^r_r`rarbrdN)rkr]r^r_r`rarbrd).0chunkrjrs r- z0Params4bit.__torch_function__..nsn "&,&:&:$*$6$6"("2"2,2,F,F#)#4#4&,&:&:%}}&,&:&:   sA+A.r)r&rsplitr$__torch_function__ isinstancetuplerkr]r^r_r`rarbrd)rjfunctypesrrresultrr,s` @r-rzParams4bit.__torch_function__cs >F EKK- -!WFW/eT6JF&%( "(   "("6"6 & 2 2$..(.(B(B%00"("6"6!=="("6"6  w)$tVDDr.)FrNNF....)rN) rJrKrLr&uint8rrr rNrPstrr rlrrrtrzr| classmethoddictrrrrrr rrrrrCrrQrRs@r-rZrZs=(,,0#'$(%*[[)-#u||$j)  C=  " {{& : &  $)- llc3h  &  2 %_8E#vs*:$;<_SW_^(5fc)9#:;^RV^03-0 sF{+,eSj)*  QQ5,QDQ1QQ GGFG$GGG,%E%Er.rZrb) Embedding4bitrcclt|jddyt|ddtjd|jjddk(sJt |jt s't |j|jd|_|j|j_y)Nr]zhFP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.T)rard) getattrr!warningswarnshaperrZrar])rbs r-'fix_4bit_weight_quant_state_from_modulersv}}mT2>v}d+3 v ==  q !Q && & fmmZ 0"6==@T@Tdhi & 2 2FMMr.cteZdZdZddddej dffd ZdZfdZdejfd Z xZ S) rca This class is the base module for the 4-bit quantization algorithm presented in [QLoRA](https://arxiv.org/abs/2305.14314). QLoRA 4-bit linear layers uses blockwise k-bit quantization under the hood, with the possibility of selecting various compute datatypes such as FP4 and NF4. In order to quantize a linear layer one should first load the original fp16 / bf16 weights into the Linear4bit module, then call `quantized_module.to("cuda")` to quantize the fp16 / bf16 weights. Example: ```python import torch import torch.nn as nn import bitsandbytes as bnb from bnb.nn import Linear4bit fp16_model = nn.Sequential( nn.Linear(64, 64), nn.Linear(64, 64) ) quantized_model = nn.Sequential( Linear4bit(64, 64), Linear4bit(64, 64) ) quantized_model.load_state_dict(fp16_model.state_dict()) quantized_model = quantized_model.to(0) # Quantization happens here ``` TNr[c t |||||t|jjd|||||_||_|du|_d|_||_y)aw Initialize Linear4bit class. Args: input_features (`str`): Number of input features of the linear layer. output_features (`str`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. Frkr_r`rarbN) r$r%rZr!r\ compute_dtypecompute_type_is_setr]ra) r+input_featuresoutput_featuresbiasrr_r`rar r,s r-r%zLinear4bit.__init__sg, $G KK   3!'   +#0#< *r.ch|jtjtjfvr|j|_y|jtj k(r|jdtjfvrL|j |jdk(r,tjdtjdd|jdtjfvrN|j |jdk7r-tjdtjddyyyy)NzInput type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference.ignorez .*inference.)messagezInput type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference or training speed.z.*inference or training) r r&float32bfloat16rfloat16numelrrrfilterwarningsr+xs r-set_compute_typezLinear4bit.set_compute_types 77u}}enn5 5"#D  WW %!!dEMM%:: QWWUW[@X Y''.I!!dEMM%:: QWWUW[@X k'':ST AY:&r.c t||||t|jddZ|jjj dj D]"\}}|r|n|j||dz|z<$yy)zc save weight and bias, then fill state_dict with components of quant_state r]NT)packedzweight.)r$_save_to_state_dictrr!r]as_dictitemsdetach)r+ destinationprefix keep_varskvr,s r-rzLinear4bit._save_to_state_dicts #KC 4;; t 4 @ //77t7DJJL U1;Da!((* FY.23 U Ar.rct||ja|jj|jk7r>|jjj |j|j_|j s|j |d|_|j}|j|j |j}|jdn$|jj |j}|jj}tj||||jjj |S)NT)rr]) rrr r\rCrrrr!tr matmul_4bitr])r+r inp_dtyperr!s r-rGzLinear4bit.forwards/5 99 TYY__%?!YY^^..qww7DIIN''  ! !! $'+D $GG    )T''(Ayy(tdiill4;M;M.Nq&tAXAXY\\]fggr.) rJrKrLrMr&rr%rrrrGrQrRs@r-rcrcsEH kk#+JU( Uhhr.rccBeZdZdZdddej dffd ZxZS) LinearFP4z' Implements the FP4 data type. TNc 2t||||||d||y)Q Args: input_features (`str`): Number of input features of the linear layer. output_features (`str`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. r[Nr$r% r+rrrrr_rar r,s r-r%zLinearFP4.__init__*&          r.rJrKrLrMr&rr%rQrRs@r-rrs' kk  r.rcBeZdZdZdddej dffd ZxZS) LinearNF4a"Implements the NF4 data type. Constructs a quantization data type where each bin has equal area under a standard normal distribution N(0, 1) that is normalized into the range [-1, 1]. For more information read the paper: QLoRA: Efficient Finetuning of Quantized LLMs (https://arxiv.org/abs/2305.14314) Implementation of the NF4 data type in bitsandbytes can be found in the `create_normal_map` function in the `functional.py` file: https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236. TNc 2t||||||d||y)rnf4Nrrs r-r%zLinearNF4.__init__Grr.rrRs@r-rr;s'  kk  r.rc eZdZ ddeej deej deej fdZfdZdZddee e e e fde fd Zddee e e e fde fd Zd Ze dd edee e e fd ee ee fde def dZedd ed e ee fde defdZedd edede defdZfdZxZS) Int8Paramsr\CBSCBc|tjd}tjj|||}||_||_||_|Sr9)r&rhrrirrhas_fp16_weights)rjr\rkrrrobjs r-rlzInt8Params.__new__gsI <;;q>Dll))#t]C/ r.c |jrt| |S|jj j|t j }tjj|\}}}||_||_ ||_ |S)Nr r ) rr$rCr\rr&rrrint8_vectorwise_quantrr)r+r Brr_r,s r-rzInt8Params._quantizewsw  7:f% % II " % %V5== % I^^99!< C  r.c&|jdSrrr5s r-rzInt8Params.cpurr.r rc<|j|d|S||Srrrs r-rzInt8Params.cudarr.c<|j|d|S||Srrrs r-rzInt8Params.xpurr.c 4t|jt|tj|j||j |j tj|j|tj|j|}|S)N)r\rkrrr) rvrlrorwr\rkrrr)r+rxrys r-rzzInt8Params.__deepcopy__srDz)) Jtyy$/,,!22}}TWWd+ dhh- * r.r+r rcyr0rrs r-rCz Int8Params.torr.cyr0rrs r-rCz Int8Params.torr.rcyr0rrs r-rCz Int8Params.torr.c&tjjj|i|\}}}}|jj tj k(}|sE|C|jdk7r4|jjjdk(r|j|Stt |1||||j|j}|r?|j|_|j "| |j j||_|S)Nrrr)rkr)r&rrrr\r int8rvr rrr$rCrkrrr) r+rrr r rr is_quantizedrr,s r-rCz Int8Params.tos9>9O9OQU9`Y_9`6|%6yy%**4  2v{{f7LQUQZQZQaQaQfQfjoQo>>&) ) GJfE J M,,!22  $>>ILxx#(: $ F 3 r.)NTFNNrrr)rJrKrLrr&rrlrrrrNr rrPrrrzrrr rCrQrRs@r-rrfsq(,%)&* u||$ U\\ "  ell #  %_8E#vs*:$;<_SW_^(5fc)9#:;^RV^ 03-0 sF{+,eSj)*  QQ5,QDQ1QQ GGFG$GGGr.rcd|j|d}|y|j|dd}t|tjr|j }t|t r|tvrtd|t|t r|tvr t|}|dk7rtd|y)Nr! weight_formatrowz'Expected supported weight format - got z+Only 'row' weight format is supported, got ) getpoprr&ritemrNr ValueError) state_dictrlocal_metadatastrict missing_keysunexpected_keys error_msgsr!rs r-maybe_rearrange_weightrs ^^vhf- .F ~NNfX]#;UCM-.%**, -%-?i*iB=/RSS M3 'M=g,gB=Q F}oVWWr.c:eZdZdZdfd ZdZdedefdZxZS) Embedding8bita This class implements [LLM.int8()](https://arxiv.org/abs/2208.07339) algorithm for embedding layer Quantization API is similar to Linear8bitLt: ```python import torch import torch.nn as nn from bitsandbytes.nn import Embedding8bit fp16_module = nn.Embedding(128, 64) int8_module = Embedding8bit(128, 64) int8_module.load_state_dict(fp16_module.state_dict()) int8_module = int8_module.to(0) # Quantization happens here ``` ct||||||jjj|_t |jjdd|_y)NrFrrk)r$r%r!r\r r)r+rrr r r,s r-r%zEmbedding8bit.__init__sK vUS[[%%++  !1!1EY^_ r.ctd)Nz.Saving Embedding8bit module is not implementedNotImplementedErrorr+rrrs r-rz!Embedding8bit._save_to_state_dict!"RSSr.r?rct|jds td|jj}|jj}|j |j |jfk(sJ|j |j fk(sJtj||}tj||j|j d}||dz z}|j|jS)NrzKEmbedding layer is not quantized. Please call .cuda() or .to(device) first.rg_@) hasattrr! RuntimeErrorr\rrrrrArBviewrCr )r+r?rows row_statscompressed_outputcompressed_output_statsoutputs r-rGzEmbedding8bit.forwardst{{E*lm m{{KKOO zzd1143E3EFFFF4#6#6"8888KKt4"#++eY^^DDWDWYZ5["\"&=&EFyy$$r.)NN) rJrKrLrMr%rrrGrQrRs@r-r r s'&` T%V%%r.r cbeZdZdZddej dffd ZdefdZdZ dedefd Z xZ S) ra3 This is the base class similar to Linear4bit. It implements the 4-bit quantization algorithm presented in [QLoRA](https://arxiv.org/abs/2305.14314) for embeddings. Quantization API is similar to Linear4bit: ```python import torch import torch.nn as nn from bitsandbytes.nn import Embedding4bit fp16_module = nn.Embedding(128, 64) quantized_module = Embedding4bit(128, 64) quantized_module.load_state_dict(fp16_module.state_dict()) quantized_module = quantized_module.to(0) # Quantization happens here ``` Nr[c@t||||||jjj|_t |jjdd||||_|jj }||zdk7rtjd|d|dyy)NrFrrzEmbedding size z is not divisible by block size z#. This will lead to slow inference.) r$r%r!r\r rZr^rr) r+rrr r`rar r^r,s r-r%zEmbedding4bit.__init__s vUS[[%%++  KK   $!'   KK)) 9 $ ) MM!-0PQZP[\44  *r.r?c|j|jjjzdk(sJ|jjj t jj |j|jzdzd}t jjj|j |j|jdz|j dd}|j|j|jzdzdfk(sJ|j|jjz}|jjj}|j|j|zfk(sJt jjj|j |j||j d}|j|j|zfk(sJtj |jj}||_t j"g|j|j|_ t$jj'||}|jg|j|jk(sJ|j)|j*S)Nrrr!r?r)rr!r]r^r\r+r&rrr rrBrrabsmaxrorwSizerdequantize_4bitrCr ) r+r? w_4bit_uint8 output_4bitblocks_per_embr6 output_absmaxoutput_quant_stater0s r- _forward_with_partial_dequantizez.Embedding4bit._forward_with_partial_dequantize8sI!!DKK$;$;$E$EEJJJ{{'',,U[[9>>t?R?RUYUgUg?gkl?lnop hh))33$$T%8%8$:L:LPQ:QR4  $r1+   U[[]T5G5G%G1%La$PPPP++t{{/D/DD((//|| 3 3n DFFFF++55;;t22NC6  $    ""u{{}~'E&GGGG!]]4;;+B+BC$1!#(::.P .PT=O=O.P#Q // =OP||AAd.@.@AAAAyy$$r.ctd)Nz.Saving Embedding4bit module is not implementedr$r&s r-rz!Embedding4bit._save_to_state_dictYr'r.rct||j|jjjzdk(r|j |St jj|jj|jj}tjjj||j|jS)Nrr5)rrr!r]r^r>rrr8r\r&r rBrCr )r+r?dequantized_weights r-rGzEmbedding4bit.forward\s/5    7 7 A A AQ F88? ? ^^;;DKKrrGrQrRs@r-rrsE0kk:%f%BT V  r.rc:eZdZdejdffd ZxZS) EmbeddingFP4Nc0t||||d||y)Nr[r r`rar rr+rrr rar r,s r-r%zEmbeddingFP4.__init__k)   '  r.rJrKrLr&rr%rQrRs@r-rCrCj kk   r.rCc:eZdZdejdffd ZxZS) EmbeddingNF4Nc0t||||d||y)NrrErrFs r-r%zEmbeddingNF4.__init__~rGr.rHrRs@r-rKrK}rIr.rKc|eZdZdZ d dedeffd ZfdZfdZdZfdZ d e jfd Z xZ S) Linear8bitLtaZ This class is the base module for the [LLM.int8()](https://arxiv.org/abs/2208.07339) algorithm. To read more about it, have a look at the paper. In order to quantize a linear layer one should first load the original fp16 / bf16 weights into the Linear8bitLt module, then call `int8_module.to("cuda")` to quantize the fp16 weights. Example: ```python import torch import torch.nn as nn import bitsandbytes as bnb from bnb.nn import Linear8bitLt fp16_model = nn.Sequential( nn.Linear(64, 64), nn.Linear(64, 64) ) int8_model = nn.Sequential( Linear8bitLt(64, 64, has_fp16_weights=False), Linear8bitLt(64, 64, has_fp16_weights=False) ) int8_model.load_state_dict(fp16_model.state_dict()) int8_model = int8_model.to(0) # Quantization happens here ``` rrcVt|||||tj|_||_||j_||j_|dkDr|sd|j_t|jj|||_ |jty)ay Initialize Linear8bitLt class. Args: input_features (`int`): Number of input features of the linear layer. output_features (`int`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. Tr"N)r$r%r MatmulLtStaterqindex thresholdruse_poolrr!r\"_register_load_state_dict_pre_hookr) r+rrrrrSrRr r,s r-r%zLinear8bitLt.__init__s* $G&&(  ( &6 # s?#3"&DJJ  !1!1DTdtu  //0FGr.ct ||||d}t|j|}t|j|}||z}|dz}|jj s|@|r|n|j ||<tjdtj||<y|@|r|n|j ||<tjdtj||<yyy)Nrrr)r ) r$rrr!rqrrr&rr) r+rrrscb_nameparam_from_weightparam_from_statekey_name format_namer,s r-rz Linear8bitLt._save_to_state_dicts #KC$DKK:"4::x8xj). zz** ,=F(9L]LdLdLf H%+0<<+M K(!->-%)[[__DJJN&&s+! ,r.c|jj|j_|jj|j_d|j_d|j_yr0r!rrqrr5s r-init_8bit_statezLinear8bitLt.init_8bit_state>     r.ct||i|}tjjj |i|\}}}}||j j4|j jj||j _|j j4|j jj||j _|Sr0) r$rCr&rrrrqrr) r+rrrr r rrr,s r-rCzLinear8bitLt.tosT,V,9>9O9OQU9`Y_9`6|%6  ||*"(,,//"4"4V"< ||+#)<<#3#3#6#6v#>   r.rcr|j|j_|jj|j |j a|j j|jk7r>|j jj|j|j _tj||j|j |j}|jjs;|jj%|jj|j_|SN)rrq) trainingrq is_trainingr!rrhrr r\rCrmatmulr)r+routs r-rGzLinear8bitLt.forward&s!%  ;;>> %  " 99 TYY__%?!YY^^..qww7DIINjjDKKdiitzzJzz**tzz}}/H#zz}}DKK  r.)TTrPNN)rJrKrLrMrNr%rr]rhrCr&rrGrQrRs@r-rNrNs[F H H HDN0%,N r.rNc2eZdZdfd ZdZdZdZxZS)OutlierAwareLinearcFt|||||d|_d|_yr)r$r% outlier_dimr)r+rrrr r,s r-r%zOutlierAwareLinear.__init__8s& $G!r.ctd)NzJPlease override the `forward_with_outliers(self, x, outlier_idx)` functionr$)r+r outlier_idxs r-forward_with_outliersz(OutlierAwareLinear.forward_with_outliers=s!"noor.ctd)NzEPlease override the `quantize_weights(self, w, outlier_idx)` functionr$)r+rrvs r-quantize_weightz"OutlierAwareLinear.quantize_weight@s!"ijjr.c||jQtj}|js t d|j |j }||_|jsS|j|j |j}|j jj|d|_yy)NzTPlease use OutlierTracer.initialize(model) before using the OutlierAwareLinear layerT) rtrr)is_initializedprint get_outliersr!rryr\r`)r+rtracerrvrs r-rGzOutlierAwareLinear.forwardCs    #"//1F((*lm --dkk:K*D   $$T[[$2B2BCA KK   " "1 % $D !r.)TN)rJrKrLr%rwryrGrQrRs@r-rrrr7s" pk %r.rrc8eZdZ dfd ZdZdZxZS)SwitchBackLinearBnbc Nt |||||tj|_||_||j_||j_||j_|dkDr|sd|j_ t|jj|||_ y)NrPTr") r$r%rrQrqrRrSrmemory_efficient_backwardrTrr!r\) r+rrrrrrSrRr r,s r-r%zSwitchBackLinearBnb.__init__Ss $G&&(  ( &6 #/H , s?#3"&DJJ  !1!1DTdtu r.c|jj|j_|jj|j_d|j_d|j_yr0rgr5s r-rhz#SwitchBackLinearBnb.init_8bit_statejrir.c0|j|j_|jj|j t j|j|jjd|j|jzSrl) rmrqrnr!rrhr matmul_mixedhalfrrs r-rGzSwitchBackLinearBnb.forwardpsf!%  ;;>> %  "$++*:*:*<4tzzZ]a]f]fffr.)TTFrPNN)rJrKrLr%rhrGrQrRs@r-rrRs( "'v. gr.r).rotypingrrrrrrr&rr r r torch.nn.functionalrrA bitsandbytesrbitsandbytes.cextensionr bitsandbytes.functionalr bitsandbytes.optimrbitsandbytes.utilsrrrrTr ParameterrZrLinearrcrrrrr rrCrKrNrrrrr.r-rs\  :: ++3.1X C()g4ehh((g4TL""L^zE##zEz3E:W4X3"whwht! ! H( ( VY##YxX*,%BLL,%^aBLLaH = & = &d299dN%%6$g"))$gr.