bi1vddlmZmZddlmZddlmZddlm Z ddl Z ddl m Z ddlmZGddZGd d ZGd d e j$j&ZGd deZGddeZy))abc defaultdict)deepcopy)chain)OptionalN)sync_gpuceZdZdZy)MockArgsc2|D]}t||||yN)setattr)self initial_datakeys W/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/bitsandbytes/optim/optimizer.py__init__zMockArgs.__init__s" 2C D#|C0 1 2N)__name__ __module__ __qualname__rrrr r s2rr cDeZdZdZdZdZdZedZdZ d dZ dZ y) GlobalOptimManagerzK A global optimizer manager for enabling custom optimizer configs. Nctd)NzCall get_instance() instead) RuntimeErrorrs rrzGlobalOptimManager.__init__s899rcJi|_i|_d|_d|_g|_y)NF) pid2config index2config optimizeruses_config_overridemodule_weight_config_triplers r initializezGlobalOptimManager.initialize s($)!+-(rc|j0|j||_|jj|jSr ) _instance__new__r#)clss r get_instancezGlobalOptimManager.get_instance's6 == KK,CM MM $ $ &}}rct|}t|dtsd|ig}t|D]Z\}}t|dD]D\}}t ||j vs|j t ||j ||f<F\y)Nrparams)list isinstancedict enumerateidrr)rr* param_groups group_indexgroupp_indexps rregister_parametersz&GlobalOptimManager.register_parameters.sF| ,q/40%|45L"+L"9 W K'h8 W a5DOO+@DPRSTPU@VD%%{G&<= W Wrctd|_t|tjjr|g}t|tj r|g}| ||J||i}|_|D]Y}t ||jvr(|jt |j|B||jt |<[yy)a Override initial optimizer config with specific hyperparameters. The key-values of the optimizer config for the input parameters are overridden This can be both, optimizer parameters like `betas` or `lr`, or it can be 8-bit specific parameters like `optim_bits` or `percentile_clipping`. Arguments: parameters (`torch.Tensor` or `list(torch.Tensors)`): The input parameters. key (`str`): The hyperparameter to override. value: The hyperparameter value. key_value_dict (`dict`): A dictionary with multiple key-values to override. Example: ```py import torch import bitsandbytes as bnb mng = bnb.optim.GlobalOptimManager.get_instance() model = MyModel() mng.register_parameters(model.parameters()) # 1. register parameters while still on CPU model = model.cuda() # use 8-bit optimizer states for all parameters adam = bnb.optim.Adam(model.parameters(), lr=0.001, optim_bits=8) # 2. override: the parameter model.fc1.weight now uses 32-bit Adam mng.override_config(model.fc1.weight, 'optim_bits', 32) ``` TN) r!r,torchnn ParameterTensorr/rupdate)r parametersrvaluekey_value_dictr4s roverride_configz"GlobalOptimManager.override_config8sJ%)! j%(("4"4 5$J j%,, /$J ?u0!) ))!5\N  % <a5DOO+OOBqE*11.A-;DOOBqE*  < &rc@|jj|||fyr )r"append)rmodule param_nameconfigs rregister_module_overridez+GlobalOptimManager.register_module_overridems ((//V0LMr)NNN) rrr__doc__r%rr# classmethodr(r5r?rErrrrrs>I:. W3 max1max2qmap1qmap2state1state2absmax1absmax2new_max1new_max2 gnorm_vec unorm_vecN) superr initialized name2qmapis_pagedFGlobalPageManagerr(page_mngrmngnon_castable_tensor_keys fill_qmap)rr*defaults optim_bitsr[ __class__s rrzOptimizer8bit.__init__rsr *   ++88: %224 ) % ? NN  rctjd|jd<tjd|jd<y)NT)signeddynamicFudynamic)r\create_dynamic_maprZrs rrazOptimizer8bit.fill_qmaps4$%$8$8$Ey!%&%9%9%Gz"rc$t||yr )rX __setstate__)rstaterds rrkzOptimizer8bit.__setstate__s U#rct|}j}|d}t|t|k7r tdd|D}d|D}t dt ||Dr tdt t jd|Dt jd|DDcic]\}}|| } }}fd tt} |d jD] \} } | | vr| | } | | | | <| | | <"d }t ||Dcgc]\}}|||}}}j| |d y cc}}wcc}}w)a1Load an optimizer state. Arguments: state_dict (`dict`): An optimizer state (should be returned from a call to `state_dict`) to load. move_to_device (`bool`, defaults to `True`): Whether to move the optimizer's state to the device. r0zz0Optimizer8bit.load_state_dict..s71c!H+&7c38K|]}t|dywrorprrs rruz0Optimizer8bit.load_state_dict..s=1c!H+&=rvc3,K|] \}}||k7ywr r)rsp_lens_lens rruz0Optimizer8bit.load_state_dict..sN,%u~Nsz]loaded state dict contains a parameter group that doesn't match the size of optimizer's groupc3&K|] }|d ywrorrrs rruz0Optimizer8bit.load_state_dict..s#FAAhK#Fc3&K|] }|d ywrorrrs rruz0Optimizer8bit.load_state_dict..s#@AAhK#@r|ct|tjrJjr8|jtj k7r|j j}|St|trV|jD]A\}}|jvr"s|j j||<6|||<C|St|tjrt|fd|DS|S)zBMake a deep copy of value, casting all tensors to device of param.c30K|] }|ywr r)rsvcastparams rruz>Optimizer8bit.load_state_dict..cast..s"Aa4q>"As)r,r7r:is_floating_pointdtypeuint8tor-itemsr`devicecontainer_abcsIterabletype)rr=krrmove_to_devicers` rrz+Optimizer8bit.load_state_dict..casts%.**, 1K!HHU[[1E E4(!KKM2DAqD999)'(ttELL'9E!H#'q>a 2 E>#:#:;"tE{"A5"AAA rrlc|d|d<|S)Nr*r)r2 new_groups r update_groupz3Optimizer8bit.load_state_dict..update_groups"'/Ih  r)rlr0N) rr0rq ValueErroranyzipr from_iterablerr-rrk)r state_dictrgroups saved_groups param_lens saved_lensold_idr4id_maprlrrrrrtngr0rs` ` @rload_state_dictzOptimizer8bit.load_state_dictstj) ""!.1 v;#l+ +[\ \77 = = N#j*2MN No !###F#FF###@#@@  AI   2D!w'--/ DAqF{q #E1~e a   :=V\9RS2 Q+S S E<HI_ \Ts > E,Ect|jD]\}}t|dD]\}}||jvs|j|}|jD][\}}t |t j s!t|dd}|r1|j|j|j||<]y)Nr*r[F) r.r0rlrr,r7r:getattrrr) rgindexr2pindexr4valuesrrr[s rto_gpuzOptimizer8bit.to_gpus&t'8'89 BMFE&uX7 B  ?!ZZ]F & B1%a6'.q*e'DH#+3444> 1 a 0 B B Brc:|jjD]\}}}t||}|Jt|tj st|tj sJd}t|jD]\}}|rot|dD]\}} |r"t| t|k(s"||jjt| <|jjt| |jj||f<d}y)NFr*T) r_r"rr,r7r:r9r.r0r/rr) rrBattrrDpmodulefoundrr2rr4s rcheck_overrideszOptimizer8bit.check_overridess$(HH$H$H % FD&fd+G& &&gu||4 7EOO8\ \\E!*4+<+? $% %  %rcLd}|$tj5|}ddd|js'|j|j d|_d}t |j D]\}}t |dD]s\}}|j|j|}t|dk(r|j|||||j||j||||t|u|jr | t||S#1swYxYw)zPerform a single optimization step. Arguments: closure (`Callable`, *optional*, defaults to `None`): A closure that reevaluates the model and returns the loss. NTr*r)r7 enable_gradrYrrr.r0gradrlrq init_stateprefetch_state update_steprr[)rclosurelossr4rr2rrls rstepzOptimizer8bit.steps  ""$ !y !  " KKM#D  &t'8'89 MFE&uX7  66> 1 u:?OOE1ff=##A&  66:    ==Q] QK 5 ! !s DD#c|i}|d|d<|d|d<|d|d<|d|d<|jdd|d<|jdd|d<|jd d|d <|jj|d <|jj|d <|jj|d <|jj |d <|jj |d<|jj|d<||f|jjvr*|j|jj||f|S)Nbetaseps weight_decaylralphat_alphart_beta3rc min_8bit_sizepercentile_clipping block_wise max_unorm skip_zeros) getargsrcrrrrrr_rr;)rrrr2rDs r get_configzOptimizer8bit.get_config,s0.we u !&~!6~T{t ))GS1w!IIi3y!IIi3y#yy33|"&))"9"9(, (E(E$%#yy33|"ii11{#yy33| F txx44 4 MM$((//0@A B rctd)Nz(init_state method needs to be overriddenNotImplementedErrorrr2r4rrs rrzOptimizer8bit.init_state@s!"LMMrctd)Nz-The update_step method needs to be overriddenrrs rrzOptimizer8bit.update_stepCs!"QRRrcR|jr|jdkr"tj|||jSt j |j||jd}t j|d|jjj||S)Ngj@rrr) r[numelr7 zeros_likerr\ get_pagedshapefillr^ paged_tensorsrA)rr4rbuffs rget_state_bufferzOptimizer8bit.get_state_bufferFst}} C##AU188D D;;uQXXFD FF4O MM ' ' . .t 4Krc|jrZ|j|}|d}t|dd}|r6tj|dd|vrtj|dyyyy)NrOr[FrP)r[rlrr\prefetch_tensor)rr4rls1r[s rrzOptimizer8bit.prefetch_statePsk ==JJqMExBr:u5H!!%/2u$%%eHo6% r) F)Tr )rrrrrarkrrrr7no_gradrrrrfloat32rr __classcell__rds@rrIrIqsh#JH$HJT B%&U]]_##J(NS). 7rrIceZdZ ddeedeeffd ZejdZejdZ xZ S)Optimizer2Staterrc d|kstd|d|kstd|t|trW|jddjddj j d}|Dcgc] }t |}}tt|D]&}d||cxkrdkrntd |d ||d|kstd |t||||||| }t|-|||||8i}||d<| |d<| |d<| |d<| |d<| |d<t||_ ||_y ||_ ||_y cc}w)ah Base 2-state update optimizer class. Arguments: optimizer_name (`str`): The name of the optimizer. params (`torch.Tensor`): The input parameters to optimize. lr (`float`, defaults to 1e-3): The learning rate. betas (`tuple`, defaults to (0.9, 0.999)): The beta values for the optimizer. eps (`float`, defaults to 1e-8): The epsilon value for the optimizer. weight_decay (`float`, defaults to 0.0): The weight decay value for the optimizer. optim_bits (`int`, defaults to 32): The number of bits of the optimizer state. args (`object`, defaults to `None`): An object with additional arguments. min_8bit_size (`int`, defaults to 4096): The minimum number of elements of the parameter tensors for 8-bit optimization. percentile_clipping (`int`, defaults to 100): Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability. block_wise (`bool`, defaults to `True`): Whether to independently quantize each block of tensors to reduce outlier effects and improve stability. max_unorm (`float`, defaults to 0.0): The maximum value to normalize each block with. skip_zeros (`bool`, defaults to `False`): Whether to skip zero values for sparse gradients and models to ensure correct updates. is_paged (`bool`, defaults to `False`): Whether the optimizer is a paged optimizer or not. alpha (`float`, defaults to 0.0): The alpha value for the AdEMAMix optimizer. t_alpha (`Optional[int]`, defaults to `None`): Number of iterations for alpha scheduling with AdEMAMix. t_beta3 (`Optional[int]`, defaults to `None`): Number of iterations for beta scheduling with AdEMAMix. rInvalid learning rate: Invalid epsilon value: (),? Invalid beta parameter at index : Invalid weight_decay value: )rrrrrrrNrcrrrrr)rr,strreplacestripsplitfloatrangerqr-rXrr roptimizer_name)rrr*rrrrrcrrrrrrr[rrrbirbrds rrzOptimizer2State.__init__\sxby6rd;< <cz6se<= = eS !MM#r*223;AACII#NE',-!U1X-E-s5z" UA%((S( #CA3bq !STT Ul";L>JK KCl%Y`jq  :x@ <D!+D $1D !*=D& '!+D  )D !+D  DI-DI,5.s;E cp|j|||}|ddk(rtj}n*|ddk(rtj}nt d|d|j |dkrtj}|j |}d|d<|tjk(rJ|j|tj|d <|j|tj|d <np|tjk(r\|ddk(rd |jvr|j|jd j|j|jd <|jd j|j|jd <|j|tj|d <|jd |d <|j|tj|d <|jd |d<|drd}|j } | |zt| |zz} tj| ftj|j|d<tj| ftj|j|d<ntjdtj|j|d<tjdtj|j|d<tjdtj|j|d<tjdtj|j|d<|ddkr$tjd|j|d<|ddkDr%tjd|j|d <yy)!NrcrrW(Amount of optimizer bits not supported: rrrrrOrPrgrhrMrNrrrQrRrKrSrLrTrdrrrUrrrVrr7rrrrrlrrZrarrboolzeros rr2r4rrrDrrl blocksizenblockss rrzOptimizer2State.init_states7 , 2 %MME L !Q &KKE%(PQWXdQePf&gh h 779vo. .MME 1 f EMM !"33AU]]3KE(O"33AU]]3KE(O ekk !V}!DNN2NN$,0NN9,E,H,H,Ry)-1^^J-G-J-J188-Tz*"33AU[[3IE(O!^^I6E'N"33AU[[3IE(O!^^J7E'Nl# GGIy.DY,??#(;;y VWV^V^#_i #(;;y VWV^V^#_i % D ahh Wf $)KKEMMRSRZRZ$[j! % D ahh Wf $)KKEMMRSRZRZ$[j! ' (3 .!&VAHH!EE+  +  $!&T!((!CE+  %rcr|jj|_|jj|_|j|}|j}|j |||}|dxxdz cc<|d}|ddkr#t j ||d||d\} } } nd} |djtjk(rt j|j|||d|dd |d ||d |d |ddt|dd k\r|ddnd|jdd|d| |ddkDr|dnd|d|dy|djtjk(r|dst j|j|||d|d |dd |dd|d ||d |d|d|d|d|d|d|d| |ddkDr|dnd|d|d|dc|d<|d<|d|dc|d<|d<y|djtjk(r|drt j |j|||d|d |dd |ddt|dd k\r|ddnd|jdd|d ||d |d|d|d|d|d| |d yyy)!NrrrrrUrrOrrrrrPrrrrrVrrrrrMrNrKrLrSrT) gnorm_scalerVrrQrRrr)data contiguousrrlrr\rrr7roptimizer_update_32bitrrqrroptimizer_update_8bitoptimizer_update_8bit_blockwise rr2r4rrrlrrDr current_gnorm clip_valuers rrzOptimizer2State.update_stepsR""$""$ 1 vv7 f  V} ' (3 .565J5Jk",- 6 2M:{K ? EKK / $ $##hw"u t hw"&)&/&:a&?w"S 7C(~&&,[&9C&?k"T -!,/# (8_ " "ekk 1&:N # ###hhw"w"u t ggf f j!j!~&'06{0Cc0I% ,t -) 005Z/@%- ,E&M5,/4Z/@%- ,E&M5, 8_ " "ekk 1f\6J - -##hhw"w"&)&/&:a&?w"S 7C(u t ggi i ~&'!,/' 7K 1r)MbP?)?g+?:0yE>rrNrTrFFrNN) rrrrintrr7rrrrrs@rrr[s   !%!%%]-"##]-$#%]-~U]]_1D1DfU]]_\\rrceZdZ dfd Zej dZej dZxZS)Optimizer1Statecd|kstd|d|kstd|tt|D]&}d||cxkrdkrntd|d||d|kstd|t||||}t||||||8i}||d <| |d <| |d <| |d <| |d<| |d<t ||_||_y ||_||_y )a Base 1-state update optimizer class. Arguments: optimizer_name (`str`): The name of the optimizer. params (`torch.Tensor`): The input parameters to optimize. lr (`float`, defaults to 1e-3): The learning rate. betas (`tuple`, defaults to (0.9, 0.0)): The beta values for the optimizer. eps (`float`, defaults to 1e-8): The epsilon value for the optimizer. weight_decay (`float`, defaults to 0.0): The weight decay value for the optimizer. optim_bits (`int`, defaults to 32): The number of bits of the optimizer state. args (`object`, defaults to `None`): An object with additional arguments. min_8bit_size (`int`, defaults to 4096): The minimum number of elements of the parameter tensors for 8-bit optimization. percentile_clipping (`int`, defaults to 100): Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability. block_wise (`bool`, defaults to `True`): Whether to independently quantize each block of tensors to reduce outlier effects and improve stability. max_unorm (`float`, defaults to 0.0): The maximum value to normalize each block with. skip_zeros (`bool`, defaults to `False`): Whether to skip zero values for sparse gradients and models to ensure correct updates. is_paged (`bool`, defaults to `False`): Whether the optimizer is a paged optimizer or not. rrrrrrr)rrrrNrcrrrrr) rrrqr-rXrr rr)rrr*rrrrrcrrrrrrr[rrbrds rrzOptimizer1State.__init__Ps/dby6rd;< <cz6se<= =s5z" UA%((S( #CA3bq !STT Ul";L>JK K2U,O :x@ <D!+D $1D !*=D& '!+D  )D !+D  DI-DI,rc|j|||}|ddk(rtj}n*|ddk(rtj}nt d|d|j |dkrtj}|j |}d|d<|tjk(r&|j|tj|d <nk|tjk(rW|ddk(rSd |jvr|j|jd j|j|jd <|j|tj|d <|jd |d <|d r[d }|j } | |zt| |zz} tj| ftj|j|d<nftjdtj|j|d<tjdtj|j|d<|ddkr$tjd|j|d<|ddkDr%tjd|j|d<yy)NrcrrWrrrrrrOrgrMrrrrQrrKrSrrrrrUrrrVrrs rrzOptimizer1State.init_states7 , 2 %MME L !Q &KKE%(PQWXdQePf&gh h 779vo. .MME 1 f EMM !"33AU]]3KE(O ekk !V}!DNN2NN$,0NN9,E,H,H,Ry)"33AU[[3IE(O!^^I6E'Nl# GGIy.DY,??#(;;y VWV^V^#_i % D ahh Wf $)KKEMMRSRZRZ$[j! ' (3 .!&VAHH!EE+  +  $!&T!((!CE+  %rc||jj|_|jj|_|j|}|j}|j |||}|dxxdz cc<|d}|ddkr#t j ||d||d\} } } nd} |djtjk(r\t j|j|||d|dd |d ||d d|ddd d |d | |dd kDr|dnd|d|dy|djtjk(r{|dsvt j|j|||dd|dd |dd|d ||d |dd|dd|dd|d | |dd kDr|dnd|d|d|dc|d<|d<y|djtjk(rZ|drTt j|j|||dd|dd |ddd d |d ||d |dd|dd|d | |dyyy)NrrrrrUrrOrrrrrrrrVrrrrMrKrS)rrQr)rrrrlrr\rrr7rrrrr r r s rrzOptimizer1State.update_steps""$""$ 1 vv7 f  V} ' (3 .565J5Jk",- 6 2M:{K ? EKK / $ $##hw"u t w"~&&,[&9C&?k"T -!,/# (8_ " "ekk 1&:N # ###hw"w"u t gf j!~&&,[&9C&?k"T -) .05Z/@%- ,E&M5, 8_ " "ekk 1f\6J - -##hw"w"u t gi ~&'!,/' 7K 1r) r)rrrrrNrrTrFF) rrrrr7rrrrrs@rrrOsm   K-ZU]]_)D)DVU]]_ZZrr) collectionsrrrcopyr itertoolsrtypingrr7bitsandbytes.functional functionalr\bitsandbytes.utilsrr roptim OptimizerrIrrrrrr!sk ; #'22 XNXNvg7EKK))g7TqmqhUmUr