bi*ddlZddlmZddlZddlmZddlmcmZddl m Z ddl m Z m Z ddlmZGdde ZGdd ej"eZy) N)Optional)Conv1D)BaseTunerLayercheck_adapters_to_merge) transposecveZdZdZdej fdZedefdZ dde de de d e d e d e d e fd Z dZy) VBLoRALayer)vblora_logits_Avblora_logits_Bvblora_vector_bank base_layerc @||_i|_i|_tji|_tj i|_tj i|_d|_ g|_ |j}t|tjr|j|j}}nUt|t rEt#|j$dr|j$j&n|j$j(\}}|_|_||_y)NFds_shape)r rtopknn ModuleDictvblora_dropout ParameterDictr r _disable_adaptersmerged_adaptersget_base_layer isinstanceLinear in_features out_featuresrhasattrweightrshapekwargs)selfr r rrs S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/vblora/layer.py__init__zVBLoRALayer.__init__s$  mmB/ "//3!//3"'!((* j")) ,(2(>(> @W@WK  F +.5j6G6G.T !!**ZdZkZkZqZq &K'( returnc,t|jSN)boolr)r!s r"mergedzVBLoRALayer.merged9sD(())r$ adapter_namerr num_vectors vector_lengthrinit_logits_stdc |dkrtd|d|dkrtd|d|j|zdk7rtd|jd||j|zdk7rtd|jd|||j|<||j|<|dkDrt j | } nt j} |jjt j|| it jtj||j|z|d |j|<t jtj|j|z||d |j|<||_|j#|||j%||j'|j(y) Nrz`r` z# should be a positive integer valuez`topk` z`in_features` z& must be divisible by `vector_length` z`out_features` )pT) requires_grad) ValueErrorrrrrrDropoutIdentityrupdater Parametertorchzerosr r r reset_vblora_logits%_move_adapter_to_device_of_base_layer set_adapteractive_adapters) r!r*r rrr+r,rr-vblora_dropout_layers r" update_layerzVBLoRALayer.update_layer=s 6tA3&IJK K 19wtf,OPQ Q   m +q 0~d.>.>-??efsetuv v   } , 1!$"3"3!44Z[hZij  !|"& , C #%::#? #%;;=  ""2==,@T1U#VW-/\\ KK4++}r9r$r"r r sV2994***!$!%'/'/  '/  '/  '/'/'/'/RXr$r cNeZdZ ddedededededededed ed d ffd Zdd edee ed d fdZ ddZ de jd e jfdZdd ee je jffdZd e jfdZde jd e jfdZxZS)rr*rr+r,rrr-fan_in_fan_outis_target_conv_1d_layerr%Nc ttj|t j||fi| | |_||_|j|||||||| | |_yr') superrrr#r rP_active_adapterr>rQ)r!r r r*rr+r,rrr-rPrQr __class__s r"r#zLinear.__init__osg bii')T:88,+  ,a{MSacr (?$r$ safe_merge adapter_namesct||}|sy|D]}||jjvs |j}|r||jj j }||j|z }tj|jstd|d||j_n.|jxj |j|z c_|jj|y)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rr r@rrdatacloneget_delta_weightr7isfiniteallr2rappend)r!rVrWactive_adapterr orig_weightss r"mergez Linear.merges0mD  + >,7;;=(OP^O__rs.:J%%*%%**d.C.CN.SS*$$++N; 5ZZ1$&&'!+r$logitsc|j|d\}}tj|d}|jd||zj dS)N)dim)rFsoftmax unsqueezesum)r!rhr r top_k_logitsindices topk_weightss r"_get_low_rank_matrixzLinear._get_low_rank_matrixsP & Db 9 gyy26 &&r*-?-HHMMbQQr$ch|j|}|j|}|jr,|djj r t d|j |j|j}|j|}|r0|j}|j}|j}|j|||j|jdd}|j|||jddjd|jd}||fS)N)rrzoFound infinity values in VB-LoRA logits. Ensure training was not resumed from a `save_only_topk_weights` model.rrj)r r trainingisinfany RuntimeErrorr todevicerrMrtreshaperr) r!adapter cast_to_fp32r r r rABs r"_get_lora_matriceszLinear._get_lora_matricess2..w7..w7 ==_T288:>>@B "44W=@@AWAWXyy! -335O-335O!3!9!9!;   % %o7I4 P X XYhYnYnopYqsu v  % %o7I4 P Yq!_ WR..q1 2 !t r$c|j|j}|j|j}|jdk(xr|tj k(}|j ||\}}t||z|j}|S)z Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. cpu) r r}dtypetyper7float16rrrP)r!rr}rrrr output_tensors r"r[zLinear.get_delta_weights~%%g.55$$W-33{{e+F0F &&w =1!!a%)<)<= r$xc |j}|jr3|jr|j|j|g|i|}n|jr|j|g|i|}n|j|g|i|}|j D]}||j jvr |j|\}}|j|j|j}|j|} |tjtj| |||z}|j|}|Sr')rdisable_adaptersr)rgr r<r r@rr|r rrmlinear) r!rargsr previous_dtyperesultr_rrdropouts r"forwardzLinear.forwards&  {{ $T__Q888F [[$T__Q888F$T__Q888F"&"6"6 G!)=)=)B)B)DD..~>1DD00@FFG--n=!((188GAJ+BA"FF  G>* r$)rwr/rDFF)FN)r%N)F)rErFrGrKrLrMr(r#rlistrargr7tensorTensorrttuplerr[r __classcell__)rUs@r"rrms #!%$(-? ?  ?  ??????"&? ?2!<!rsG -L&OX.OXdLRYY Lr$