bih/ddlZddlmZddlZddlmZddlmcmZddl m Z ddl m Z m Z ddlmZddlmZGdd e ZGd d ej&eZy) N)Optional)Conv1D)BaseTunerLayercheck_adapters_to_merge) transpose) BufferDictcpeZdZdZdZdej fdZede fdZ d de de d e fd Z d d e fd Zy ) VeraLayer) vera_lambda_b vera_lambda_d)vera_Avera_B base_layerc N||_i|_tji|_tj i|_tj i|_d|_d|_ d|_ g|_ |j}t|tjr|j|j }}nUt|t"rEt%|j&dr|j&j(n|j&j*\}}|_|_||_y)NFds_shape)rrnn ModuleDict vera_dropout ParameterDictr r rr_disable_adaptersmerged_adaptersget_base_layer isinstanceLinear in_features out_featuresrhasattrweightrshapekwargs)selfrr"rrs Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/vera/layer.py__init__zVeraLayer.__init__"s$MM"- --b1--b1-1 ,0 "'!((* j")) ,(2(>(> @W@WK  F +.5j6G6G.T !!**ZdZkZkZqZq &K'( returnc,t|jSN)boolr)r#s r$mergedzVeraLayer.merged@sD(())r&rr d_initialcR|dkrtd|||j|<|dkDrtj|}ntj}|j j tj||itjtj|jd|j|<tjtj|d|j|<||_||_||vrt#|jdkr tdt%|jj'd} t%|j j'd} d } | j(d|j*kr3t| j-d | j(d|j*| j(d|jkr3t| j-d | j(d|jd } | j(d|j|kr6t| j-d | j(d|j|| j(d|j|kr6t| j-d | j(d|j|| |j|<| |j |<|r|j/|| |j1||j3|j4y)Nrz?`r` should be a positive integer value but the value passed is )pT) requires_gradz^The `vera_A` and `vera_B` buffers are empty. This should not happen. Please report this issue.z{} has a size of {} but {} or greater is required; this probably happened because an additional VeRA adapter was added after the first one with incompatible shapes.rrz{} has a size of {} but {} or greater is required; this probably happened because an additional VeRA adapter with a lower rank was added after the first one; loading the adapters in reverse order may solve this.r,) ValueErrorrrDropoutIdentityrupdater Parametertorchonesrr randnr rrlenlistvaluesr!rformatreset_vera_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters) r# adapter_namerrrr init_weightsr,vera_dropout_layer vera_A_param vera_B_param error_tmpls r$ update_layerzVeraLayer.update_layerDs 6^_`^abc c | # !#l!; !#     >P/Q!RS+-<< 4CTCT8Uei+j<(+-<< AVZ+[<(  v %4;;!# t  2 2 45a8L 2 2 45a8LR  !!!$t'7'77 !2!28\=O=OPQ=RTXTdTd!eff!!!$t'8'88 !2!28\=O=OPQ=RTXTeTe!fgg3  !!!$tvvl';; !2!28\=O=OPQ=RTXTZTZ[gTh!ijj!!!$tvvl';; !2!28\=O=OPQ=RTXTZTZ[gTh!ijj(4DKK %(4DKK %   & &|y & I 22<@ --.r&c^||jjvrtj5tj j |j|j|tj j |j|dddyy#1swYyxYwr)) r keysr8no_gradrinitzeros_fill_r )r#rCr,s r$r?zVeraLayer.reset_vera_parameterss 4--224 4 At11,?@FFyQt11,?@ A A 5 A As A(B##B,N)皙?)__name__ __module__ __qualname__adapter_layer_namesother_param_namesrModuler%propertyr*r+r floatrIr?r&r$r r st<,299<***@/@/ @/@/DAUAr&r ceZdZ ddedededededededed ed d ffd Zdd ede e ed d fdZ ddZ d e jfdZde jd e jfdZd effd ZxZS)rrrrCrrfan_in_fan_outis_target_conv_1d_layerrDr,r'Nc ttj|t j||fi| ||_||_|j|||||| | ||_y)Nr2) superrrr%r r[_active_adapterrIr\) r#rrrrCrrr[r\rDr,r" __class__s r$r%zLinear.__init__sa bii')46v6,+ ,<ajk'>$r& safe_merge adapter_namesct||}|sy|D]}||jjvs |j}|r||jj j }||j|z }tj|jstd|d||j_n.|jxj |j|z c_|jj|y)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rr rKrr datacloneget_delta_weightr8isfiniteallr3rappend)r#rarbactive_adapterr orig_weightss r$mergez Linear.merges0mD  + >,7;;=(OP^O__rs.:J%%*%%**d.C.CN.SS*$$++N;% 5ZZ1$&&'!+r&c|j|}|j|}|j}|j}|jdk(xr(|t j k(xs|t jk(}|j|}|j|}|r@|j}|j}|j}|j}|ddd|jfj|j} |d|jddfj|j} |jd}|jd}t|| z|| zz|j } |r| j|} | S)z Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. cpuN)dtype)rrdevicerutyper8float16bfloat16r r rXrtor unsqueezerr[) r#adapterrrrvru cast_to_fp32lambda_dlambda_bsliced_Asliced_B output_tensors r$rfzLinear.get_delta_weightsdW%W%  {{e+c%--1G1b5TYTbTbKb %%g.%%g. \\^F\\^F~~'H~~'H!/t////033HOOD-D---q0144X__E%%b)%%b)!8h#68h;N"OQUQdQde ),,5,9Mr&xc |j}|jr4|jr|j|j|g|i|}ne|jr|j|g|i|}nA|j|g|i|}|j D]}||j jvr!|j |}|j|}|j|} |j|} | ddd|jfj|j} | d|jddfj|j} |j|} |j|j}||t!j"|t!j"| || z| zz}|j|}|Sr))rudisable_adaptersr+rqrrBr rKr rrrrzrvrrFlinear)r#rargsr"previous_dtyperesultrjr~rrrrrdropouts r$forwardzLinear.forwards  {{ $T__Q888F [[$T__Q888F$T__Q888F"&"6"6 k!););)@)@)BB--n=--n=^4^4 "!%7t'7'7%7"78;;AHHE!"5D$5$5"5q"89<* r&c*t|}d|zS)Nzvera.)r^__repr__)r#repr`s r$rzLinear.__repr__sg }r&)rr.FFTrP)FN)r'N)rQrRrSr strintrXr*r%rr<rlrqr8Tensorrfrr __classcell__)r`s@r$rrs!$(-!?? ?  ?  ???"&??? ?.$<$rsJ -L&%mAmA`TRYY Tr&