bi !ddlZddlmZmZmZddlZddlmZddlmcm Z ddl m Z ddl mZmZGddeZGddej"eZy) N)AnyOptionalUnion)Conv1D)BaseTunerLayercheck_adapters_to_mergeceZdZdZdZdej ddfdZdZe jdZ de jfd Z y) FourierFTLayer)fourierft_spectrum)fourierft_n_frequencyfourierft_scalingfourierft_random_loc_seed base_layerreturnNc 0||_i|_i|_tji|_i|_i|_d|_g|_ ||_ |j}t|tjr$|j|jc|_|_yt|t rPt#|j$dr|j$j&n|j$j(\|_|_yt+dt-|)NFds_shapezUnsupported layer type )rr r nn ParameterDictr indicesr_disable_adaptersmerged_adapterskwargsget_base_layer isinstanceLinear in_features out_featuresrhasattrweightrshape ValueErrortype)selfrrs V/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/fourierft/layer.py__init__zFourierFTLayer.__init__ s$%'"!#"$"2"22"6 )+&!&! ((* j")) ,2<2H2H*JaJa /D d/  F +.5j6G6G.T !!**ZdZkZkZqZq 0D d/6tJ7G6HIJ Jc|dkrtd|||j|jzkDr(td|d|j|jz||j|<||j|<t j |j|jzt jj|j|d||j|<t j|j||jz|j||jzgd|j|<||j|<tjt j|d|j|<|r|j!||j#||j%|j&y) NrzI`n_frequency` should be a positive integer value but the value passed is zu`n_frequency` should be less than or equal to the product of the input and output dimensions but the value passed is z and the product is ) generator)dimT) requires_grad)r!rrr rtorchrandperm Generator manual_seedrstackr r Parameterrandnr reset_fourier_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)r# adapter_name n_frequencyscaling init_weightsrandom_loc_seeds r$ update_layerzFourierFTLayer.update_layer6s ! hithuvw w ))D,=,== =++6-7KDL\L\_c_p_pLpKqs 4?""<07F&&|4%*^^    0 0 0oo'33D4R4RS_4`a&  ;& \"&+[[ \\, '4+;+; ;T\\,=WZ^ZjZj=j kqr&  \"07|,02 U[[=Uei0j -   ) ), 7 22<@ --.r&c||jjvr-tjj |j|yyN)r keysrinitzeros_)r#r6s r$r2z'FourierFTLayer.reset_fourier_parametersQs9 422779 9 GGNN422<@ A :r&c|j|}|j|j|j}t j |j |j|j}|j||dddf|dddff<tjj|j|j|z}|j|jS)N)devicer)r rtorBr+zerosrrfloatfftifft2realr dtype)r#adapterspectrumrdense_spectrum delta_weights r$get_delta_weightzFourierFTLayer.get_delta_weightVs**73,,w'**8??;T%6%68H8HQYQ`Q`a7?~~7Gwq!t}gadm34yy~6;;d>T>TU\>]] x~~..r&)__name__ __module__ __qualname__adapter_layer_namesother_param_namesrModuler%r;r+no_gradr2TensorrOr&r$r r s[1cK299K4K,/6U]]_BB/5<</r&r ceZdZ ddededededeeefdeddffd Zdd ed e e eddfd Z dd Z de jffd Zde jdedede jfdZdeffd ZxZS)FourierFTLinearr6r7r8fan_in_fan_outr9r:rNc t |tj||fi|||_||_|j |||||yr=)superr%r r[_active_adapterr;) r#rr6r7r8r[r9r:r __class__s r$r%zFourierFTLinear.__init__bsK j;F;,+ , WlO\r& safe_merge adapter_namesct||}|sy|D]}||jjvs |j}|r||jj j }||j|z }tj|jstd|d||j_n.|jxj |j|z c_|jj|y)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rr r>rrdataclonerOr+isfiniteallr!rappend)r#r`raactive_adapterr orig_weightss r$mergezFourierFTLinear.mergess0mD  + >,7;;=(OP^O__rs.:J%%*%%**d.C.CN.SS*$$++N;# rrrcrO)r#rhs r$unmergezFourierFTLinear.unmerges{{ MM< = $&&'!+!11557N!8!8!=!=!??##%,,11T5J5J>5ZZ1$&&'!+r&c"t||Sr=)r]rO)r#rKr_s r$rOz FourierFTLinear.get_delta_weightsw'00r&xargsrc|j}|jr3|jr|j|j|g|i|}n|jr|j|g|i|}n|j|g|i|}|j D]d}||j jvr |j|}|j|j}|tj||z}f|j|}|Sr=) rJdisable_adaptersrlrqrr5r r>rOrDFlinear)r#rsrtrprevious_dtyperesultrhdelta_ws r$forwardzFourierFTLinear.forwards  {{ $T__Q888F [[$T__Q888F$T__Q888F"&"6"6 7!)@)@)E)E)GG//?DD'!((1g"66  7>* r&c*t|}d|zS)Nz fourierft.)r]__repr__)r#repr_s r$r~zFourierFTLinear.__repr__sg c!!r&)igb@FFi )FN)rN)rPrQrRstrintrFboolrr%rlistrjrqr+rWrOrr|r~ __classcell__)r_s@r$rZrZ`s  $)."]] ]  ]  ]D#I&]] ]"#<#rsE'' -LC/^C/L_"bii_"r&