bi;ddlZddlmZddlZddlmZddlmcmZddl m Z ddl m Z m Z ddlmZddlmZGdd ej$j&ZGd d e ZGd d ej,eZy)N)Optional)Conv1D)BaseTunerLayercheck_adapters_to_merge) transpose) BufferDictc,eZdZedZedZy)UniqueBaseGradcV|dddddf|z|dz}|j||||S)NN)save_for_backward)ctx randlora_Arandlora_lambdarandlora_gammaouts U/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/randlora/layer.pyforwardzUniqueBaseGrad.forwards8aDj)J69NN j/>J c.|j\}}}|j|j|j|j|j|j}}}tjd|||}tjd|||}d||fS)Nzkbj,kvj,bj->kbzkbj,kvj,kb->bj) saved_tensorstodtypetorcheinsum)r grad_outputrrrgrad_randlora_lambdagrad_randlora_gammas rbackwardzUniqueBaseGrad.backward%s696G6G3 O^ MM+++ ,   {00 1   k// 0&4O %||,>>rN)__name__ __module__ __qualname__ staticmethodrr rrr r s(  ? ?rr c`eZdZdZdZdej fdZede fdZ de de fd Z d Z y ) RandLoraLayer)rr)r randlora_B base_layerc j||_i|_i|_tji|_tj i|_tj i|_d|_ d|_ d|_ g|_ d|_ |j}t|tj r|j"|j$}}nUt|t&rEt)|j*dr|j*j,n|j*j.\}}|_|_||_y)NFTds_shape)r)rscalingnn ModuleDictrandlora_dropout ParameterDictrrrr(_disable_adaptersmerged_adapterscast_input_dtype_enabledget_base_layer isinstanceLinear in_features out_featuresrhasattrweightr+shapekwargs)selfr)r=r8r9s r__init__zRandLoraLayer.__init__7s$ " b 1 "//3 ..r21504"'!)-%((* j")) ,(2(>(> @W@WK  F +.5j6G6G.T !!**ZdZkZkZqZq &K'( rreturnc,t|jSr )boolr3)r>s rmergedzRandLoraLayer.mergedYsD(())rrr(c |dkrtd|||j|<|dkDrtj|}ntj}|j j tj||it|j|j|z } | jr t| n t| dz|_ tjtj ||jd|j"|<tjtj$|jt|j|jt'|j|jz d|j(|<||z |j*|<||_||_||vrt1|j,dkr tdt3|j,j5d} t3|j.j5d} d } t'|j|jt|j|j}} | j6d| kr)t| j9d | j6d| | j6d |kr)t| j9d | j6d|d } | j6d|j|kr6t| j9d | j6d|j|| j6d |j|kr6t| j9d | j6d |j|| |j,|<| |j.|<|r|j;||j=||j?|j@y)Nrz?`r` should be a positive integer value but the value passed is )pT) requires_gradzfThe `randlora_A` and `randlora_B` buffers are empty. This should not happen. Please report this issue.z{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter was added after the first one with incompatible shapes.r(rz{} has a size of {} but {} or greater is required; this probably happened because an additional RandLora adapter with a lower rank was added after the first one; loading the adapters in reverse order may solve this.)! ValueErrorr,r.DropoutIdentityr0updater/minr8r9 is_integerint num_bases Parameterrrandnronesmaxrr-rr(lenlistvaluesr<formatreset_randlora_parameters%_move_adapter_to_device_of_base_layer set_adapteractive_adapters)r> adapter_namerr(r,randlora_alphar0 init_weightsrandlora_dropout_layerrQrandlora_A_paramrandlora_B_param error_tmplmax_dimmin_dims r update_layerzRandLoraLayer.update_layer]su 6^_`^abc c | c !%'ZZ2B%C "%'[[] " $$R]]LBX3Y%Z[(($*;*;N>N'O P$##T%5%56 7- L) &4a%7 \"%$ z )4??#a' | $DOO$:$:$<=a@ #DOO$:$:$<=a@ R  #4#3#3T5F5FGTM]M]_c_p_pIqWG%%a(72 !2!2r^s rrZz'RandLoraLayer.reset_randlora_parameterss 4//446 6 wt33LAB!!$"5"5l"CQTM`M`amMnMtMtIuEuv w w 7 w ws A=B88CN)r!r"r#adapter_layer_namesother_param_namesr.Moduler?propertyrBrCr rgrZr%rrr'r'2s]?4 299 D***L/L/ L/\wrr'c(eZdZ ddedededededededed ed d ffd Zdd ede e ed d fdZ ddZ dd e ejejffdZd ejfdZdejd ejfdZd effd ZxZS)r7rr(r^r,r_r0fan_in_fan_outis_target_conv_1d_layerr`r@Nc ttj|t j||fi| ||_||_|j||||||| | |_yr ) superr.r7r?r'rs_active_adapterrgrt) r>r)rr(r^r,r_r0rsrtr`r= __class__s rr?zLinear.__init__s_ bii')tZ:6:,+ , J>Sceqr'>$r safe_merge adapter_namesct||}|sy|D]-}||jjvs!|j}|jj }|r|jj j}||j|z }tj|jstd|d|j||j_n?|j|}|jxj |j|z c_|jj|0y)a^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`list[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. Nz1NaNs detected in the merged weights. The adapter z seems to be broken)rrrir5r;rdatacloneget_delta_weightrisfiniteallrJrr3append)r>ryrzactive_adapterr) orig_dtype orig_weights delta_weights rmergez Linear.merges0mD  + >,7;;=(OP^O__rs.:__Z-HJ%%*#'#8#8#HL%%**looj.II*$$++N;- r)rrrs runmergezLinear.unmerges{{ MM< = $&&'!+,,.J#**00J!11557N!5!5!:!:!<<#44^D !!&&,//**EE& $&&'!+rc|j|}|j|}| |j}|j}|jdk(xr(|t j k(xs|t jk(}|j|j|}|j|j|}|r@|j}|j}|j}|j}t|j|jt|j|j} } |ddd|j d| fj|} |d| d|j ddfj|} | j#d} t$j'| ||j#d}| |jk(r|| fS| j(|j(fS)a4 Performs scaling on the smallest random base (randlora_A) and returns randlora_A and randlora_B in the correct order to fit the target layers' dimensions Args: adapter (str): The name of the adapter for which the delta weight should be computed. NcpurG) start_dim)end_dim)rr(devicertyperfloat16bfloat16rrrfloatrNr9r8rUrQflattenr applyT)r>adapterrrr(r cast_to_fp32rrrfresliced_Asliced_Bupdate_Bupdate_As rget_scaled_baseszLinear.get_scaled_basess__W- __W- >&&F   {{e+c%--1G1b5TYTbTbKb ..w7::6B,,W588@ #))+J#))+J-335O+113Nt00$2B2BCSIZIZ\`\l\lEm a!14>>!18G8;<??Ghwh(8$..(8!;<??G##a#0!''/>RZZcdZe d&& &X% %zz8::%%rc|j|\}}|j|jzj}t||j}|j|}||zS)z Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. )rrrrsr-)r>rrrrM output_tensorr-s rr~zLinear.get_delta_weight4s\"227;(**xzz),,!&$*=*=> ,,w'w&&rxc |j}|jr3|jr|j|j|g|i|}n|jr|j|g|i|}n|j|g|i|}|j D]}||j jvr |j|}|j||j\}} |j| j}|j|} |tjtj|||| | zz}|j|}|S)N)r)rdisable_adaptersrCrr)r]rrir0rrrr-Flinear) r>rargsr=previous_dtyperesultrdropoutrrr-s rrzLinear.forwardEs:  {{ $T__Q888F [[$T__Q888F$T__Q888F"&"6"6 _!)=)=)B)B)DD//?%)%:%:>RSRZRZ%:%["(DD(,,~6!((188GAJ+I8"TW^"^^ _>* rc*t|}d|zS)Nz randlora.)rv__repr__)r>reprxs rrzLinear.__repr__Zsg S  r)rrrEFFT)FN)r@Nr )r!r"r#r strrPrrBr?rrWrrtuplerTensorrr~rr __classcell__)rxs@rr7r7s"%$(-!?? ?  ?  ?? ??"&?? ?,(<(X8Y.&`'5<<'"5<<*!#!!rr7)rtypingrrtorch.nnr.torch.nn.functional functionalrtransformers.pytorch_utilsrpeft.tuners.tuners_utilsrrpeft.utils.otherr _buffer_dictr autogradFunctionr r'r7r%rrrs` -L&%?U^^,,?*}wN}w@j!RYY j!r