biyRddlmZddlZddlZddlZddlmZddlmZm Z m Z ddl Z ddl m Z ddlm cmZddlmZddlmZmZdaedZdZGd d eZGd d e j4ZGd deZGdde j4eZGdde j4eZy)) annotationsN)contextmanager)AnyOptionalUnion)Function)BaseTunerLayercheck_adapters_to_mergec+Ki}|jD]Y\}}|j}|tjvrtj|||<t |tj|<[d|D]M}|j}||vr||tj|<.tjj |dOyw)a A context manager that will add each keyword argument passed to `os.environ` and remove them when exiting. Will convert the values in `kwargs` to strings and upper-case all the keys. Example: ```python >>> import os >>> from accelerate.utils import patch_environment >>> with patch_environment(FOO="bar"): ... print(os.environ["FOO"]) # prints "bar" >>> print(os.environ["FOO"]) # raises KeyError ``` N)itemsupperosenvironstrpop)kwargs existing_varskeyvalues Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/boft/layer.pypatch_environmentr&s$Mlln% Uiik "** !#CM# e* 3 %  &iik - +C0BJJsO JJNN3 % &sCCcttSddlm}tjj t } tdd5|d|d|dgd }dddatS#1swYxYw#t$rA}tjd |d tjd d}Yd}~|atSd}~wwxYw) Nr)loadgcc)CCCXXfbd_cudaz/fbd/fbd_cuda.cppz/fbd/fbd_cuda_kernel.cuT)namesourcesverbosez#Failed to load the CUDA extension: z, check if ninja is available.zHSetting boft_n_butterfly_factor to 1 to speed up the finetuning process.) _FBD_CUDAtorch.utils.cpp_extensionrrpathdirname__file__r Exceptionwarningswarn)rcurr_dirres r get_fbd_cudar+Js/wwx(H  %U 3 $:%67H:E\9]^H I     ;A3>\]^ `aI  s/ A4A(A4(A1-A44 B>=0B99B>c0eZdZdZedZedZy) FastBlockDiaga  Implements a custom autograd Function for a fast block diagonal operation using CUDA. This function is optimized for 4D tensors where the last two dimensions are equal, representing block diagonal matrices for efficient computation on CUDA devices. c`tj|d}|j||S)a| The forward method for FastBlockDiag. Computes the block diagonal operation on the input tensor using a CUDA-optimized function. This method assumes that the input is a 4D tensor where the last two dimensions are equal, which represent the blocks to be diagonalized. Parameters: ctx: A context object that can be used to stash information for backward computation. input (Tensor): The input tensor of shape (N, D, H, H), where `N` is the batch size, `D` represents one additional dimension (In BOFT, the number of BOFT blocks), and `H` is the size of the square blocks along the last two dimensions (In BOFT, the block size). Returns: Tensor: The resulting tensor after applying the block diagonal operation, will have the shape (N, DxH, DxH). r)r+forwardsave_for_backward)ctxinputoutputs rr/zFastBlockDiag.forwardos.&''.q1 e$ c\|j\}tj||d}|S)Nr) saved_tensorsr+backward)r1 grad_outputr2 grad_inputs rr7zFastBlockDiag.backwards.$$!^,,[%@C r4N)__name__ __module__ __qualname____doc__ staticmethodr/r7r4rr-r-gs/,r4r-c*eZdZdZdfd ZdZxZS)MultiplicativeDropoutLayerz? Implements the multiplicative dropout layer for BOFT. c0t|||_y)z Initializes the multiplicative dropout layer. Parameters: p (float): The probability of dropping out a block. Defaults to 0.0. N)super__init__p)selfrE __class__s rrDz#MultiplicativeDropoutLayer.__init__s r4c|jrf|jd|jdk7r td|j\}}}}tjd|dj }t |j|z}||z }tjtj||jtj||jg} | tj|jd|dd} tj||dd|j} | | |<tj||jj||dd} d| z |z| | zz}|S)a Applies multiplicative dropout to the input tensor. Parameters: x (Tensor): The input tensor of shape (N, D, H, H), where `N` is the batch size, `D` represents one additional dimension (In BOFT, the number of BOFT blocks), and `H` is the size of the square blocks along the last two dimensions (In BOFT, the block size). z4The last two dimensions of input should be the same!r)devicerK)trainingshape ValueErrortorchrandintitemintrEcatonesrMzerosrandpermvieweyerepeat) rFxNDH_n_randomnum_to_replace num_zerosmask full_mask eye_matrixs rr/z"MultiplicativeDropoutLayer.forwards? ==wwr{aggbk) !WXXJAq!Q}}Q40557H!!_NN*I99ejjI5;;W`ijiqiqKrstDq)*//1a;D Aq!Qqxx@I"&Ih 1QXX6==aAqIJY!#i*&<(> @W@WK  BII .(2(>(> @W@WK6tJ7G6HIJ J&(r4cL||jvrytjdy)NGScaling operation for BOFT not supported! Automatically set scale to 1.)scalingr'r()rFadapterscales r set_scalezBOFTLayer.set_scales $,, &  _`r4c|dk(ry|jD]4}||jjvr tjd6y)NrKractive_adaptersrlkeysr'r(rFractive_adapters r scale_layerzBOFTLayer.scale_layersH A: "22 eNT[[%5%5%77 MMc d  er4Nc|jD]4}||jjvr tjd6y)Nz?Unscaling operation for BOFT not supported! Keeping scale to 1.rrs r unscale_layerzBOFTLayer.unscale_layers>"22 ]NT[[%5%5%77 MM[ \  ]r4c ts d|_d}nd|_|dz }|dkrtd|dzd|dkDr t|}nt j }|j jt j||i|dk(r|dk7r|j|zdk7rtd |jd |d |dk7rV|ttj|kDrtd |dzd |d |d|zzdk7rtd|d|dzd t|j|z}n|dk7r|dk(r|j|zdk7rtd |jd|d |dk7rq|j|d|zzkr"td|jd|dzd|d |j|d|zzzdk7r"td|jd|dzd|d t|j|z}n td|dk7r.|dzdk7rtd|d|dzdk7rtd|dtj|dz|j|jf}t|dzD]Q} |j!|jt|d| zz t|dz |} |j#| } | || <S|j%d|dt j&tj(|dz||||j*|<t j&tj,t|j.d|j0|<|j3||||j4|<||j6|<|j9||j;|j<y)zf Update the linear layer with trainable BOFT weights. Override for other layer types. FrKTr-You can only specify boft_n_butterfly_factor ! to be a positive integer number.rgrEz in_features (') must be divisible by boft_block_num ()!0Invalid combination of boft_n_butterfly_factor () and boft_block_num (boft_block_num (J) must be a multiple of 2 raised to the power of boft_n_butterfly_factor (() must be divisible by boft_block_size (z$Invalid combination of in_features (), boft_n_butterfly_factor () and boft_block_size (ZSomething went wrong, please report this error: https://github.com/huggingface/peft/issues) must be an even number!boft_block_size (boft_P persistentN)r+fbd_cuda_availablerPrArsIdentityrpupdatertr|rTmathlog2rQemptyrangeblock_butterfly_permperm2matregister_buffer ParameterrWrlrVr}rmreset_boft_parametersrnro%_move_adapter_to_device_of_base_layer set_adapterr) rF adapter_namernroboft_n_butterfly_factorrp init_weightsboft_dropout_layerPipermperm_mats r update_layerzBOFTLayer.update_layers~&+D #&' #&*D ##:A"= "Q &?@WZ[@[?\\}~  # !;l!K !#     >P/Q!RS a Na$7.0A5 #D$4$4#55\]k\llno'!+*S>1J-KK$JKbefKfJgg}M~NNPQ"Q(?%?@AE$*>*:;EF]`aFaEbbde"$"2"2n"DEO  !n&9/1Q6 #D$4$4#55]^m]nnpq'!+##!=T:T'UV$>t?O?O>PPlnEHInImJJabqarrtu##!=T:T'UVZ[[$>t?O?O>PPlnEHInImJJabqarrtu!!1!1_!DENl  #a '!Q& #3N3CC\!]^^"a' #4_4EE^!_`` KK014d6F6FHXHXY Z.23 A,,  #na&A"BCZ[H[D\^uD}}T*HAaD   XqU;$&LL KK/!3^_Ve f%  L!%'LLC@Q@Q.=\*,:L) 22<@ --.r4c|dur_tjj|j|ddtjj|j|ddy||jj vrk|durYtjj |j|tjj|j|ytd|y) z, Reset the BOFT parameters. Frg皙?)meanstdg?NTz$Unknown initialization init_weights=) rsinitnormal_rlrmrzeros_ones_rP)rFrrs rrzBOFTLayer.reset_boft_parametersms 5 GGOODKK 5CSO I GGOODKK 5CSO I  4;;++- -t#t{{<89 dkk,78 #H.sort_blocksq5D!LLOM ;;q ;LLLD!,E<<4+DE4=a8J#J/ k3EEaAN LL  v0&82C%  Pr4)rr nn.ModulereturnNone)rfloatrrNrr)rK)r:r;r<r=adapter_layer_namesother_param_namesrDrrrrrrrrr?r4rrkrksJ /M!)Fae]d/LL"&%Nr4rkceZdZdZ d d fd Zd d dZd dZd dZddZdfd Z xZ S)r{z, BOFT implemented in a dense layer. c  t |tj||fi| ||_||_|j ||||||| |_yr)rCrDrkfan_in_fan_out_active_adapterris_target_conv_1d_layer) rFrrrrnrorrprrrrrGs rrDzLinear.__init__s] 46v6,+  />;RT`bn (?$r4ct||}|sy|D];}||jjvs!|j}|jj }|r|jj j}|j|\}}tj|dd}tj||j|j }tj|dd}||z}tj|jstd|d|j!j||j"j_n|j|\}}|jj j}tj|dd}tj||j|j }tj|dd}||z}|j!j||j"j_|j$j'|>y)^ Merge the active adapter weights into the base weights Args: safe_merge (`bool`, *optional*): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. NrrKz1NaNs detected in the merged weights. The adapter z seems to be broken)r rlrryweightrrcloneget_delta_weightrQrmmtoisfiniteallrP contiguousrrrwappend rF safe_merge adapter_namesrrr orig_dtype orig_weightbutterfly_oft_matrms rmergez Linear.merges0mD  + ">"@K040E0En0U-%v"'//+q!"DK"'((+e"fK"'//+q!"DK"-"6K >>+6::<(OP^O__rs3>2H2H2J2M2Mj2YDOO**/040E0En0U-%v","3"3"8"8">">"@K"'//+q!"DK"'((+e"fK"'//+q!"DK"-"6K2=2H2H2J2M2Mj2YDOO**/$$++N;= ef #ook1a@ *5V*D)H)H)T !!&$&&'!+r4c|j|}|j|}|j\}}}}|j||z||}|j |}|j||||}|j rt j|} nG|jd}tjtj|} | jd} |jj| j} tj | | j#ddd} tj | | } | d} t%d| jdD] } | | | z} | |fS) Compute the delta weight for the given adapter. Args: adapter (str): The name of the adapter for which the delta weight should be computed. rrrK)rlrmrOrYrrr-applysqueezerQ block_diagunbindrrrrMbmmpermuterrFrrlrmr]r^r_r`orth_rotate_butterflyblock_diagonal_butterflyrbutterfly_oft_mat_batchr rs rrzLinear.get_delta_weight/saW%W%\\ 1aQUAq) $ 1 1& 9 5 : :1aA F  " "'4':':;P'Q $$9$A$A!$D !','7'7F[9\'] $'?'I'I!'L $ 8 ? ?@"')),DfnnUVXY[\F]"^"'))F4K"L3A6q177:; OA 7 :=N N  O!&((r4c |j}|jr4|jr|j|j|g|i|}n|jr|j|g|i|}nnt j |j|j|}t jt|jdf|j|}|jD]}||jjvr!|j|} |j|} |j |} | j"\} } }}| j%| | z||} |j'| }|j%| | ||}| |}|j(rt*j-|}nG|j/d}t j0t j2|}|j5d}|j6j9|}|j9|}t j:||j=ddd}t j:||}|d}t?d|j"dD] }|||z} ||z}| |z}|j9|jAjBjDj}|jAjBjD}t jF|dd}|j9|}|j9|}t jH||}t jF|dd}||z}|j9|}|jjJ4|jjJj9||j_%tMjN|||jjJ}|j9|}|S)NrMrrKrr)r2rbias)(rdisable_adaptersr rrrrQrZr|rMrVrTr}rrlrrmrprOrYrrr-rrrrrrrrrrryrrrrrFlinear)rFr\argsrprevious_dtyperesult boft_rotation boft_scalerrlrmdropoutr]r^r_r`rrrrr rrrotated_weightscaled_rotated_weights rr/zLinear.forwardOss  {{ $T__Q888F [[$T__Q888F!IId&6&6qxx~^MS):):%;Q$?XfgJ"&"6"6 1!)9)9);;^4^4++N;#\\ 1aQUAq1(,(9(9&(A%(=(B(B1aA(N%(/0E(F%**/!q"9"?"?"BCWA(?(BEV(V%W!2M A #j0 = 1@T((*1166<<=A--/66;;K//+q!'>~'N$XXA.C$//J^J^_F>* r4c*t|}d|zSNzboft.rC__repr__rFreprGs rr.zLinear.__repr__g }r4)rrrFTF)rrrnrTrorTrrTrprrboolrUnion[bool, str]rr3rrFNrr3rzOptional[list[str]]rrrrz!tuple[torch.Tensor, torch.Tensor]r\ torch.Tensorr#rrrrr9rr) r:r;r<r=rDr rrr/r.rhris@rr{r{s !'(!$)-(-?? ?  ? "% ???'?"&? ?00@r4r{ceZdZdZ d d fd ZdZd d dZd dZddZddZ dfd Z xZ S)r~z- BOFT implemented in a Conv2d layer. c t |tj||||_|j ||||||yr)rCrDrkrr) rFrrrrnrorrprrrGs rrDzConv2d.__init__sF 4,+  />;RT`bn r4c ts d|_d}nd|_|dz }|dkrtd|dzd|dkDr t|}nt j }|j jt j||i|j}|j|jdz|jdz} |dk(r|dk7r| |zdk7rtd | d |d |dk7rV|ttj|kDrtd |dzd |d |d|zzdk7rtd|d|dzd t| |z}n|dk7r||dk(rw| |zdk7rtd | d|d |dk7rI| |d|zzkrtd| d|dzd|d | |d|zzzdk7rtd| d|dzd|d t| |z}n td|dk7r.|dzdk7rtd|d|dzdk7rtd|dtj |dz| | f} t#|dzD]G} |j%| t|d| zz t|dz |} |j'| } | | | <I|j)d| dt j*tj,|dz||||j.|<t j*tj0dt|j2|j4|<|j7||||j8|<||j:|<|j=||j?|j@y)zF Update the conv2d layer with trainable BOFT weights. FrKTrrrrgrz Convolutional kernel dimension (rrrrrrrrz7Invalid combination of convolutional kernel dimension (rrrrrrrN)!r+rrPrArsrrprrtryr| kernel_sizerTrrrQrrrrrrrWrlrVr}rmrrnrorrr)rFrrnrorrprrrrconv_filter_dimrrrrs rrzConv2d.update_layers~&+D #&' #&*D ##:A"= "Q &?@WZ[@[?\\}~  # !;l!K !#     >P/Q!RS((* **Z-C-CA-FFI_I_`aIbb a Na$7/14 66GGno}n~AB'!+*S>1J-KK$JKbefKfJgg}M~NNPQ"Q(?%?@AE$*>*:;EF]`aFaEbbde"/^"CDO  !n&90A5 66GGoppAACD'!+"o.=\*,:L) 22<@ --.r4cbt||}|sy|D]}||jjvs!|j}|jj }|r\|jj j}|j|\}}|j|j|j|jdz|jdz}tj|dd}tj||j!|j }tj|dd}||z}|j|j|j|jd|jd}|j#j!||j$j_nZ|j|\}}|jj j}|j|j|j|jdz|jdz}tj|dd}tj||j!|j }tj|dd}||z}|j|j|j|jd|jd}|j#j!||j$j_|j&j)|y)rNrrK)r rlrryrrrrrrYr}r|r>rQrrrrrrrwrrs rr z Conv2d.merges0mD  +' ">"@K040E0En0U-%v"-"2"2))4+;+;j>T>TUV>W+WZdZpZpqrZs+s#K#(//+q!"DK"'((+e"fK"'//+q!"DK"-"6K"-"2"2))4+;+;Z=S=STU=VXbXnXnopXq#K3>2H2H2J2M2Mj2YDOO**/040E0En0U-%v","3"3"8"8">">"@K"-"2"2))4+;+;j>T>TUV>W+WZdZpZpqrZs+s#K#(//+q!"DK"'((+e"fK"'//+q!"DK"-"6K"-"2"2))4+;+;Z=S=STU=VXbXnXnopXq#K3>2H2H2J2M2Mj2YDOO**/$$++N;O' rQrrrrrs rrzConv2d.unmergeSs{{ MM< = $&&'!+!11557N,,.J#**00J!1!1!33,0,A,A.,Q)!6(//44::< )..%%$$z'='=a'@@:CYCYZ[C\\ $ook1a@ #hh'8':':'ef #ook1a@ )QZ8 )..%%$$**1-**1-  *5 )C !!&/$&&'!+r4c|j|}|j|jdd}|j\}}}}|j ||z||}|j |}|j ||||}|j rtj|} nG|jd}tjtj|} | jd} |jj| j } tj"| | j%ddd} tj"| | } | d} t'd| jdD] } | | | z} | |fS)rrrKr)rlrmrrOrYrrr-rrrQrrrrrrMrrrrs rrzConv2d.get_delta_weightssnW%W%//15\\ 1aQUAq) $ 1 1& 9 5 : :1aA F  " "'4':':;P'Q $$9$A$A!$D !','7'7F[9\'] $'?'I'I!'L $ 8 ? ?@"')),DfnnUVXY[\F]"^"'))F4K"L3A6q177:; OA 7 :=N N  O!&((r4c  |j}|jr4|jr|j|j|g|i|}n}|jr|j|g|i|}nYt j |j|jjdz|jjdz|j|j}t jt|jdf|j|j}|jD]}||jjvr!|j|} |j |j#dd} |j$|} | j&\} } }}| j)| | z||} |j+| }|j)| | ||}| |}|j,rt.j1|}nG|j3d}t j4t j6|}|j9d}|j:j=|}|j=|}t j>||jAddd}t j>||}|d}tCd|j&dD] }|||z} ||z}| |z}|j=|jjDjFj}|jjDjF}|j)|j|j|jjdz|jjdz}t j"|dd}t jH||}t j"|dd}||z}|j)|j|j|jjd|jjd}|jK||j}|jK|jjL|j}tOjP||||jjRd|jjTd}|j=|}|S)NrrrKr)r2rrpaddingstride)+rr r rrrrQrZr|r>rMrVrTr}rrlrrmrrprOrYrrr-rrrrrrrrrrrrr_cast_input_dtyperr!conv2drDrE)rFr\r#rr$r%r&r'rrlrmr(r]r^r_r`rrrrr rrr)r*rs rr/zConv2d.forwardsA  {{ $T__Q888F [[$T__Q888F!II  4??#>#>q#AADOOD_D_`aDbbxxggM S):):%;Q$?XYX_X_`J"&"6"6 1!)9)9);;^4^4>>q!D++N;#\\ 1aQUAq1(,(9(9&(A%(=(B(B1aA(N%(/0E(F%**/!q"9"?"?"BCWA(?(BEV(V%W!2M A #j0 ; 1>T__++00667A//0055K%**!!  4??#>#>q#AADOOD_D_`aDbbK //+q!$>!!4#3#3T__5P5PQR5SUYUdUdUpUpqrUs% !&&q*?*E*EFA))$//*>*>@U@[@[\DXX,//2--a0 F>* r4c*t|}d|zSr,r-r/s rr.zConv2d.__repr__r1r4)r2rrrT)rrrrrrnrTrorTrrTrprrr4rrr5r6rr7r8r:) r:r;r<r=rDrr rrr/r.rhris@rr~r~s !'(!)-       "%   '   &j/X9rRs$# %'' #L   & &F:#H#L22jEEPGRYY GTORYY Or4