
    biL                       d dl mZ d dlZd dlmZ d dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ dd	lmZmZ  e
       r  G d
 dej(                  j*                  e      Z e	       r! G d dej(                  j*                  e      Zyy)    )annotationsN)Optional)is_bnb_4bit_availableis_bnb_available)check_adapters_to_merge)dequantize_bnb_weight)	transpose   )RandLoraLayerUniqueBaseGradc                       e Zd Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 fdZd
ddZddZdddZddZddZd fdZ	 xZ
S )Linear8bitLtc
           	         t         |           t        j                  | |       || _        || _        | j                  |||||||	       y N)randlora_alpharandlora_dropoutinit_weightssuper__init__r   fan_in_fan_out_active_adapterupdate_layerself
base_layeradapter_name
randlora_A
randlora_Brr   r   r   r   kwargs	__class__s              S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/randlora/bnb.pyr   zLinear8bitLt.__init__!   sX     G""44"0D#/D -!1)      c                n   t        | |      }|sy|D ]  }|| j                  j                         vr!t        j                  d       | j                  |      }| j                         j                  }| j                         j                  }|j                  |j                  |_	        t        ||      }|j                  |j                        j                  |j                        |z   }|r2t        j                  |      j!                         st#        d| d      t$        j&                  j)                  |j                  d      d|j*                        j                  |j                        | j                         _        |j-                          | j.                  j1                  |        y)  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            Nz[Merge RandLora module to 8-bit linear may get different generations due to rounding errors.1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)r   randlora_lambdakeyswarningswarnget_delta_weightget_base_layerweightstateSCBr   todtypedevicetorchisfiniteall
ValueErrorbnbnn
Int8Paramsr,   reset_gradsmerged_adaptersappend)	r   
safe_mergeadapter_namesactive_adapterrandlora_datar3   r4   outputw_datas	            r#   mergezLinear8bitLt.merge=   sw    4D-HM "/ <!)=)=)B)B)DDq !% 5 5n E,,.55++-3399$ &

EI.vu==#6#67::=;O;OPS``ennV&<&@&@&B$KNK[[no  03vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#$$++N;5<r$   c                   | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }|| j                  j                         vrPt        j                  d       | j                  |      }| j                         j                  }| j                         j                  }|j                  |j                  |_        t        ||      }|j                  |j                        j                  |j                         |z
  }t"        j$                  j'                  |j                  d      d|j(                        j                  |j                         | j                         _
        |j+                          t        | j                        dkD  ryy)	_
            This method unmerges all merged adapter layers from the base weights.
            Already unmerged. Nothing to doNr   z]Unmerge randlora module to 8-bit linear may get different generations due to rounding errors.)r4   r)   Fr*   )mergedr/   r0   lenrA   popr-   r.   r1   r2   r3   r4   r5   r   r6   r7   r8   r=   r>   r?   r,   r@   )r   rE   rF   r3   r4   rG   rH   s          r#   unmergezLinear8bitLt.unmergek   sa    ;;?@d**+a/!%!5!5!9!9!;!)=)=)B)B)DDs !% 5 5n E,,.55++-3399$ &

EI.vUC=#6#67::=;O;OPS``/2vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#) d**+a/r$   c                   | j                   |   }| j                  |   }||j                  }|j                  }|j                  dk(  xr( |t
        j                  k(  xs |t
        j                  k(  }| j                  |   j                  |      }| j                  |   j                  |      }|r@|j                         }|j                         }|j                         }|j                         }t        | j                  | j                        t        | j                  | j                        }
}	|ddd| j                   d|	f   j                  |      }|d|
d| j                   ddf   j                  |      }|j#                  d      }t%        j&                  |||      j#                  d      }|	| j                  k(  r||fS |j(                  |j(                  fS aL  
            Performs scaling on the smallest random base (randlora_A) and returns randlora_A and randlora_B in the
            correct order to fit the target layers' dimensions

            Args:
                adapter (str):
                    The name of the adapter for which the delta weight should be computed.
            Nr)   r
   )	start_dim)end_dimr   r   r8   r7   typer9   float16bfloat16r-   r6   randlora_gammafloatminout_featuresin_featuresmax	num_basesflattenr   applyTr   adapterr8   r   r   r7   cast_to_fp32r-   rY   min_dimmax_dimsliced_Asliced_Bupdate_Bupdate_As                  r#   get_scaled_baseszLinear8bitLt.get_scaled_bases   s    1J1J~#**$$E
 ";;%/gUemm5K5fuX]XfXfOfL"227;>>vFO!009<<VDN'--/
'--/
"1"7"7"9!/!5!5!7  #4#4#4d6F6FGTM^M^`d`p`pIqWG
 "!%5t~~%5xx"?@CCFKH!(7(,<dnn,<a"?@CCFKH  ''!'4H%++Ho~V^^gh^iH$***))::xzz))r$   c                    | j                  |      \  }}||z  }t        || j                        }| j                  |   }||z  S z
            Compute the delta weight for the given adapter.

            Args:
                adapter (str):
                    The name of the adapter for which the delta weight should be computed.
            rl   r	   r   scalingr   rd   rj   rk   updateoutput_tensorrp   s          r#   r1   zLinear8bitLt.get_delta_weight   sN     "&!6!6w!?Hh(F%fd.A.ABMll7+G 7**r$   c                   | j                   r4| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}na | j                  |g|i |}| j                  D ];  }|| j
                  j                         vr!| j                  ||j                        \  }}t        j                          }|r8|j                  }	|j                  }
|j                  |
k7  r|j                  |
      }| j                  |   } ||j                  |j                              }t        j                  j                  j!                  t        j                  j                  j!                  ||      |      }|r|j                  	      }| j"                  |   }|||z  z   }> |j                  |j                        S )a  
            Perform the forward pass using the RandLora adapter.

            Args:
                x (torch.Tensor): Input tensor.

            Returns:
                torch.Tensor: Output tensor after applying the RandLora adaptation.

            Note:
                This method implements the RandLora-specific forward pass. It applies the shared projections
                (randlora_A and randlora_B) along with the per-layer trainable parameters (lambda and gamma) to compute
                the adapter output.
            r8   )disable_adaptersrM   rP   r   active_adaptersr-   r.   rl   r8   r9   is_autocast_enabledr7   r6   r   r>   
functionallinearrp   r   xargsr!   resultrE   rj   rk   requires_conversionexpected_dtypecompute_dtypedropoutx_tempadapter_outputrp   s                  r#   forwardzLinear8bitLt.forward   s    $$;;LLN(<T<V<(<T<V<(<T<V<&*&:&: ?N%T-A-A-F-F-HH )-)>)>~VWV^V^)>)_&Hh.3.G.G.I*I'*)/(077m3 !] 3A"33NCG$QTT(..%9:F%*XX%8%8%?%?@S@S@Z@Z[ack@lnv%wN*)7):):>)J"ll>:G#nw&>>F-?2 99QWW%%r$   c                *    t         |          }d|z   S Nz	randlora.r   __repr__r   repr"   s     r#   r   zLinear8bitLt.__repr__       '"$C$$r$   r   r   g        FTr   ztorch.nn.Moduler   strr    intr   r   r   rZ   r   boolr   r   returnNoneFNrC   r   rD   zOptional[list[str]]r   r   r   r   Nr   z list[torch.Tensor, torch.Tensor]r   torch.Tensorr|   r   r   r   r   r   __name__
__module____qualname__r   rI   rP   rl   r1   r   r   __classcell__r"   s   @r#   r   r       s     "#&)#(!%	'	 	 	  	 $	 !	 	 	8,	<\	$<.	*`	+$0	&d	% 	%r$   r   c                       e Zd Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 fdZd
ddZddZdddZddZddZd fdZ	 xZ
S )
Linear4bitc
           	         t         |           t        j                  | |       || _        || _        | j                  |||||||	       y r   r   r   s              r#   r   zLinear4bit.__init__  sX     G""44"0D#/D -!1)  r$   c                   t        | |      }|sy|D ]O  }|| j                  j                         vr!t        j                  d       | j                  |      }| j                         j                  }|j                  }t        j                  j                  |j                  |j                        |z   }|r2t        j                  |      j!                         st#        d| d      t        j$                  j&                  |j)                  d      fddi|j)                  |j*                        | j                         _        | j,                  j/                  |       R y)r&   Nz[Merge RandLora module to 4-bit linear may get different generations due to rounding errors.r'   r(   r)   r+   F)r   r-   r.   r/   r0   r1   r2   r3   __dict__r=   ry   dequantize_4bitdataquant_stater9   r:   r;   r<   r>   
Params4bitr6   r8   rA   rB   )r   rC   rD   rE   rF   r3   r!   rH   s           r#   rI   zLinear4bit.merge   s4    4D-HM "/ <!)=)=)B)B)DDq !% 5 5n E,,.5577VEWEWX[hhennV&<&@&@&B$KNK[[no  03vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, $$++N;+<r$   c                   | j                   st        j                  d       yt        | j                        dkD  r2| j                  j                         }|| j                  j                         vrPt        j                  d       | j                  |      }| j                         j                  }|j                  }t        j                  j                  |j                  |j                         |z
  }t        j"                  j$                  |j'                  d      fddi|j'                  |j(                        | j                         _
        t        | j                        dkD  r1yy)rK   rL   Nr   z]Unmerge RandLora module to 4-bit linear may get different generations due to rounding errors.r)   r+   F)rM   r/   r0   rN   rA   rO   r-   r.   r1   r2   r3   r   r=   ry   r   r   r   r>   r   r6   r8   )r   rE   rF   r3   r!   rH   s         r#   rP   zLinear4bit.unmergeI  s    ;;?@d**+a/!%!5!5!9!9!;!)=)=)B)B)DDs !% 5 5n E,,.5577VEWEWX[hh/2vv/@/@5AQ/qaf/qjp/q/t/tMM0##%, d**+a/r$   c                   | j                   |   }| j                  |   }||j                  }|j                  }|j                  dk(  xr( |t
        j                  k(  xs |t
        j                  k(  }| j                  |   j                  |      }| j                  |   j                  |      }|r@|j                         }|j                         }|j                         }|j                         }t        | j                  | j                        t        | j                  | j                        }
}	|ddd| j                   d|	f   j                  |      }|d|
d| j                   ddf   j                  |      }|j#                  d      }t%        j&                  |||      j#                  d      }|	| j                  k(  r||fS |j(                  |j(                  fS rR   rU   rc   s                  r#   rl   zLinear4bit.get_scaled_basesb  s    1J1J~#**$$E
 ";;%/gUemm5K5fuX]XfXfOfL"227;>>vFO!009<<VDN'--/
'--/
"1"7"7"9!/!5!5!7  #4#4#4d6F6FGTM^M^`d`p`pIqWG
 "!%5t~~%5xx"?@CCFKH!(7(,<dnn,<a"?@CCFKH''!'4H%++Ho~V^^gh^iH$***))::xzz))r$   c                    | j                  |      \  }}||z  }t        || j                        }| j                  |   }||z  S rn   ro   rq   s          r#   r1   zLinear4bit.get_delta_weight  sN     "&!6!6w!?Hh(F%fd.A.ABMll7+G 7**r$   c                   | j                   r4| j                  r| j                           | j                  |g|i |}n| j                  r | j                  |g|i |}nq | j                  |g|i |}|j	                         }| j
                  D ];  }|| j                  j                         vr!| j                  ||j                        \  }}t        j                          }|r8|j                  }	|j                  }
|j                  |
k7  r|j                  |
      }| j                  |   } ||j                  |j                              }t        j                  j                   j#                  t        j                  j                   j#                  ||      |      }|r|j                  	      }| j$                  |   }|||z  z   }> |j                  |j                        S )Nru   )rv   rM   rP   r   clonerw   r-   r.   rl   r8   r9   rx   r7   r6   r   r>   ry   rz   rp   r{   s                  r#   r   zLinear4bit.forward  s   $$;;LLN(<T<V<(<T<V<(<T<V<&*&:&: ?N%T-A-A-F-F-HH )-)>)>~VWV^V^)>)_&Hh.3.G.G.I*I'*)/(077m3 !] 3A"33NCG$QTT(..%9:F%*XX%8%8%?%?@S@S@Z@Z[ack@lnv%wN*)7):):>)J"ll>:G#nw&>>F-?2 99QWW%%r$   c                *    t         |          }d|z   S r   r   r   s     r#   r   zLinear4bit.__repr__  r   r$   r   r   r   r   r   r   r   r   r   r   r   r   s   @r#   r   r     s     "#&)#(!%	'	 	 	  	 $	 !	 	 	6'	<R	2,	*\	+"#	&J	% 	%r$   r   )
__future__r   r/   typingr   bitsandbytesr=   r9   peft.import_utilsr   r   peft.tuners.tuners_utilsr   peft.utils.integrationsr   peft.utils.otherr	   layerr   r   r>   Moduler   r    r$   r#   <module>r      sk    #     E < 9 & 0 _%uxx _%D D%UXX__m D% r$   