
    bi[                    .   d dl mZ d dlZd dlmZmZ d dlZd dlZd dl	m
Z
mZ d dlmZmZ d dlmZ d dlmZ dd	lmZmZ  e       r$ G d
 dej,                  j.                  e      ZddZ e
       r% G d dej,                  j.                  e      ZddZyy)    )annotationsN)AnyOptional)is_bnb_4bit_availableis_bnb_available)BaseTunerLayercheck_adapters_to_merge)dequantize_bnb_weight)	transpose   )	LoraLayerLoraVariantc                       e Zd Z	 	 	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
 fdZddZdddZddZd Z	 	 	 	 	 	 	 	 	 	 ddZddZ	d fdZ
 xZS )Linear8bitLtc
           
         t         |           t        j                  | |       d| _        || _        | j                  ||||||||	       y NF)
lora_alphalora_dropoutinit_lora_weights
use_rslorause_dora	lora_biassuper__init__r   fan_in_fan_out_active_adapterupdate_layerself
base_layeradapter_namerr   r   r   r   r   r   kwargs	__class__s              O/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/lora/bnb.pyr   zLinear8bitLt.__init__"   [     GtZ0"'D#/D %)"3%!#  	    c               "    |sy ddl m}  |       S Nr   )DoraLinearVariantvariantsr+   r    r   r$   r+   s       r&   resolve_lora_variantz!Linear8bitLt.resolve_lora_variant?       3$&&r(   c                   t        | |      }|sy|D ]b  }|| j                  j                         vr!t        j                  d       | j                         j                  }| j                         j                  }|j                  |j                  |_        t        ||      }|| j                  vrI| j                  |      }|j                  |j                        j                  |j                        |z   }n | j                  |   j                  | ||      }|r2t!        j"                  |      j%                         st'        d| d      t(        j*                  j-                  |j                  d      d|j.                        j                  |j                        | j                         _        | j0                  |   r| j                         j2                  j4                  | j6                  |   j2                  z   }	|r$t!        j"                  |	      st'        d| d      |	| j                         j2                  _        |j9                          | j:                  j=                  |       e y)	  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            NzWMerge lora module to 8-bit linear may get different generations due to rounding errors.state1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)r	   lora_Akeyswarningswarnget_base_layerweightr4   SCBr
   lora_variantget_delta_weighttodtypedevice
merge_safetorchisfiniteall
ValueErrorbnbnn
Int8Paramsr:   r   biasdatalora_Breset_gradsmerged_adaptersappend)
r    
safe_mergeadapter_namesactive_adapterr@   r4   output	lora_dataw_data	bias_datas
             r&   mergezLinear8bitLt.mergeG   s"    4D-HM "/ (<!)9)9);;m ,,.55++-3399$ &

EI /vUC!):):: $ 5 5n EI#YYy7::9;K;KLyXF!..~>II$P^`fgFennV&<&@&@&B$KNK[[no  03vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, >>.1 $ 3 3 5 : : ? ?$++nB]BbBb bI!%..*C(OP^O__rs  7@D'')..3!!#$$++N;Q(<r(   c                   | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }|| j                  j                         vrPt        j                  d       | j                         j                  }| j                         j                  }|j                  |j                  |_        t        ||      }|| j                  vrI| j                  |      }|j                  |j                         j                  |j"                        |z
  }n | j                  |   j%                  | ||      }t&        j(                  j+                  |j                  d      d|j,                        j                  |j"                        | j                         _	        | j.                  |   rD| j                         j0                  xj2                  | j4                  |   j0                  z  c_        |j7                          t        | j                        dkD  ryy)	_
            This method unmerges all merged adapter layers from the base weights.
             Already unmerged. Nothing to do.Nr   zYUnmerge lora module to 8-bit linear may get different generations due to rounding errors.r3   r7   Fr8   )mergedr=   r>   lenrS   popr;   r<   r?   r@   r4   rA   r
   rB   rC   rD   rE   rF   unmergerL   rM   rN   r:   r   rO   rP   rQ   rR   )r    rW   r@   r4   rX   rY   rZ   s          r&   rc   zLinear8bitLt.unmerge   s    ;;@Ad**+a/!%!5!5!9!9!;!)9)9);;o ,,.55++-3399$ &

EI.vUC!):):: $ 5 5n EI#YYy7::9;K;KLyXF!..~>FFt^]cdF/2vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, >>.1'')..33t{{>7R7W7WW3!!#5 d**+a/r(   c                    t        | j                  |   j                  | j                  |   j                  z  d      | j                  |   z  S NFr   rQ   r@   r;   scalingr    adapters     r&   rC   zLinear8bitLt.get_delta_weight   K    KK(//$++g2F2M2MM ,,w'	(r(   c          	         | j                   |g|i |}t        |      }g }|D ]5  }|j                  t        |      D 	
cg c]  \  }	}
|
|k(  s|	 c}
}	       7 t        |      D ]  \  }}|dk(  r|| j                  j                         vr)| j                  |   }| j                  |   }| j                  |   }| j                  |   }t        j                          }|r2|j                  }| j                  ||j                  j                        }|||      } | | ||                  |z  }|r|j                        }|||   xx   |z  cc<    |S c c}
}	w N__base__r!   setrT   	enumerater;   r<   rQ   r   rg   rH   is_autocast_enabledrE   _cast_input_dtyper@   rD   r    xrV   argsr$   resultunique_adapterssub_batch_indices_listri   indexitemirW   r;   rQ   dropoutrg   requires_conversionexpected_dtype	sub_batchrX   s                        r&   _mixed_batch_forwardz!Linear8bitLt._mixed_batch_forward   |   
 %T__Q888F!-0O%'"* v&--	R_H`.tdhlsdsu.tuv &/%? <!>!Z/!)9)9);;^4^4++N;,,~6*/*C*C*E&E#&%+\\N..q&--2E2EFA 4Q78	wy'9 :;gE&#YY~6F-a01V;1-<0 M5 /u   EEc                    | j                   |g|i | |j                  dd       }| j                  r4| j                  r| j	                           | j
                  |g|i |}|S | | j                  |g|d|i|}|S | j                  r | j
                  |g|i |}|S  | j
                  |g|i |}| j                  D ]  }|| j                  j                         vr!| j                  |   }| j                  |   }| j                  |   }	| j                  |   }
t        j                          }|r2|j                  }| j!                  ||j"                  j                        }|| j$                  vr0 | | |	|                  |
z  }|r|j'                        }||z   }| j$                  |   j)                  | |||      }|s
|j'                        } |S NrV   )rW   rt   rv   )_check_forward_argsrb   disable_adaptersr`   rc   r!   r   active_adaptersr;   r<   rQ   r   rg   rH   rq   rE   rr   r@   rB   rD   forwardr    rt   ru   r$   rV   rv   rW   r;   rQ   r|   rg   r}   r~   rX   s                 r&   r   zLinear8bitLt.forward   s	   $D$$Q888"JJ=M$$;;LLN(<T<V<F ME *2221ctc=c\bcB MA (<T<V<> M; )<T<V<&*&:&: ?N%T[[-=-=-?? ![[8F![[8F"//?G"ll>:G.3.G.G.I*I'*)/ 221fmm6I6IJ%T->->>!'wqz(:!;g!E.%+YY~%>F!'&!%!2!2>!B!J!J +9#)	 "K " /%+YY~%>F5?8 Mr(   c                *    t         |          }d|z   S Nzlora.r   __repr__r    repr%   s     r&   r   zLinear8bitLt.__repr__      '"$CS= r(   r   r   g        TFFFr!   torch.nn.Moduler"   strr#   intr   r   r   floatr   boolr   r   r   r   r   r   returnNoner   r   r   zOptional[LoraVariant]FNrU   r   rV   zOptional[list[str]]r   r   r   r   
rt   torch.Tensorru   r   rV   z	list[str]r$   r   r   r   rt   r   r   r   r   r   __name__
__module____qualname__r   r/   r\   rc   rC   r   r   r   __classcell__r%   s   @r&   r   r       s     "%&*$"#	'	 	 		
 	  	  $	 	 	 	 	:	':	<x"	$H	$	!$	*-$	>G$	SV$	$	L*	X	! 	!r(   r   c                   d }t        | t              r| j                         }n| }|j                  dd      }|rt        |t        j
                  j                        rd|j                         }|j                  | j                  j                  | j                  j                  | j                  d       t        | |fi |}|S )Nloaded_in_8bitF)r:   	thresholdry   )
isinstancer   r?   getrL   rM   r   copyupdater4   r:   r   ry   )targetr"   r$   
new_moduletarget_base_layerr   eightbit_kwargss          r&   dispatch_bnb_8bitr     s    
fn- & 5 5 7 &$4e<j):CFF<O<OP$kkmO""(.(E(E!'!7!7#\\ &flNoNJr(   c                       e Zd Z	 	 	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
 fdZddZdddZddZd Z	 	 	 	 	 	 	 	 	 	 ddZddZ	d fdZ
 xZS )
Linear4bitc
           
         t         |           t        j                  | |       d| _        || _        | j                  ||||||||	       y r   r   r   s              r&   r   zLinear4bit.__init__!  r'   r(   c               "    |sy ddl m}  |       S r*   r,   r.   s       r&   r/   zLinear4bit.resolve_lora_variant>  r0   r(   c                   t        | |      }|sy|D ]H  }|| j                  j                         vr!t        j                  d       | j                         j                  }|j                  }t        ||j                        }|| j                  vr| j                  |      }||z   }n | j                  |   j                  | ||      }|r2t        j                  |      j                         st!        d| d      d|v rd|d<   d|d<   |j#                  d	d       |j%                         D 	
ci c]  \  }	}
|	j'                  d
      r|	|
 }}	}
t)        j*                  j,                  |j/                  d      fi |j/                  |j0                        | j                         _        | j2                  |   r| j                         j4                  j6                  | j8                  |   j4                  z   }|r$t        j                  |      st!        d| d      || j                         j4                  _        | j:                  j=                  |       K yc c}
}	w )r2   NzWMerge lora module to 4-bit linear may get different generations due to rounding errors.r3   r5   r6   bnb_quantizedFr9   rP   _r7   )r	   r;   r<   r=   r>   r?   r@   __dict__r
   quant_staterB   rC   rG   rH   rI   rJ   rK   rb   items
startswithrL   rM   
Params4bitrD   rF   r   rO   rP   rQ   rS   rT   )r    rU   rV   rW   r@   r$   rX   rY   rZ   kvr[   s               r&   r\   zLinear4bit.mergeF  s*    4D-HM "/ '<!)9)9);;m ,,.55.vV=O=OP!):):: $ 5 5n EI#i/F!..~>II$P^`fgFennV&<&@&@&B$KNK[[no  #f,.3F?+*/'

64(+1<<>S41acAR!Q$SS/2vv/@/@5AQ/\U[/\/_/_`f`m`m/n##%,>>.1 $ 3 3 5 : : ? ?$++nB]BbBb bI!%..*C(OP^O__rs  7@D'')..3$$++N;O'<8 Ts   1IIc                   | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }|| j                  j                         vrPt        j                  d       | j                         j                  }|j                  }t        ||j                        }|| j                  vr| j                  |      }||z
  }n | j                  |   j                  | ||      }d|v rd|d<   d|d<   |j                  d	d       t!        j"                  j$                  |j'                  d
      fi |j'                  |j(                        | j                         _	        | j*                  |   rD| j                         j,                  xj.                  | j0                  |   j,                  z  c_        t        | j                        dkD  ryy)r^   r_   Nr   zYUnmerge lora module to 4-bit linear may get different generations due to rounding errors.r3   r   Fr9   rP   r7   )r`   r=   r>   ra   rS   rb   r;   r<   r?   r@   r   r
   r   rB   rC   rc   rL   rM   r   rD   rF   r   rO   rP   rQ   )r    rW   r@   r$   rX   rY   rZ   s          r&   rc   zLinear4bit.unmerge  s    ;;@Ad**+a/!%!5!5!9!9!;!)9)9);;o ,,.55.vV=O=OP!):):: $ 5 5n EI#i/F!..~>FFt^]cdF"f,.3F?+*/'

64(/2vv/@/@5AQ/\U[/\/_/_`f`m`m/n##%,>>.1'')..33t{{>7R7W7WW33 d**+a/r(   c                    t        | j                  |   j                  | j                  |   j                  z  d      | j                  |   z  S re   rf   rh   s     r&   rC   zLinear4bit.get_delta_weight  rj   r(   c          	         | j                   |g|i |}t        |      }g }|D ]5  }|j                  t        |      D 	
cg c]  \  }	}
|
|k(  s|	 c}
}	       7 t        |      D ]  \  }}|dk(  r|| j                  j                         vr)| j                  |   }| j                  |   }| j                  |   }| j                  |   }t        j                          }|r2|j                  }| j                  ||j                  j                        }|||      } | | ||                  |z  }|r|j                        }|||   xx   |z  cc<    |S c c}
}	w rl   rn   rs   s                        r&   r   zLinear4bit._mixed_batch_forward  r   r   c                    | j                   |g|i | |j                  dd       }| j                  r4| j                  r| j	                           | j
                  |g|i |}|S | | j                  |g|d|i|}|S | j                  r | j
                  |g|i |}|S  | j
                  |g|i |}|j                         }| j                  D ]  }|| j                  j                         vr!| j                  |   }| j                  |   }| j                  |   }	| j                  |   }
t        j                          }|r2|j                   }| j#                  ||j$                  j                         }|| j&                  vr0 | | |	|                  |
z  }|r|j)                        }||z   }| j&                  |   j+                  | |||      }|s
|j)                        } |S r   )r   rb   r   r`   rc   r!   r   cloner   r;   r<   rQ   r   rg   rH   rq   rE   rr   r@   rB   rD   r   r   s                 r&   r   zLinear4bit.forward  s   $D$$Q888"JJ=M$$;;LLN(<T<V<T MS *2221ctc=c\bcP MO (<T<V<L MI )<T<V<  &*&:&: ?N%T[[-=-=-?? ![[8F![[8F"//?G"ll>:G.3.G.G.I*I'*)/ 221fmm6I6IJ%T->->>!'wqz(:!;g!E.%+YY~%>F!'&!%!2!2>!B!J!J +9#)	 "K " /%+YY~%>F5?8 Mr(   c                *    t         |          }d|z   S r   r   r   s     r&   r   zLinear4bit.__repr__  r   r(   r   r   r   r   r   r   r   r   r   r   r   s   @r&   r   r     s     "%&*$"#	'	 	 		
 	  	  $	 	 	 	 	:	'9	<v!	XF	$	!$	*-$	>G$	SV$	$	L1	f	! 	!r(   r   c                   d }t        | t              r| j                         }n| }|j                  dd      }|rt	               rt        |t
        j                  j                        rd|j                         }|j                  |j                  |j                  j                  |j                  j                  d       t        | |fi |}|S )Nloaded_in_4bitF)compute_dtypecompress_statistics
quant_type)r   r   r?   r   r   rL   rM   r   r   r   r   r@   r   r   )r   r"   r$   r   r   r   fourbit_kwargss          r&   dispatch_bnb_4bitr   
  s    
fn- & 5 5 7 &$4e<35*EVX[X^X^XiXi:j#[[]N!!%6%D%D+<+C+C+W+W"3":":"E"E $FLKNKJr(   )r   r   r"   r   )
__future__r   r=   typingr   r   bitsandbytesrL   rH   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r	   peft.utils.integrationsr
   peft.utils.otherr   layerr   r   rM   Moduler   r   r   r    r(   r&   <module>r      sw    #      E L 9 & ) d!uxx	 d!L. i!UXX__i i!V[ r(   