
    bi	>                       d dl mZ d dlZd dlmZ d dlZd dlZd dlm	Z	m
Z
 d dlmZmZ d dlmZ ddlmZ  e
       r$ G d	 d
ej$                  j&                  e      ZddZ e	       r% G d dej$                  j&                  e      ZddZyy)    )annotationsN)Optional)is_bnb_4bit_availableis_bnb_available)BaseTunerLayercheck_adapters_to_merge)dequantize_bnb_weight   )OFTLayerc                       e Zd Z	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZd	d
dZddZd ZddZd fdZ xZ	S )Linear8bitLtc                    t         |           t        j                  | |       d| _        || _        | j                  |||||||	||
|
       y NF)oft_block_sizemodule_dropoutcoftepsblock_shareinit_weightsuse_cayley_neumannnum_cayley_neumann_termssuper__init__r   fan_in_fan_out_active_adapterupdate_layer)self
base_layeradapter_namerr   r   r   r   r   r   r   r   kwargs	__class__s                N/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/oft/bnb.pyr   zLinear8bitLt.__init__!   a     GdJ/"'D#/D --')#5)A      c                $   t        | |      }|sy|D ]  }|| j                  j                         vr!t        j                  d       | j                         j                  }| j                         j                  }|j                  |j                  |_        t        ||      }| j                  |      }t        j                  |dd      }t        j                  ||j                  |j                              }t        j                  |dd      }|j                  |j                        j                  |j                         }|r2t        j"                  |      j%                         st'        d| d      t(        j*                  j-                  |j                  d      d	|j.                  
      j                  |j                         | j                         _        |j1                          | j2                  j5                  |        y)  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            NzVMerge oft module to 8-bit linear may get different generations due to rounding errors.stater   r
   1NaNs detected in the merged weights. The adapter  seems to be brokencpuFrequires_gradhas_fp16_weights)r   oft_Rkeyswarningswarnget_base_layerweightr*   SCBr	   get_delta_weighttorch	transposemmtodtypedeviceisfiniteall
ValueErrorbnbnn
Int8Paramsr0   reset_gradsmerged_adaptersappend)	r   
safe_mergeadapter_namesactive_adapterr6   r*   outputoft_dataw_datas	            r$   mergezLinear8bitLt.mergeB   s    4D-HM "/ <!)::vw,,.55++-3399$ &

EI /vUC00@A6(FIIhnn,EFA68>>255hooFennV&<&@&@&B$KNK[[no  03vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#$$++N;?<r&   c                R   | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }|| j                  j                         vrPt        j                  d       | j                         j                  }| j                         j                  }|j                  |j                  |_        t        ||      }| j                  |      }t        j                  |dd      }t        j                   |j#                         |j%                  |j&                              }t        j                  |dd      }|j%                  |j&                        j%                  |j(                        }t*        j,                  j/                  |j%                  d      d|j0                  	      j%                  |j(                        | j                         _	        |j3                          t        | j                        dkD  ryy)
_
            This method unmerges all merged adapter layers from the base weights.
             Already unmerged. Nothing to do.Nr   zXUnmerge oft module to 8-bit linear may get different generations due to rounding errors.r)   r
   r-   Fr.   )mergedr3   r4   lenrF   popr1   r2   r5   r6   r*   r7   r	   r8   r9   r:   r;   tr<   r=   r>   rB   rC   rD   r0   rE   )r   rJ   r6   r*   rK   rL   rM   s          r$   unmergezLinear8bitLt.unmergeu   s    ;;@Ad**+a/!%!5!5!9!9!;!)::n ,,.55++-3399$ &

EI.vUC00@A6(**,		(..0IJA68>>255hooF/2vv/@/@IIe$EFLcLc 0A 0"V]]# ##%, !!#3 d**+a/r&   c                <    | j                   |   j                         S Nr1   
get_weightr   adapters     r$   r8   zLinear8bitLt.get_delta_weight       ::g&1133r&   c                >   | j                   r4| j                  r| j                           | j                  |g|i |}|S | j                  r | j                  |g|i |}|S | j                  D ]  }|| j
                  j                         vr | j
                  |   }t        j                          }|r2|j                  }| j                  ||j                  j                        } ||      }|s|j                        }  | j                  |g|i |}|S rX   disable_adaptersrR   rV   r   active_adaptersr1   r2   r9   is_autocast_enabledr=   _cast_input_dtyper6   r<   	r   xargsr"   resultrJ   r1   requires_conversionexpected_dtypes	            r$   forwardzLinear8bitLt.forward   s   $$;;LLN(<T<V<( M' (<T<V<$ M! '+&:&: 1N%TZZ__->>  JJ~6E.3.G.G.I*I'*)* 221ell6H6HIaA*DD01 )<T<V<Mr&   c                *    t         |          }d|z   S Nzoft.r   __repr__r   repr#   s     r$   rn   zLinear8bitLt.__repr__       '"$CC<r&   )	   r           TFiUMu?FF   )r   torch.nn.Moduler    strr!   intr   rx   r   floatr   boolr   rz   r   ry   r   rz   r   rz   r   rx   returnNoneFNrH   rz   rI   zOptional[list[str]]r{   r|   r{   r|   re   torch.Tensorr{   r   r{   rw   
__name__
__module____qualname__r   rN   rV   r8   rj   rn   __classcell__r#   s   @r$   r   r      s     "#$'!% %',,-	'	 	 		
  	 "	 	 	 	 	 !%	 '*	 	B1	<f!	$F	4	4	  	 r&   r   c                   d }t        | t              r| j                         }n| }|j                  dd      }|rt        |t        j
                  j                        rd|j                         }|j                  | j                  j                  | j                  j                  | j                  d       t        | |fi |}|S )Nloaded_in_8bitF)r0   	thresholdindex)
isinstancer   r5   getrB   rC   r   copyupdater*   r0   r   r   )targetr    r"   
new_moduletarget_base_layerr   eightbit_kwargss          r$   dispatch_bnb_8bitr      s    
fn- & 5 5 7 &$4e<j):CFF<O<OP$kkmO""(.(E(E!'!7!7#\\ &flNoNJr&   c                       e Zd Z	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZd	d
dZddZd ZddZd fdZ xZ	S )
Linear4bitc                    t         |           t        j                  | |       d| _        || _        | j                  ||||||||	|
|
       y r   r   )r   r   r    r!   r   r   r   r   r   r   r   r   r"   r#   s                r$   r   zLinear4bit.__init__   r%   r&   c                `   t        | |      }|sy|D ]  }|| j                  j                         vr!t        j                  d       | j                         j                  }|j                  }t        ||j                        }| j                  |      }t        j                  |dd      }t        j                  ||j                  |j                              }t        j                  |dd      }|j                  |j                        j                  |j                         }|r2t        j"                  |      j%                         st'        d| d      d|v rd	|d<   d	|d
<   |j)                  dd       |j+                         D 	
ci c]  \  }	}
|	j-                  d      r|	|
 }}	}
t/        j0                  j2                  |j                  d      fi |j                  |j                         | j                         _        | j4                  j7                  |        yc c}
}	w )r(   NzVMerge oft module to 4-bit linear may get different generations due to rounding errors.r)   r   r
   r+   r,   bnb_quantizedFr/   data_r-   )r   r1   r2   r3   r4   r5   r6   __dict__r	   quant_stater8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rT   items
startswithrB   rC   
Params4bitrF   rG   )r   rH   rI   rJ   r6   r"   rK   rL   rM   kvs              r$   rN   zLinear4bit.merge   s    4D-HM "/ <!)::vw,,.55.vV=O=OP00@A6(FIIhnn,EFA68>>255hooFennV&<&@&@&B$KNK[[no  #f,.3F?+*/'

64(+1<<>S41acAR!Q$SS/2vv/@/@5AQ/\U[/\/_/_`f`m`m/n##%,$$++N;=<6 Ts   H*(H*c                   | j                   st        j                  d       yt        | j                        dkD  r| j                  j                         }|| j                  j                         vrPt        j                  d       | j                         j                  }|j                  }t        ||j                        }| j                  |      }t        j                  |dd      }t        j                   |j#                         |j%                  |j&                              }t        j                  |dd      }|j%                  |j&                        j%                  |j(                        }d|v rd|d<   d|d	<   |j                  d
d       t+        j,                  j.                  |j%                  d      fi |j%                  |j(                        | j                         _	        t        | j                        dkD  ryy)rP   rQ   Nr   zXUnmerge oft module to 4-bit linear may get different generations due to rounding errors.r)   r
   r   Fr/   r   r-   )rR   r3   r4   rS   rF   rT   r1   r2   r5   r6   r   r	   r   r8   r9   r:   r;   rU   r<   r=   r>   rB   rC   r   )r   rJ   r6   r"   rK   rL   rM   s          r$   rV   zLinear4bit.unmerge'  s    ;;@Ad**+a/!%!5!5!9!9!;!)::n ,,.55.vV=O=OP00@A6(**,		(..0IJA68>>255hooF"f,.3F?+*/'

64(/2vv/@/@5AQ/\U[/\/_/_`f`m`m/n##%,/ d**+a/r&   c                <    | j                   |   j                         S rX   rY   r[   s     r$   r8   zLinear4bit.get_delta_weightH  r]   r&   c                >   | j                   r4| j                  r| j                           | j                  |g|i |}|S | j                  r | j                  |g|i |}|S | j                  D ]  }|| j
                  j                         vr | j
                  |   }t        j                          }|r2|j                  }| j                  ||j                  j                        } ||      }|s|j                        }  | j                  |g|i |}|S rX   r_   rd   s	            r$   rj   zLinear4bit.forwardK  s   $$;;LLN(<T<V<6 M5 (<T<V<2 M! '+&:&: 1N%TZZ__->>  JJ~6E.3.G.G.I*I'*)* 221ell6H6HIaA*DD01 )<T<V<Mr&   c                *    t         |          }d|z   S rl   rm   ro   s     r$   rn   zLinear4bit.__repr__l  rq   r&   )	rr   r   rs   Frt   FTFru   )r   rv   r    rw   r!   rx   r   rx   r   ry   r   rz   r   ry   r   rz   r   rz   r   rz   r   rx   r{   r|   r}   r~   r   r   r   r   r   s   @r$   r   r      s     "#$' %!%',,-	'	 	 		
  	 "	 	 	 	 	 !%	 '*	 	B0	<d	oB	4	B	  	 r&   r   c                   d }t        | t              r| j                         }n| }|j                  dd      }|rt	               rt        |t
        j                  j                        rd|j                         }|j                  |j                  |j                  j                  |j                  j                  d       t        | |fi |}|S )Nloaded_in_4bitF)compute_dtypecompress_statistics
quant_type)r   r   r5   r   r   rB   rC   r   r   r   r   r6   r   r   )r   r    r"   r   r   r   fourbit_kwargss          r$   dispatch_bnb_4bitr   p  s    
fn- & 5 5 7 &$4e<35*EVX[X^X^XiXi:j#[[]N!!%6%D%D+<+C+C+W+W"3":":"E"E $FLKNKJr&   )r   rv   r    rw   )
__future__r   r3   typingr   bitsandbytesrB   r9   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r   peft.utils.integrationsr	   layerr   rC   Moduler   r   r   r    r&   r$   <module>r      st    #     E L 9  X uxx X t. \ UXX__h \ |A r&   