
    bi<                         d dl mZ  G d de      Z G d de      Z G d de      Z G d d	e      Z G d
 de      Z G d de      Zy)    )Optimizer2Statec                   6     e Zd Z	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )AdamWc                 <    t         |   d||||||||	|
||       y)a  
        Base AdamW optimizer.

        Arguments:
            params (`torch.Tensor`):
                The input parameters to optimize.
            lr (`float`, defaults to 1e-3):
                The learning rate.
            betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
                The beta values are the decay rates of the first and second-order moment of the optimizer.
            eps (`float`, defaults to 1e-8):
                The epsilon value prevents division by zero in the optimizer.
            weight_decay (`float`, defaults to 1e-2):
                The weight decay value for the optimizer.
            amsgrad (`bool`, defaults to `False`):
                Whether to use the [AMSGrad](https://hf.co/papers/1904.09237) variant of Adam that uses the maximum of past squared gradients instead.
            optim_bits (`int`, defaults to 32):
                The number of bits of the optimizer state.
            args (`object`, defaults to `None`):
                An object with additional arguments.
            min_8bit_size (`int`, defaults to 4096):
                The minimum number of elements of the parameter tensors for 8-bit optimization.
            percentile_clipping (`int`, defaults to 100):
                Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.
            block_wise (`bool`, defaults to `True`):
                Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.
            is_paged (`bool`, defaults to `False`):
                Whether the optimizer is a paged optimizer or not.
        adamis_pagedNsuper__init__selfparamslrbetasepsweight_decayamsgrad
optim_bitsargsmin_8bit_sizepercentile_clipping
block_wiser	   	__class__s                S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/bitsandbytes/optim/adamw.pyr   zAdamW.__init__
   s<    X 	 	 	
    MbP?g?g+?:0yE>{Gz?F    N   d   TF__name__
__module____qualname__r   __classcell__r   s   @r   r   r   	   0     9
 9
r   r   c                   6     e Zd Z	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )	AdamW8bitc                 v    |rt        d      |dk7  rt        d      t        | 	  d|||||d||	|
||       y)an  
        8-bit AdamW optimizer.

        Arguments:
            params (`torch.Tensor`):
                The input parameters to optimize.
            lr (`float`, defaults to 1e-3):
                The learning rate.
            betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
                The beta values are the decay rates of the first and second-order moment of the optimizer.
            eps (`float`, defaults to 1e-8):
                The epsilon value prevents division by zero in the optimizer.
            weight_decay (`float`, defaults to 1e-2):
                The weight decay value for the optimizer.
            amsgrad (`bool`, defaults to `False`):
                Whether to use the [AMSGrad](https://hf.co/papers/1904.09237) variant of Adam that uses the maximum of past squared gradients instead.
                Note: This parameter is not supported in AdamW8bit and must be False.
            optim_bits (`int`, defaults to 32):
                The number of bits of the optimizer state.
                Note: This parameter is not used in AdamW8bit as it always uses 8-bit optimization.
            args (`object`, defaults to `None`):
                An object with additional arguments.
            min_8bit_size (`int`, defaults to 4096):
                The minimum number of elements of the parameter tensors for 8-bit optimization.
            percentile_clipping (`int`, defaults to 100):
                Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.
            block_wise (`bool`, defaults to `True`):
                Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.
            is_paged (`bool`, defaults to `False`):
                Whether the optimizer is a paged optimizer or not.
        z'AdamW8bit does not support amsgrad=Truer"   zGAdamW8bit only supports optim_bits=32 (default value for compatibility)r      r   N
ValueErrorr   r   r   s                r   r   zAdamW8bit.__init__G   sa    ^ FGG fgg 	 	
r   r   r%   r*   s   @r   r-   r-   F   s2     D
 D
r   r-   c                   6     e Zd Z	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )
AdamW32bitc                 <    t         |   d|||||d||	|
||       y)a  
        32-bit AdamW optimizer.

        Arguments:
            params (`torch.Tensor`):
                The input parameters to optimize.
            lr (`float`, defaults to 1e-3):
                The learning rate.
            betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
                The beta values are the decay rates of the first and second-order moment of the optimizer.
            eps (`float`, defaults to 1e-8):
                The epsilon value prevents division by zero in the optimizer.
            weight_decay (`float`, defaults to 1e-2):
                The weight decay value for the optimizer.
            amsgrad (`bool`, defaults to `False`):
                Whether to use the [AMSGrad](https://hf.co/papers/1904.09237) variant of Adam that uses the maximum of past squared gradients instead.
            optim_bits (`int`, defaults to 32):
                The number of bits of the optimizer state.
            args (`object`, defaults to `None`):
                An object with additional arguments.
            min_8bit_size (`int`, defaults to 4096):
                The minimum number of elements of the parameter tensors for 8-bit optimization.
            percentile_clipping (`int`, defaults to 100):
                Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.
            block_wise (`bool`, defaults to `True`):
                Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.
            is_paged (`bool`, defaults to `False`):
                Whether the optimizer is a paged optimizer or not.
        r   r"   r   Nr
   r   s                r   r   zAdamW32bit.__init__   s<    X 	 	 	
r   r   r%   r*   s   @r   r3   r3      r+   r   r3   c                   4     e Zd Z	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )
PagedAdamWc                 <    t         |   d||||||||	|
|d       y)a=  
        Paged AdamW optimizer.

        Arguments:
            params (`torch.Tensor`):
                The input parameters to optimize.
            lr (`float`, defaults to 1e-3):
                The learning rate.
            betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
                The beta values are the decay rates of the first and second-order moment of the optimizer.
            eps (`float`, defaults to 1e-8):
                The epsilon value prevents division by zero in the optimizer.
            weight_decay (`float`, defaults to 1e-2):
                The weight decay value for the optimizer.
            amsgrad (`bool`, defaults to `False`):
                Whether to use the [AMSGrad](https://hf.co/papers/1904.09237) variant of Adam that uses the maximum of past squared gradients instead.
            optim_bits (`int`, defaults to 32):
                The number of bits of the optimizer state.
            args (`object`, defaults to `None`):
                An object with additional arguments.
            min_8bit_size (`int`, defaults to 4096):
                The minimum number of elements of the parameter tensors for 8-bit optimization.
            percentile_clipping (`int`, defaults to 100):
                Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.
            block_wise (`bool`, defaults to `True`):
                Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.
        r   Tr   Nr
   r   r   r   r   r   r   r   r   r   r   r   r   r   s               r   r   zPagedAdamW.__init__   s<    R 	 	 	
r   
r   r   r    r!   Fr"   Nr#   r$   Tr%   r*   s   @r   r6   r6      -     6
 6
r   r6   c                   4     e Zd Z	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )PagedAdamW8bitc                 v    |rt        d      |dk7  rt        d      t        | 	  d|||||d||	|
|d       y)	a  
        Paged 8-bit AdamW optimizer.

        Arguments:
            params (`torch.Tensor`):
                The input parameters to optimize.
            lr (`float`, defaults to 1e-3):
                The learning rate.
            betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
                The beta values are the decay rates of the first and second-order moment of the optimizer.
            eps (`float`, defaults to 1e-8):
                The epsilon value prevents division by zero in the optimizer.
            weight_decay (`float`, defaults to 1e-2):
                The weight decay value for the optimizer.
            amsgrad (`bool`, defaults to `False`):
                Whether to use the [AMSGrad](https://hf.co/papers/1904.09237) variant of Adam that uses the maximum of past squared gradients instead.
                Note: This parameter is not supported in PagedAdamW8bit and must be False.
            optim_bits (`int`, defaults to 32):
                The number of bits of the optimizer state.
                Note: This parameter is not used in PagedAdamW8bit as it always uses 8-bit optimization.
            args (`object`, defaults to `None`):
                An object with additional arguments.
            min_8bit_size (`int`, defaults to 4096):
                The minimum number of elements of the parameter tensors for 8-bit optimization.
            percentile_clipping (`int`, defaults to 100):
                Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.
            block_wise (`bool`, defaults to `True`):
                Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.
        z,PagedAdamW8bit does not support amsgrad=Truer"   zLPagedAdamW8bit only supports optim_bits=32 (default value for compatibility)r   r/   Tr   Nr0   r8   s               r   r   zPagedAdamW8bit.__init__  sa    X KLL kll 	 	
r   r9   r%   r*   s   @r   r<   r<     s/     A
 A
r   r<   c                   4     e Zd Z	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )PagedAdamW32bitc                 <    t         |   d|||||d||	|
|d       y)aD  
        Paged 32-bit AdamW optimizer.

        Arguments:
            params (`torch.Tensor`):
                The input parameters to optimize.
            lr (`float`, defaults to 1e-3):
                The learning rate.
            betas (`tuple(float, float)`, defaults to (0.9, 0.999)):
                The beta values are the decay rates of the first and second-order moment of the optimizer.
            eps (`float`, defaults to 1e-8):
                The epsilon value prevents division by zero in the optimizer.
            weight_decay (`float`, defaults to 1e-2):
                The weight decay value for the optimizer.
            amsgrad (`bool`, defaults to `False`):
                Whether to use the [AMSGrad](https://hf.co/papers/1904.09237) variant of Adam that uses the maximum of past squared gradients instead.
            optim_bits (`int`, defaults to 32):
                The number of bits of the optimizer state.
            args (`object`, defaults to `None`):
                An object with additional arguments.
            min_8bit_size (`int`, defaults to 4096):
                The minimum number of elements of the parameter tensors for 8-bit optimization.
            percentile_clipping (`int`, defaults to 100):
                Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.
            block_wise (`bool`, defaults to `True`):
                Whether to independently quantize each block of tensors to reduce outlier effects and improve stability.
        r   r"   Tr   Nr
   r8   s               r   r   zPagedAdamW32bit.__init__K  s<    R 	 	 	
r   r9   r%   r*   s   @r   r?   r?   J  r:   r   r?   N)bitsandbytes.optim.optimizerr   r   r-   r3   r6   r<   r?    r   r   <module>rC      s[    9:
O :
zE
 E
P:
 :
z7
 7
tB
_ B
J7
o 7
r   