
    bi !                         d dl mZ d dlZd dlmc mZ d dlmZ d dlmZm	Z	 d dl
mZ  G d dej                        Z G d d	e      Z G d
 de      Z G d de      Z G d de      Z G d de      Zy)    )deepcopyN)nn)dequantize_module_weightgather_params_ctx)	transposec                   h     e Zd Z fdZdej
                  fdZddddZddd	Zde	f fd
Z
 xZS )DoraLinearLayerc                 0    t         |           || _        y N)super__init__fan_in_fan_outselfr   	__class__s     P/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/lora/dora.pyr   zDoraLinearLayer.__init__   s    ,    returnc                     t        || j                        }|||z  z   }t        j                  j	                  |d      j                  |j                        }|S )N   )dim)r   r   torchlinalgnormtodtype)r   weightlora_weightscalingweight_norms        r   get_weight_normzDoraLinearLayer.get_weight_norm   sQ    64#6#67'K//ll''A'699&,,Gr   F)place_on_cpuNc                   |j                   t        j                  k(  }|r |j                         }|j                         }t	        |j                               5  |j                  j                  dk(  rt        |      }t        |      }|j                  j                  dk\  rc|j                  d   }t        j                  |j                  d|g      |j                  |dg            }	|	j                  |j                        }	n||z  }	|r|	j!                         }	| j#                  |j%                  |j&                        |	|      }
d d d        |r
j%                  d      }
t)        j*                  
d      | _        y # 1 sw Y   9xY w)N
Linear4bit   r   cpuT)requires_grad)r   r   float16floatr   
parametersr   __name__r   r   datandimshapemmviewreshapehalfr!   r   devicer   	Parameterr   )r   
base_layerlora_Alora_Br   r"   dtype_is_fp16r   rr   r    s              r   update_layerzDoraLinearLayer.update_layer%   s?   5\\^F\\^Fz4467 	_##,,< &j1
-j9F{{1$LLO#hhv{{B7';V[[!R=QR)11&,,?$vo)..0..vyy/GV]^K!	_$ %../Kll;dC)	_ 	_s   C/F  F	base_resultc                   t        j                  |j                  j                  d   |j                  j                  |j
                        } | ||            j                  }| j                  }	t        |      }
|
j                  |j
                        }
| j                  |
|j                         |      }|j                         }|	|z  j                  dd      } | ||            }d}||j                  }|0||z
  }n*t        j                  |t        |
| j                               }|dz
  |z  ||z  |z  z   }|S )
        For DoRA, calculate the extra output from LoRA with DoRA applied. This should be added on top of the base layer
        output.
        r   )r4   r   r&   N)r   eyer   r/   r4   r   Tr   r   r!   detachr1   biasFlinearr   r   )r   xr7   r8   r   r6   r=   x_eyer   	magnituder   r    mag_norm_scalelora_resultrC   result_doras                   r   forwardzDoraLinearLayer.forwardB   s'    		&----a09M9MUVU\U\]VE]+--KK	)*5177#**6;3E3E3GQ "((*#k1772>VAY'"??D)D0((1i8K8K&LMK%)[8>K;WZa;aar   c                 *    t         |          }d|z   S Nz
lora.dora.r   __repr__r   repr   s     r   rP   zDoraLinearLayer.__repr__g       g c!!r   )r   N)r,   
__module____qualname__r   r   Tensorr!   r;   rL   strrP   __classcell__r   s   @r   r	   r	      sC    -u||  QV D: NR #J"# " "r   r	   c                   *     e Zd Zd Zdef fdZ xZS )DoraEmbeddingLayerc                    ||z  j                   }| j                  }|j                  }	| j                  |	|j                         |      }
|
j                         }
||
z  }| |||      |z  z  |z  }||fS )r?   )rA   r   r!   rB   )r   rF   r7   r8   r   r6   embed_fnr   rH   r   r    rI   rK   s                r   rL   zDoraEmbeddingLayer.forwardm   s    
 ))KK	""**6;3E3E3GQ "((*"[0$F(;f(DEO{**r   r   c                 *    t         |          }d|z   S rN   rO   rQ   s     r   rP   zDoraEmbeddingLayer.__repr__   rS   r   )r,   rT   rU   rL   rW   rP   rX   rY   s   @r   r[   r[   l   s    +("# " "r   r[   c                   P     e Zd Zdej                  fdZdddZdef fdZ xZ	S )_DoraConvNdLayerr   c                     |||z  z   }t        t        d|j                                     }|j                  d|d      j	                  dd      }|S )Nr      T)pr   keepdimr   )tupleranger   r   r   )r   r   r   r   r   r    s         r   r!   z _DoraConvNdLayer.get_weight_norm   sO    'K//E!VZZ\*+kkA3k=GG1Mr   Nr<   c          	         |j                   }|j                   j                  d   }t        j                  |j                   j	                  d|g      |j                   j	                  |dg            }	|	j                  |j                        }	| j                   }
| j                  ||	j                         |      }|j                         }|
|z  }|A| j                  ||d|j                  |j                  |j                  |j                        }n9|j                  }|+dd|j                         dz
  z  z   }| |j                  | z
  }|dz
  |z  | | ||            z  |z  z   }|S )	r?   r   r&   N)rC   stridepaddingdilationgroups)r   r&   )r   rb   r   )r   r/   r   r0   r1   r2   r!   rB   conv_fnrh   ri   rj   rk   rC   r   )r   rF   r7   r8   r   r6   r=   r   r:   r   rH   r    rI   rC   
bias_shaperK   s                   r   rL   z_DoraConvNdLayer.forward   sg   
 ""MM"hhv}}112q':FMM<N<NPQSUw<WX!))&,,7KK	**6;3E3E3GQ "((*"[0,,!(("**#,,!(( ' K ??D$t{/@1/D'EE
)IDIIz,BB%)[8>FSYZ[S\L];]`g;ggr   c                 *    t         |          }d|z   S rN   rO   rQ   s     r   rP   z_DoraConvNdLayer.__repr__   rS   r   )
r,   rT   rU   r   rV   r!   rL   rW   rP   rX   rY   s   @r   r`   r`      s1    u||  NR &P"# " "r   r`   c                        e Zd Z fdZ xZS )DoraConv1dLayerc                 N    t         |   |       t        j                  | _        y r   )r   r   rD   conv1drl   r   s     r   r   zDoraConv1dLayer.__init__       (xxr   r,   rT   rU   r   rX   rY   s   @r   rp   rp             r   rp   c                        e Zd Z fdZ xZS )DoraConv2dLayerc                 N    t         |   |       t        j                  | _        y r   )r   r   rD   conv2drl   r   s     r   r   zDoraConv2dLayer.__init__   rs   r   rt   rY   s   @r   rw   rw      ru   r   rw   c                        e Zd Z fdZ xZS )DoraConv3dLayerc                 N    t         |   |       t        j                  | _        y r   )r   r   rD   conv3drl   r   s     r   r   zDoraConv3dLayer.__init__   rs   r   rt   rY   s   @r   r{   r{      ru   r   r{   )copyr   r   torch.nn.functionalr   
functionalrD   peft.utils.integrationsr   r   peft.utils.otherr   Moduler	   r[   r`   rp   rw   r{    r   r   <module>r      so         O &P"bii P"f" "43" 3"l &   &   &  r   