
    biI                    "   d dl mZ d dlmZ d dlZd dlmZ d dlmZ d dlm	Z	 ddl
mZmZmZmZmZ dd	lmZmZmZmZmZmZmZ  G d
 de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Zy)    )annotations)AnyN)is_xpu_available)nn)	transpose   )DoraConv1dLayerDoraConv2dLayerDoraConv3dLayerDoraEmbeddingLayerDoraLinearLayer)Conv1dConv2dConv3d	EmbeddingLinearLoraVariant_ConvNdc                  f    e Zd Zedd       Zedd       Zed	d       Zedd       Zed
d       Zy)DoraLinearVariantc                $   | j                   s| j                  d d  dz   | _        t        t        | dd            }| j                  |   j
                  }| j                  |   j
                  }| j                  xr4 |j                  j                  dk(  xs |j                  j                  dk(  }| j                  r|j                  j                  dv r|j                  |j                        }n`|j                  j                  dvr-t               r|j                  d      }n|j                  d      }|j                  |j                        }| j                  |   }|j                  | j                         ||||	       || j                   |<   y )
Nlora_magnitude_vectorfan_in_fan_outFr   cpu)cudaxpur   r   )
base_layerlora_Alora_Bscalingplace_on_cpu)r   adapter_layer_namesr   getattrr    weightr!   ephemeral_gpu_offloaddevicetypetor   r"   update_layerget_base_layer)moduleadapter_namekwargs
dora_layerr    r!   r#   r"   s           T/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/lora/variants.pyinitzDoraLinearVariant.init   sW   ++)/)C)CA)FIc)cF&$GFDTV[4\]
|,33|,3333t9K9Ku9T9sX^XeXeXjXjnsXs''}}!!_46==1==%%_<')!'5!1!'6!26==1...,,.% 	  	
 6@$$\2    c                   |j                   }| j                  |      }| j                  |   j                  |t	        || j
                        d      j                         }| j                  | d|       | j                  |   j                  |z  }t	        |j                  dd      | j
                        }|||z   z  }|j                  |      }|S Nr   r"   -weight_norm)dtypeget_delta_weightr   get_weight_normr   r   detach_cache_storer&   viewr*   r-   active_adapterorig_weight
orig_dtypedelta_weightweight_normdora_factor
new_weights           r1   
merge_safezDoraLinearVariant.merge_safe;   s     &&
..~> ((8_[)L&BWBW*Xbc_dVX 	 	~.l;[I22>BIIKW 0 0Q 79N9NO K,$>?
]]:.
r3   c                   |j                   }| j                  |      }| j                  |   j                  |t	        || j
                        d      j                         }| j                  | d|       | j                  |   j                  |z  }t	        |j                  dd      | j
                        }||j                  |z   z  }|j                  |      }||_
        y r5   )r9   r:   r   r;   r   r   r<   r=   r&   r>   datar*   r?   s           r1   merge_unsafezDoraLinearVariant.merge_unsafeP   s     &&
..~>((8_[)L&BWBW*Xbc_dVX 	 	~.l;[I22>BIIKW 0 0Q 79N9NO K$4$4|$CD
]]:.
%r3   c                   |j                   }| j                  |      }| j                  | d      }| j                  |   j                  |z  }|j
                  |j                  dd      z  |z
  }|j                  |      }|S )Nr7   r8   r   r9   r:   
_cache_popr   r&   rI   r>   r*   r?   s           r1   unmergezDoraLinearVariant.unmergec   s     &&
..~>''>*:,(GH22>BIIKW %%(8(8Q(??,N
]]:.
r3   c           	     <   | j                   |   }| j                  |   }| j                  |   }| j                  |   }t	        |t
        j                        s| j                  s|}n
 ||      }d }| | j                  |   ||||| j                         |      z   }|S N)r    r!   r"   r   base_result
r    r!   lora_dropoutr"   
isinstancer   Identitytrainingr   r,   	r-   r@   xresultr    r!   dropoutr"   rQ   s	            r1   forwardzDoraLinearVariant.forwardm       ~.~.%%n5..0gr{{+6?? K
AKF&66~F,,.#
 
 r3   Nr-   r   r.   strr/   r   returnNoner-   r   r@   r^   rA   torch.Tensorr_   rb   r-   r   r@   r^   rA   rb   r_   r`   
r-   r   r@   r^   rX   rb   rY   rb   r_   rb   	__name__
__module____qualname__staticmethodr2   rG   rJ   rN   r[    r3   r1   r   r      sf    @ @:  ( & &$    r3   r   c                  f    e Zd Zedd       Zedd       Zed	d       Zedd       Zed
d       Zy)DoraEmbeddingVariantc                    | j                   | j                  d d  dz   | _        t        d      }| j                  |   }| j                  |   }| j
                  |   }|j                  | j                         |||       || j                   |<   y )Nr   Tr   r   r    r!   r"   )r   r$   r   lora_embedding_Alora_embedding_Br"   r+   r,   )r-   r.   r/   r0   ro   rp   r"   s          r1   r2   zDoraEmbeddingVariant.init   s    ''/)/)C)CA)FIc)cF&'t<
!22<@!22<@...,,.7GP`jq 	  	
 6@$$\2r3   c                j   |j                   }| j                  |      }| j                  |   j                  ||j                  d      j                         }| j                  | d|       | j                  |   j                  |z  }|j                  dd      }|||z   z  }|j                  |      }|S r5   )
r9   r:   r   r;   Tr<   r=   r&   r>   r*   r?   s           r1   rG   zDoraEmbeddingVariant.merge_safe   s     &&
..~> ((8_[,..!_DVX 	 	~.l;[I22>BIIKW!&&q"- K,$>?
]]:.
r3   c                   |j                   }| j                  |      }| j                  |   j                  ||j                  d      j                         }| j                  | d|       | j                  |   j                  |z  }|j                  dd      }||j                  |z   z  }|j                  |      }||_	        y r5   )r9   r:   r   r;   rr   r<   r=   r&   r>   rI   r*   r?   s           r1   rJ   z!DoraEmbeddingVariant.merge_unsafe   s     &&
..~>((8_[,..!_DVX 	 	~.l;[I22>BIIKW!&&q"- K$4$4|$CD
]]:.
%r3   c                   |j                   }| j                  |      }| j                  | d      }| j                  |   j                  |z  }|j
                  |j                  dd      z  |z
  }|j                  |      }|S )Nr7   r   r8   rL   r?   s           r1   rN   zDoraEmbeddingVariant.unmerge   s     &&
..~>''>*:,(GH22>BIIKW %%(8(8B(??,N
]]:.
r3   c                   | j                   |   j                  }| j                  |   j                  }| j                  |   } | j                  |   ||||| j                         | j                        \  }}||z  |z   }|S )N)r    r!   r"   r   embed_fn)ro   rr   rp   r"   r   r,   _embed)	r-   r@   rX   rY   embedding_Aembedding_Br"   mag_norm_scaledora_results	            r1   r[   zDoraEmbeddingVariant.forward   s    --n=??--n=??..0&Rf&B&B>&R,,.]]'
#  &(;6r3   N)r-   r   r.   r^   r/   r   r_   r`   )r-   r   r@   r^   rA   rb   r_   rb   )r-   r   r@   r^   rA   rb   r_   r`   )
r-   r   r@   r^   rX   rb   rY   rb   r_   rb   re   rj   r3   r1   rl   rl      sf    @ @  ( & &$    r3   rl   c                  f    e Zd Zedd       Zedd       Zed	d       Zedd       Zed
d       Zy)_DoraConvNdVariantc                0   | j                   | j                  d d  dz   | _        | j                  |   j                  }| j                  |   j                  }| j
                  |   }|j                  | j                         |||       || j                   |<   y )Nr   rn   )r   r$   r    r&   r!   r"   r+   r,   )r-   r.   r0   r    r!   r"   s         r1   init_convd_variantz%_DoraConvNdVariant.init_convd_variant   s    ''/)/)C)CA)FIc)cF&|,33|,33...6+@+@+B6Z`jqr5?$$\2r3   c                h   |j                   }| j                  |      }| j                  |   j                  ||d      j	                         }| j                  | d|       | j                  |   j                  |z  } |j                  | j                          ||z   z  }|j                  |      }|S Nr   r6   r7   )
r9   r:   r   r;   r<   r=   r&   r>   _get_dora_factor_viewr*   r?   s           r1   rG   z_DoraConvNdVariant.merge_safe   s     &&
..~> ((8HHVblmHnuuw 	 	~.l;[I22>BIIKW%[%%v'C'C'EF+XdJde
]]:.
r3   c                   |j                   }| j                  |      }| j                  |   j                  ||d      j	                         }| j                  | d|       | j                  |   j                  |z  } |j                  | j                          |j                  |z   z  }|j                  |      }||_	        y r   )r9   r:   r   r;   r<   r=   r&   r>   r   rI   r*   r?   s           r1   rJ   z_DoraConvNdVariant.merge_unsafe   s     &&
..~> ((8HHVblmHnuuw 	 	~.l;[I22>BIIKW%[%%v'C'C'EF+JZJZ]iJij
]]:.
%r3   c                   |j                   }| j                  |      }| j                  | d      }| j                  |   j                  |z  }|j
                   |j                  | j                          z  |z
  }|j                  |      }|S )Nr7   )	r9   r:   rM   r   r&   rI   r>   r   r*   r?   s           r1   rN   z_DoraConvNdVariant.unmerge	  s     &&
..~>''>*:,(GH22>BIIKW %%(8(8(8&:V:V:X(YY\hh
]]:.
r3   c           	     <   | j                   |   }| j                  |   }| j                  |   }| j                  |   }t	        |t
        j                        s| j                  s|}n
 ||      }d }| | j                  |   ||||| j                         |      z   }|S rP   rR   rW   s	            r1   r[   z_DoraConvNdVariant.forward  r\   r3   N)r-   r   r.   r^   r0   z	nn.Moduler_   r`   )r-   r   r@   r^   rA   rb   r_   rb   )r-   r   r@   r^   rA   rb   r_   r`   )
r-   r   r@   r^   rX   rb   rY   rb   r_   rb   )	rf   rg   rh   ri   r   rG   rJ   rN   r[   rj   r3   r1   r}   r}      sf    	@ 	@  " & &     r3   r}   c                      e Zd Zedd       Zy)DoraConv1dVariantc                L    t        d      }t        j                  | ||       y NFr   )r0   )r	   r}   r   r-   r.   r/   r0   s       r1   r2   zDoraConv1dVariant.init,  !    $E:
--flz-Zr3   N)r-   r   r.   r^   r/   r   r_   r`   rf   rg   rh   ri   r2   rj   r3   r1   r   r   +      [ [r3   r   c                      e Zd Zedd       Zy)DoraConv2dVariantc                L    t        d      }t        j                  | ||       y r   )r
   r}   r   r   s       r1   r2   zDoraConv2dVariant.init3  r   r3   N)r-   r   r.   r^   r/   r   r_   r`   r   rj   r3   r1   r   r   2  r   r3   r   c                      e Zd Zedd       Zy)DoraConv3dVariantc                L    t        d      }t        j                  | ||       y r   )r   r}   r   r   s       r1   r2   zDoraConv3dVariant.init:  r   r3   N)r-   r   r.   r^   r/   r   r_   r`   r   rj   r3   r1   r   r   9  r   r3   r   c                  x    e Zd Zedd       Zed	d       Zed
d       Zedd       Zed
d       Zedd       Z	y)QALoraLinearVariantc                H   d|vrt        d      | j                  4| j                  |d   z  dk7  rt        d| j                   d|d    d      |d   }d| j                  vr| j                  dz   | _        t        | d      si | _        || j                  |<   | j
                  |   }|j                  }|j                  j                  }|j                  j                  }t        j                  |j                  | j                  |   z  |d	||
      }|| j
                  |<   y)ao  
        Initializes QALoRA specific parameters for a given adapter.

        Args:
            module (Linear): The linear module to be adapted.
            adapter_name (str): The name of the adapter.
            **kwargs: Additional keyword arguments.
                qalora_group_size (int): The size of groups for pooling. This is expected to be passed.
        qalora_group_sizezx`use_qalora=True` requires 'qalora_group_size' to be provided in kwargs. Please ensure it is passed from the LoraConfig.Nr   z1`use_qalora=True` requires `module.in_features` (z)) to bedivisible by 'qalora_group_size' ())r   F)biasr(   r9   )
ValueErrorin_featuresother_param_nameshasattrr   r    out_featuresr&   r(   r9   r   r   )	r-   r.   r/   r   old_lora_A_layerrr(   r9   new_lora_A_layers	            r1   r2   zQALoraLinearVariant.initA  sO    f,C 
 )f.@.@6J]C^.^bc.cCFDVDVCW X55;<O5P4QQRT  ##67f&>&>>'-'?'?BX'XF$v23')F$1B  .!==6))!((// ''--99((F,D,D\,RR
 '7l#r3   c                    t        d      )Nz;QALoRA for GPTQ layers does not support 'get_delta_weight'.NotImplementedError)r-   r@   s     r1   r:   z$QALoraLinearVariant.get_delta_weightn  s    !"_``r3   c                    t        d      )Nz5QALoRA for GPTQ layers does not support 'safe_merge'.r   r-   r@   rA   s      r1   rG   zQALoraLinearVariant.merge_safer  s    !"YZZr3   c                    t        d      )Nz7QALoRA for GPTQ layers does not support 'merge_unsafe'.r   r   s      r1   rJ   z QALoraLinearVariant.merge_unsafev  s    !"[\\r3   c                    t        d      )Nz2QALoRA for GPTQ layers does not support 'unmerge'.r   r   s      r1   rN   zQALoraLinearVariant.unmergez  s    !"VWWr3   c                   | j                   |   j                  }| j                  |   j                  }| j                  |   }| j                  |   }| j
                  |   }| j                  r"t        |t        j                        s ||      n|}	|	j                  }
t        |
      dkD  r|	j                  d| j                        }n|	}|j                  \  }}||z  }|j                  |||      j                  d      }||z  }||j                         z  |j                         z  |z  }t        |
      dkD  r'|j                  |
d d |j!                  d      fz         }||z   S )N   r8   )dim)r    r&   r!   rS   r"   r   rV   rT   r   rU   shapelenr>   r   meantsize)r-   r@   rX   rY   lora_A_weightlora_B_weightrZ   r"   
group_size	x_dropped
orig_shapex_flat
batch_sizer   pooled_featuresx_pooledx_pooled_scaleddeltas                     r1   r[   zQALoraLinearVariant.forward~  sL   n5<<n5<<%%n5..0--n=
"(//*Wbkk:ZGAJ`a	__
 z?Q^^B(:(:;FF"(,,
K%3;;z?JGLLQRLS"_4  -//"33moo6GG'Q z?QJJz#2%**R.1BBCE~r3   Nr]   )r-   r   r@   r^   r_   rb   ra   rc   rd   )
rf   rg   rh   ri   r2   r:   rG   rJ   rN   r[   rj   r3   r1   r   r   @  s    *7 *7X a a [ [ ] ] X X  r3   r   )
__future__r   typingr   torchaccelerate.utils.importsr   r   peft.utils.otherr   dorar	   r
   r   r   r   layerr   r   r   r   r   r   r   r   rl   r}   r   r   r   r   rj   r3   r1   <module>r      s    #   5  & h h R R Rf fRQ, QhO Od[* [[* [[* []+ ]r3   