
    big#                        d dl mZ d dlmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZ  G d d	      Z G d
 de      Z G d de      Z G d de      Zy)    )annotations)AnyCallableOptionalN)Tensor)lora   )XLoraConfigc                  N    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 ddZ	 edd       Z	 ddZy)	
XLoraLayerz
    A XLoraLayer wraps any LoraLayer and performs the XLora operation on the LoRA adaptors specified. Its primary API
    is the forward method, which uses the scalings to execute the XLora algorithm.
    c                J    || _         || _        || _        || _        || _        y N)modeltarget_forwardtargetlayer_numberconfig)selfr   r   r   r   r   s         R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/xlora/layer.py__init__zXLoraLayer.__init__!   s)     
,(    c                B    |d d d d |f   j                  d      }| |z  S )N)	unsqueeze)xscalings_layeradapterscalingss       r   apply_scalings_to_xzXLoraLayer.apply_scalings_to_x3   s)     "!Q-0::2>8|r   c                   |d d d d | j                   d d f   }| j                  j                  t        j                  || j                  j                  d      \  }}t        j
                  |t        j                        }|j                  d|d       ||j                  |j                        z  }| j                  j                  r$|dk7  }t        j                  ||   d      }|||<   |S )Nr   )kdim)dtypeTr   )r"   )r   r   
top_k_loratorchtopk
zeros_likeboolscatter_tor#   enable_softmax_topksoftmax)r   r   xlora_scalings_topk_indicesmasknonzero_masksoftmax_res_nonzeros           r   get_maybe_topk_scalingsz"XLoraLayer.get_maybe_topk_scalings?   s    !)!Q0A0A1*D!E;;!!-#jj4;;;Q;QWYZOA| ##N%**EDMM"lD1+dggn6J6J.KKN;;**)Q.L"'--|0LRT"U+>N<(r   N)r   	nn.Moduler   zlora.LoraLayerr   Callable[..., Any]r   intr   r
   returnNone)r   torch.Tensorr   r9   r   r6   r7   r9   )r7   r9   )__name__
__module____qualname____doc__r   staticmethodr   r3    r   r   r   r      si    
  +	
   
  
r   r   c                  F     e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d fdZddddZ xZS )XLoraLinearLayerc                ,    t         |   |||||       y r   superr   r   r   r   r   r   r   	__class__s         r   r   zXLoraLinearLayer.__init__U        	fMr   Nr   c          	     h   |j                   }|| j                  |      } | j                  j                  |g|i |}| j                  j                  sJt        | j                  j                        D ]'  \  }}	|	| j                  j                  j                         vr.| j                  j                  |	   rt        d      | j                  j                  |	   }
| j                  j                  |	   }| j                  j                  |	   }| j                  j                  |	   }|j                  |
j                  j                         }|*| j!                  ||      }| j"                  j$                  }n|}d}| | |
 ||                  |z  |z  z  }* |j                  |      }|S 
        This method is designed to be a drop-in-replacement for the LoRA layers' .forward method. To use it, a bound
        method must be created (bound to an instance of the XLoraLayer class).
        7X-LoRA currently does not support LoRA layers with DoRAr	   r#   r3   r   
base_layermerged	enumerateactive_adapterslora_Akeysuse_dora
ValueErrorlora_Blora_dropoutscalingr*   weightr   r   global_scaling_weightr   r   r   argskwargsprevious_dtyper-   result	adapter_nactive_adapterrR   rV   dropoutrX   x_modscaling_weights                   r   forwardzXLoraLinearLayer.forward_   s    !99(CN''';D;F; {{!!-6t{{7R7R-S T)	>!););)@)@)BB;;''7$%^__++N;++N;++22>B++--n=DD,,-' 44Q	RE%)[[%F%FNE%&N&!787B^SS#T& >*r   )r   r4   r   zlora.Linearr   r5   r   r6   r   r
   r7   r8   
r   r   r\   r   r   zOptional[Tensor]r]   r   r7   r   r:   r;   r<   r   re   __classcell__rF   s   @r   rA   rA   T   sZ    NN N +	N
 N N 
N KO " "r   rA   c                  F     e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d fdZddddZ xZS )XLoraEmbeddingLayerc                ,    t         |   |||||       y r   rC   rE   s         r   r   zXLoraEmbeddingLayer.__init__   rG   r   NrH   c                  || j                  |      } | j                  j                  |g|i |}| j                  j                  s,t	        | j                  j
                        D ]	  \  }}|| j                  j                  vr | j                  j                  j                  |d      rt        d      | j                  j                  |   j                  }	| j                  j                  |   j                  }
| j                  j                  |   }| j                  j                  ||	      }|*| j                  ||      }| j                  j                   }n|}d}|||
z  |z  |z  z  } |S )rK   FrL   r	   )r3   r   rN   rO   rP   rQ   lora_embedding_ArT   getrU   Tlora_embedding_BrX   _embedr   r   rZ   )r   r   r   r\   r]   r-   r_   r`   ra   embedding_Aembedding_BrX   after_Aafter_A_modrd   s                  r   re   zXLoraEmbeddingLayer.forward   s[    !99(CN''';D;F; {{!!-6t{{7R7R-S Q)	>!)E)EE;;''++NEB$%^__"kk::>JLL"kk::>JLL++--n=++,,Q<'"&":":7NT]"^K%)[[%F%FN")K%&N;4?.PP!Q$ r   )r   r4   r   zlora.Embeddingr   r5   r   r6   r   r
   r7   r8   rf   rg   ri   s   @r   rk   rk      sZ    NN N +	N
 N N 
N KO  r   rk   c                  F     e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d fdZddddZ xZS )XLoraConv2dLayerc                ,    t         |   |||||       y r   rC   rE   s         r   r   zXLoraConv2dLayer.__init__   rG   r   NrH   c          	     h   |j                   }|| j                  |      } | j                  j                  |g|i |}| j                  j                  sJt        | j                  j                        D ]'  \  }}	|	| j                  j                  j                         vr.| j                  j                  |	   rt        d      | j                  j                  |	   }
| j                  j                  |	   }| j                  j                  |	   }| j                  j                  |	   }|j                  |
j                  j                         }|*| j!                  ||      }| j"                  j$                  }n|}d}| | |
 ||                  |z  |z  z  }* |j                  |      }|S rJ   rM   r[   s                   r   re   zXLoraConv2dLayer.forward   s    !99(CN''';D;F; {{!!-6t{{7R7R-S T)	>!););)@)@)BB;;''7$%^__++N;++N;++22>B++--n=DD,,-' 44Q	RE%)[[%F%FNE%&N&!787B^SS#T& >*r   )r   r4   r   zlora.Conv2dr   r5   r   r6   r   r
   r7   r8   rf   rg   ri   s   @r   rx   rx      sZ    NN N +	N
 N N 
N KO # #r   rx   )
__future__r   typingr   r   r   r%   torch.nnnnr   peft.tunersr   r   r
   r   rA   rk   rx   r?   r   r   <module>r      sP    # * *     6 6r-z -`** *Z.z .r   