
    bi(                         d dl Z d dlmZmZ d dlZd dlmZ d dlmc mZ	 ddl
mZ  G d dej                        Z G d de      Z G d	 d
e      Zy)    N)OptionalUnion   )TRANSFORMERS_MODEL_CONFIGc                   D     e Zd ZdZej
                  fdedef fdZ xZ	S )_BaseAdaptedAttentionzEBase module, which defines adaption prompts for multiple model types.
model_typeadapter_lenc           	         t        |t              rt        d      t        |           || _        || _        || _        t        |j                               j                  }t        | j                  d      r| j                  j                  }n | j                  j                  j                  }t        | j                  d      r| j                  j                  | _        n%| j                  j                  j                  | _        t!        j"                  t%        j&                  d||||      j)                               | _        t!        j"                  t%        j,                  d||            | _        y)aT  
        Initialize object.

        Args:
            model_type: The transformer model type. This is used to retrieve the right method to
                compute query states.
            adapter_len: The length of the adaption prompt to insert.
            model: The original transformer attention module that is being wrapped.
        z)Unable to stack multiple adaption promptshidden_size	num_headsr   )devicedtypeN)
isinstancer   
ValueErrorsuper__init__r	   modelr
   next
parametersr   hasattrr   configr   num_attention_headsnn	Parametertorchemptynormal_adaption_promptzerosadaption_gate)selfr	   r
   r   target_dtyper   r   	__class__s          \/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/adaption_prompt/layer.pyr   z_BaseAdaptedAttention.__init__   s    e23HII$
& e&&()00 4::}-**00K**++77K4::{+!ZZ11DN!ZZ..BBDN!||KK;F,W__a 
  \\%++al*[\    )
__name__
__module____qualname____doc__r   float32strintr   __classcell__r$   s   @r%   r   r      s)    ONSmm (]3 (]S (] (]r&   r   c                   T    e Zd ZdZ fdZ	 	 	 	 	 	 	 ddeeej                        deeej                        deej                     deej                     deej                     deej                     d	ee
   d
ee
   deeej                  eej                     f   df   fdZ xZS )AdaptedAttentionGPTzEThis module wraps a GPT2Attention module and injects adaption promptsc                    |j                   j                  j                  t        j                  t        j
                  fvr |j                   j                  j                  nt        j                  }t        | !  ||||       y N)r#   )	c_projweightr   r   int8uint8r+   r   r   r"   r	   r
   r   r#   r$   s        r%   r   zAdaptedAttentionGPT.__init__J   c    ).)<)<)B)B5::W\WbWbJc)cELL%%iniviv 	 	[%lSr&   hidden_states
layer_pastattention_mask	head_maskencoder_hidden_statesencoder_attention_mask	use_cacheoutput_attentionsreturn.c	                 6    | j                   d
|||||||d|	}
	 |
d   }|
dd  }t        | j                     j                  }|j                  d   }|j                  d   }|j                  d   } t        | j                   |      | j                        j                  |d      \  }}}|j                  d| j                  | j                  | j                   j                        j                  |ddd      j                  dd      }|j                  d| j                  | j                  | j                   j                        j                  |ddd      j                  dd      }t        | j                     j                  } || j                   ||      }|j                  }t!        j"                  ||j                  dd      j%                  |            t'        j(                  | j                   j                        z  }| j*                  t-        j.                  |dt         j0                  	      j%                  |      z  }t!        j"                  ||      j                  dd      j3                  ||d      }||z   }|j%                  |      }|f|z   }|S )N)r:   r<   r=   r>   r?   r@   rA   r   r      dim)r:   r>      rF   r    )r   r   r	   k_proj_layershapegetattrr   splitviewr
   r   head_dimrepeat	transposecompute_query_statesr   r   matmultomathsqrtr!   Fsoftmaxr+   reshape)r"   r:   r;   r<   r=   r>   r?   r@   rA   kwargsattn_outputsattn_outputadd_outputsc_attn_layerbszq_len	embed_dim_keyvalue	adapter_k	adapter_vrS   query_statesprevious_dtypescoresadapter_outputhidden_stateoutputs                                r%   forwardzAdaptedAttentionGPT.forwardP   s    "tzz 	
')"7#9/	
 	
	 #1o"12&0ANN"!!!$%%a(	9

L9$:N:NOUUV_efUg3 HHQ(($..$**:M:MNUUVY[\^_abcmmnoqrs 	 JJq$**DNNDJJ<O<OPWWX[]^`acdeoopqstu 	  9I^^+JJmK`
 &++lI,?,?1,E,H,H,XY\`\e\eJJ]
 

 ##aiiBemm&T&W&WXf&ggfi8BB1aHPPQTV[]_` #^3 $~6 ;.r&   )NNNNNFF)r'   r(   r)   r*   r   r   tupler   FloatTensorTensorboolr   rn   r.   r/   s   @r%   r1   r1   G   s    OT 596:158<>B$),1De&7&7 89D U5<<01D !!2!23	D
 E--.D  (5D !)):): ;D D>D $D>D 
uU\\5#667<	=Dr&   r1   c                   (     e Zd ZdZ fdZd Z xZS )AdaptedAttentionzGThis module wraps a LLamaAttention module and injects adaption prompts.c                    |j                   j                  j                  t        j                  t        j
                  fvr |j                   j                  j                  nt        j                  }t        | !  ||||       y r3   )	q_projr5   r   r   r6   r7   r+   r   r   r8   s        r%   r   zAdaptedAttention.__init__   r9   r&   c                    |j                  dd      rt        d       | j                  di |^}}|j                  d   }|j                  d   }|j                  d   }t        | j
                     j                  }t        | j
                     j                  }t        | j
                     j                  }	| j                  j                  j                  | j                  j                  j                  z  }
||k(  r< t        | j                  |      | j                        j                  |d      \  }}}nL t        | j                  |      | j                        } t        | j                  |      | j                        }t        | j                  d      r| j                  j                   }n | j                  j"                  j$                  }|j'                  d| j(                  ||
z  | j                  j*                        j-                  |ddd      j/                  dd      }|j'                  d| j(                  ||
z  | j                  j*                        j-                  |ddd      j/                  dd      }t1        j2                  ||
d	      }t1        j2                  ||
d	      }t        | j
                     j4                  } |dd
| j                  i|}|j6                  }t1        j8                  ||j/                  dd      j;                  |            t=        j>                  | j                  j*                        z  }| j@                  tC        jD                  |dt0        jF                        j;                  |      z  }t1        j8                  ||      j/                  dd      jI                  ||d      }|	 t        | j                  |	      |      }||z   }|j;                  |      }|g|S )aK  
        Forward pass for the adapter which wraps the original LlamaAttention module.

        "Official" paper implementation:
        https://github.com/ZrrSkywalker/LLaMA-Adapter/blob/41c3546fe1997ab8a65809dc8d8f9252b19d9faf/llama/model.py#L141

        Args:
            kwargs: See the original LlamaAttention module.
        output_attentionFz,output_attention is not currently supported.r   r   rD   rE   r   )repeatsrF   r   rG   rH   rI   rJ   )%getNotImplementedErrorr   rL   r   r	   rK   v_proj_layero_proj_layerk_projin_featuresout_featuresrM   r   rN   r   r   r   r   rO   r
   rP   rQ   rR   r   repeat_interleaverS   r   rT   rU   rV   rW   r!   rX   rY   r+   rZ   )r"   r[   rm   rc   r`   ra   rb   rK   r|   r}   factorrd   re   r   rf   rg   rS   rh   ri   rj   rk   s                        r%   rn   zAdaptedAttention.forward   sS    ::(%0%&TUUTZZ)&)
ll1oQLLO	0ANN0ANN0ANNJJ))TZZ->->-K-KK 	 <'=GDJJ=d>R>RSYYZcijYkMAsE3'$**l3D4H4HIC5GDJJ5d6J6JKE4::{+

,,I

))==I HHQ((9+>ATATUVCAq!Yq!_ 	 JJq$**Y&-@4::CVCVWVCAq!Yq!_ 	 ++Iv1M	++Iv1M	8I^^+G$**GG%++ lI,?,?1,E,H,H,XY\`\e\eJJ]
 

 ##aiiBemm&T&W&WXf&ggfi8BB1aHPPQTV[]_` #>WTZZ>~NN .( >*zzr&   )r'   r(   r)   r*   r   rn   r.   r/   s   @r%   rt   rt      s    QTLr&   rt   )rV   typingr   r   r   torch.nnr   torch.nn.functional
functionalrX   r   r   Moduler   r1   rt   rJ   r&   r%   <module>r      sN     "     -+]BII +]\M/ M`U, Ur&   