
    bi                         d dl Z d dlmZ d dlZd dlmZ d dlmZ ddlm	Z	 ddl
mZ  G d de      Z G d	 d
ej                  e      Zy)    N)Any)BaseTunerLayer   )
PolyConfig)
get_routerc                   @    e Zd ZdZdZdej                  fdZd Zd Z	y)	PolyLayer)poly_lora_Apoly_lora_Bpoly_router)rn_tasksn_skillsn_splits
base_layerc                    || _         i | _        i | _        i | _        i | _        i | _        t        j                         | _        t        j                         | _
        t        j                         | _        || _        | j                         }t        |t        j                        r|j                   |j"                  }}nt%        dt'        |             || _        || _        y )NzUnsupported layer type )r   r   r   r   r   	poly_typenn
ModuleDictr   ParameterDictr
   r   kwargsget_base_layer
isinstanceLinearin_featuresout_features
ValueErrortype)selfr   r   r   r   s        Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/poly/layer.py__init__zPolyLayer.__init__!   s    $==?++-++-((*
j")),(2(>(>
@W@WK6tJ7G6HIJJ&(    c           	         |j                   dk  rt        d|j                          |j                   | j                   |<   |j                  | j                  |<   |j                  | j                  |<   |j                  | j                  |<   |j
                  | j
                  |<   t        j                  t        j                  |j                  |j                  | j                  |j                  z  |j                               | j                  |<   t        j                  t        j                  |j                  |j                  |j                   | j                  |j                  z              | j                  |<   t        |      | j                  |<   | j!                  ||j"                         | j%                  |       | j'                  | j(                         y )Nr   z?`r` should be a positive integer value but the value passed is )init_weights)r   r   r   r   r   r   r   	Parametertorchemptyr   r
   r   r   r   r   reset_poly_parametersr$   %_move_adapter_to_device_of_base_layerset_adapteractive_adapters)r   adapter_namepoly_configs      r    update_layerzPolyLayer.update_layer6   s   ==A^_j_l_l^mnoo*}}|%0%8%8\"&1&:&:l#&1&:&:l#'2'<'<|$)+KK$$$$  K$8$88	*
& *,KK$$$$!![%9%99	*
& *4K)@&""<k>V>V"W22<@--.r"   c           	         || j                   j                         v r| j                   |   j                  \  }}}}t        |      D ]  }t        |      D ]  }t	        j
                  ||f      }	t        j                  j                  j                  |	t        j                  d             |	j                  | j                   |   j                  ||d d d d f<     |r7t        j                  j                  j                  | j                  |          n| j                  |   j                  \  }}}}t        |      D ]  }t        |      D ]  }t	        j
                  ||f      }	t        j                  j                  j                  |	t        j                  d             |	j                  | j                  |   j                  ||d d d d f<     | j                  |   j!                          y y )N   )a)r
   keysshaperanger&   r'   r   initkaiming_uniform_mathsqrtTdatazeros_r   r   reset)
r   r,   r$   r   r   dr   skillsplitparams
             r    r(   zPolyLayer.reset_poly_parametersW   s   4++0022 (,'7'7'E'K'K$Hh1x V"8_ VE!KKA/EHHMM225DIIaL2INSggD$$\277ua8JKVV $$T%5%5l%CD ,0+;+;L+I+O+O((Aq"8_ ZE!&x Z %QF 366u		!6MRWRYRY((6;;E5!Q<NOZZ \*002/ 3r"   N)
__name__
__module____qualname__adapter_layer_namesother_param_namesr   Moduler!   r.   r(    r"   r    r	   r	      s)    G@)299 )*/B3r"   r	   c                        e Zd Zdededdf fdZdddej                  ded	ej                  d
edej                  f
dZ	def fdZ
 xZS )r   r,   r-   returnNc                     t         |           t        j                  | |fi | || _        | j	                  ||       y )N)superr!   r	   _active_adapterr.   )r   r   r,   r-   r   	__class__s        r    r!   zLinear.__init__t   s=     	46v6+,4r"   )task_idsxargsrN   r   c                >   |j                   }| j                  r | j                  |g|i |}n[ | j                  |g|i |}| j                  D ]5  }|| j                  j                         vr!| j                  |   }| j                  |   }	| j                  |   }
| j                  |   } |	||      }|j                         \  }}}t        j                  d||
f      }t        j                  d||f      }|j                  || j                  |      }|j                  dd      j                  ||| j                        }|j!                  |j                         }||j#                  |      j#                  |      |z  z  }8 |j!                  |      }|S )N)rN   	input_idszbqs,qsdr->bqdrzbqs,qsrd->bqrdr      )dtypedisable_adaptersr   r+   r
   r2   r   r   r   sizer&   einsumreshaper   	transposer   tobmm)r   rO   rN   rP   r   previous_dtyperesultactive_adapterr   r   r
   r   mixing_weightsbsr   r   ABs                     r    forwardzLinear.forward   s     $T__Q888F$T__Q888F"&"6"6 .!)9)9)>)>)@@FF>*"..~>"..~>"..~> "-h!!L)7)<)<)>&Hh LL!1NK3PQLL!1NK3PQIIb$"2"2A6KK1%--b!T5F5FGDDM!%%(,,q/A--/.2 >*r"   c                 *    t         |          }d|z   S )Nzpoly.)rK   __repr__)r   reprM   s     r    re   zLinear.__repr__   s    g }r"   )rA   rB   rC   strr   r!   r&   Tensorr   rc   re   __classcell__)rM   s   @r    r   r   r   sw    5 5  	5 
5 MQ    c  U\\  \_  didpdp  D#  r"   r   )r7   typingr   r&   torch.nnr   peft.tuners.tuners_utilsr   configr   routerr   r	   rF   r   rG   r"   r    <module>ro      s=        3  T3 T3n3RYY	 3r"   