
    bi                        d dl mZ d dlZd dlmZmZ d dlZd dlmZ ddl	m
Z
 eej                  ej                  ej                  f   Z G d dej                        Z G d d	ej                        Zy)
    )annotationsN)OptionalUnion   )XLoraConfigc                  &     e Zd Zd fd	Zd Z xZS )TemperatureScaledSoftmaxc                f    t         |           || _        t        j                  d      | _        y )N)dim)super__init__temperaturennSoftmaxsoftmax)selfr   	__class__s     W/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/peft/tuners/xlora/classifier.pyr   z!TemperatureScaledSoftmax.__init__   s&    &zzb)    c                B    || j                   z  }| j                  |      S )N)r   r   )r   logitsscaled_logitss      r   forwardz TemperatureScaledSoftmax.forward"   s!    !1!11||M**r   )g      ?)__name__
__module____qualname__r   r   __classcell__r   s   @r   r	   r	      s    *
+r   r	   c                  x     e Zd ZdZ	 	 	 	 	 	 	 	 	 	 d fdZ	 	 d	 	 	 	 	 d	dZ	 	 d	 	 	 	 	 d	dZd
dZddZ xZ	S )XLoraClassifierz7
    A classifier to select LoRA layers for XLora.
    c                   t         
|           || _        || _        || _        g | _        t        | j                  j                        | _        |j                  | _
        d| _        t        |j                               j                  | _        |j                  dkD  }g }| j                  j                   dk(  r|j"                  rOt%        j&                  |j(                  ||z  d      j+                  |      j+                  | j                        }nt%        j&                  |j(                  |d      j+                  |      j+                  | j                        }nT| j                  j                   dk  rt-        d      |j/                  t%        j&                  |j(                  |j0                  d      j+                  |      j+                  | j                               |j/                  t%        j2                                |r/|j/                  t%        j4                  |j                  	             t7        |j                   d
z
        D ]  }	|j/                  t%        j&                  |j0                  |j0                  d      j+                  |      j+                  | j                               |j/                  t%        j2                                |s|j/                  t%        j4                  |j                  	              |j"                  rNt%        j&                  |j0                  ||z  d      j+                  |      j+                  | j                        }nJt%        j&                  |j0                  |d      j+                  |      j+                  | j                        }t%        j8                  g || | _        y)z
        Construct an X-LoRA classifier from a model, config and some metadata. Note that n_layers is the number of LoRA
        adapter layers, not the number of model layers.
        )r   Fg        r   T)biasr   z'X-LoRA depth must be strictly positive.)p   N)r   r   	n_classesn_layersconfiglog_scalingsr	   softmax_temperaturer   scaling_pass_valueoverride_scaling_pass_valuescalings_loggingnext
parametersdtypexlora_dropout_pxlora_depthlayerwise_scalingsr   Linearhidden_sizeto
ValueErrorappend
xlora_sizeReLUDropoutrange
Sequentiallayers)r   modelr(   r&   r'   deviceadd_dropoutr>   last_r   s             r   r   zXLoraClassifier.__init__.   s    	" /DKK<[<[\393L3L( %%**,-33
,,s2;;""a'((yy!3!3Y5IPTUXXY_`ccdhdndnoyy!3!3YTJMMfUXXY]YcYcd{{&&!+ !JKKMM"))F$6$68I8IPTUXXY_`ccdhdndnopMM"'')$bjj6+A+ABC6--12 Hbii(9(96;L;LSWX[[\bcffgkgqgqrsbggi(MM"**v/E/E"FGH ((yy!2!2I4HtTWWX^_bbcgcmcmnyy!2!2IDILLVTWWX\XbXbcmm2V2T2r   c                V   |+|j                   d   }|j                  }|j                   d   }n*|j                   d   }|j                  }|j                   d   }t        j                  ||| j                  | j
                  f| j                        j                  || j                        S )aP  
        Make some dummy scalings for the scalings pass (the one to get the logits for the X-LoRA classifier). These are
        of shape (batch_size, seq_len, n_layers, n_classes) and filled with the override scalings pass value. Note that
        n_layers is the number of LoRA adapter layers, not the number of model layers.
        r   r   )r@   r0   )	shaper@   torchfullr'   r&   r,   r6   r0   )r   	input_idsinputs_embedsargskwargs
batch_sizer@   seq_lens           r   make_dummy_scalingsz#XLoraClassifier.make_dummy_scalingse   s      "+J%%Fooa(G&,,Q/J"))F#))!,Gzz$--@,,
 "F$**"
-	.r   c                R   ||j                   d   }|j                   d   }n|j                   d   }|j                   d   }|j                  }|d   }	| j                  j                  |	      }
| j                  j
                  s/|
j                  d      }
|
j                  dd| j                  d      }
|
j                  ||| j                  | j                        }| j                  j                  r| j                  |      }| j                  r| j                  j                  |       |S )zt
        Using the hidden states of the model, predict `n_classes` LoRA alpha values. Returns the scalings.
        r   r   r   r%   )rE   hidden_statesr>   r   r(   r3   	unsqueezeexpandr'   reshaper&   enable_softmaxr   r-   r)   r8   )r   resultrH   rI   rJ   rK   rL   rM   rP   hidden_stater   scalingss               r   r   zXLoraClassifier.forward   s     "+Jooa(G&,,Q/J#))!,G,,$R( $$\2
 {{--%%a(F]]2r4=="=F >>*gt}}dnnU ;;%%||H-H  $$X.r   c                    i }t        | j                        D ]P  \  }}|j                  d   }||vr
|g|gf||<   #||   d   j                  |       ||   d   j                  |       R |S )aL  
        Returns bucketed scalings, bucketed by seq_len. Each value consists of the positions (the first) and the
        associated tensors. The positions are paired with the associated tensors and give the position in the scaling
        log. Each scaling is a tensor of shape (batch_size, seq_len, n_layers, n_classes)).
        r   r   )	enumerater)   rE   r8   )r   seqlens_mapiscalingrM   s        r   _get_bucketed_scalingsz&XLoraClassifier._get_bucketed_scalings   s     HJ#D$5$56 	8JAwmmA&Gk))*gY'7G$G$Q'..q1G$Q'..w7	8 r   c                v    |d| j                   z  | _        n|| _        | j                  | j                  _        y )Nr   )r&   r,   r(   r+   )r   values     r    _set_override_scaling_pass_valuez0XLoraClassifier._set_override_scaling_pass_value   s2    =/04>>/AD,/4D,)-)I)I&r   )
r?   z	nn.Moduler(   r   r&   intr'   ra   r@   ztorch.device)NN)rH   zOptional[torch.LongTensor]rI   zOptional[torch.FloatTensor]returnztorch.Tensor)rb   z/dict[int, tuple[list[int], list[torch.Tensor]]])r_   zUnion[Number, None])
r   r   r   __doc__r   rN   r   r]   r`   r   r   s   @r   r!   r!   )   s    5353 53 	53
 53 53r 1559.-. 3. 
.: 1559	, ., 3	, 
,\"Jr   r!   )
__future__r   builtinstypingr   r   rF   torch.nnr   r(   r   ra   floatboolNumberModuler	   r!    r   r   <module>rm      s\    #  "    
x||X^^X]]:	;
+ryy 
+ZJbii ZJr   