
    bi                     T    d dl mZ d dlmZ ddlmZmZ ddlmZ d
dZ	d Z
ddZd	 Zy)    )
MethodTypeN   )is_hpu_availableis_transformer_engine_available)GatheredParametersc                 H   t               st        d      t               r&ddl}t	        |d      st
        j                  |_        nddlm} | j                         D ]  \  }}t        |t
        j                        r|r|r|j                  du}|j                  g}|r|j                  |j                         t        |d      5  t!        d |j                  j"                  D              r	 ddd        y|j                  |j$                  |j&                  ||j                  j(                        }	|	j                  j+                  |j                         |r%|	j                  j+                  |j                         t-        | ||	       ddd       ?t        |t
        j                        r|r|rt        |j                  |j                  gd      5  |j                  |j.                  d   |j0                  |j                  j(                        }	|	j                  j+                  |j                         |	j                  j+                  |j                         ddd       t-        | |	       !t        ||j                        r|s|r|j                  du}t        j                  |j$                  |j&                  ||j                  j(                        }
|
j                  j+                  |j                         |r%|
j                  j+                  |j                         t-        | ||
       t        ||j                        r|s|rt        j                  |j.                  d   |j0                  |j                  j(                        }
|
j                  j+                  |j                         |
j                  j+                  |j                         t-        | ||
       t3        ||||	        y# 1 sw Y   xY w# 1 sw Y   xY w)
zu
    Recursively converts the linear and layernorm layers of a model to their `transformers_engine` counterpart.
    zBUsing `convert_model` requires transformer_engine to be installed.r   N	LayerNorm)modifier_rankc              3   ,   K   | ]  }|d z  dk7    yw)   r   N ).0ps     ^/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/accelerate/utils/transformer_engine.py	<genexpr>z convert_model.<locals>.<genexpr>2   s     @qq2v{@s   )biasparams_dtype)epsr   )to_transformer_engine_convert_linear_convert_ln)r   ImportErrorr   intel_transformer_enginehasattrnnr	   transformer_engine.pytorchpytorchnamed_children
isinstanceLinearr   weightappendr   anyshapein_featuresout_featuresdtypecopy_setattrnormalized_shaper   convert_model)modelr   r   r   tenamemodulehas_biasparams_to_gather	te_module
new_modules              r   r+   r+      s$    +,^__-r;'<<BL/,,. 0ffbii(-B{{$.H & ''4#$4AF 
0@FMM,?,?@@
0 
0 II&&(;(;(Y_YfYfYlYl & 	   &&v}}5NN((5tY/
0 
0 -2GK#V]]FKK$@PQR 2LL)@)@)CbhbobobubuLv	  &&v}}5$$V[[12
 E4+		*3H_{{$.H""F$7$7hU[UbUbUhUhJ ##FMM2%%fkk2E4,-6KP[f&=&=a&@fjj_e_l_l_r_rsJ##FMM2OO!!&++.E4,&; /'	W0
0 
02 2s    	(P
;BP
B
P
P	P!	c                     t               st        d      t               rddl}|j                  }n)ddlm} |j                  |j                  |j                  f}| j                         D ]  }t        ||      s y y)zS
    Returns whether a given model has some `transformer_engine` layer or not.
    zRUsing `has_transformer_engine_layers` requires transformer_engine to be installed.r   NTF)r   r   r   r   r    r   r   r	   TransformerLayermodulesr   )r,   r-   module_cls_to_checkms       r   has_transformer_engine_layersr9   ]   so     +,noo- ii/!||RYY8K8KL]]_ a,-     c                      t               st        d      t               rddlm nddlm  fd} |_        |S )z
    Wrapper for a model's forward method to apply FP8 autocast. Is context aware, meaning that by default it will
    disable FP8 autocast during eval mode, which is generally better for more accurate metrics.
    zLUsing `contextual_fp8_autocast` requires transformer_engine to be installed.r   )fp8_autocastc                 t    xs | j                   } |      5   |i |cd d d        S # 1 sw Y   y xY w)N)enabled
fp8_recipe)training)selfargskwargsr>   r<   r?   model_forwarduse_during_evals       r   forwardz(contextual_fp8_autocast.<locals>.forward   s=    !2T]]'jA 	2 $1&1	2 	2 	2s   .7)r   r   r   r   r<   r   __wrapped__)rD   r?   rE   rF   r<   s   ``` @r   contextual_fp8_autocastrH   t   s7    
 +,hii9;2 (GNr:   c                    t               st        d      t               rddlm} n	ddlmc m} ||j                         ni }d|v rt        |j                  |d         |d<   |j                  dd      } |j                  di |}t        | j                  ||      }t        | j                  d      rt        ||       | _        | S || _        | S )	zC
    Applies FP8 context manager to the model's forward method
    zGUsing `apply_fp8_autowrap` requires transformer_engine to be installed.r   N
fp8_formatuse_autocast_during_evalF__func__r   )r   r   r   intel_transformer_engine.reciperecipe transformer_engine.common.recipecommon	to_kwargsgetattrFormatpopDelayedScalingrH   rF   r   r   )r,   fp8_recipe_handler	te_reciperC   rE   r?   new_forwards          r   apply_fp8_autowraprY      s     +,cdd;<</A/M))+SUFv&y'7'79MN|jj!;UCO)))3F3J)%--_UKu}}j)";6 L $Lr:   )TTT)F)typesr   torch.nnr   importsr   r   
operationsr   r+   r9   rH   rY   r   r:   r   <module>r^      s)      F *@F.0r:   