
    ukiS                       d dl mZ d dlmZ d dlZd dlZd dlZd dlmZm	Z	 d dl
mZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlZeedf   Z ejB                  Z!ee"df   Z#ee!   Z$ G d d      Z% G d d      Z& e&       Z'eZ(	 ejR                  e(ef   Z*e	e*e%e&f   Z+d Z, eejZ                         G d dej\                               Z-de-_/        d Z0	 	 	 	 d,dZ1ejd                   G d d             Z3d Z4 ejd                  d       G d d             Z5d-dZ6 edd !      	 	 	 	 d.d"       Z7d/d#Z8 ed$d !       e9       fd%       Z: G d& d'e;      Z<d0	 d1d(Z=d) Z>d* Z?d+ Z@y)2    )annotations)SequenceN)AnyUnion)use_cpp_classcacheuse_cpp_method)
xla_client)sdymesh)AxisType)PartitionSpec)sharding.c                  ,    e Zd ZddZddZed        Zy)AUTOc                    || _         y Nr   )selfr   s     R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/named_sharding.py__init__zAUTO.__init__(   s	    DI    c                    t        |      D cg c]  }t        g d       }}t        | j                  j                  |      S c c}w )NTaxesis_open)
mesh_shapedim_shardings)rangeSdyDimSdyArrayr   shape_tuple)r   ndim_r   s       r   _to_sdy_shardingzAUTO._to_sdy_sharding+   sK    #Dk+ T2 +M +tyy44"/1 1+s   Ac                .    | j                   j                  S r   )r   _flat_devices_tupler   s    r   _device_assignmentzAUTO._device_assignment1   s    99(((r   N)r   zmesh_lib.Mesh)r#   intreturnr!   )__name__
__module____qualname__r   r%   propertyr)    r   r   r   r   &   s     1 ) )r   r   c                      e Zd Zd Zy)UnspecifiedValuec                     y)Nr2   r0   r(   s    r   __repr__zUnspecifiedValue.__repr__7   s    r   N)r,   r-   r.   r4   r0   r   r   r2   r2   6   s    r   r2   c                     t        | |||      S )Nmemory_kind_logical_device_ids)NamedSharding)r   specr7   logical_device_idss       r   _unpickle_named_shardingr<   Q   s    	tT{+=
? ?r   c                     e Zd ZU dZded<   ded<   ded<   ded	<    e       d
d
d	 	 	 	 	 dd       Zd Zd Ze	d d       Z
 e       d        Z e       d        Zd!dZe	d"d       Ze	d#d       Ze	d$d       Ze	d%d       Ze	d%d       Ze	d#d       Zej*                  d%d       Zej*                  d&d       Zd'dZd(dZd)dZd*dZy
)+r9   a,  A :class:`NamedSharding` expresses sharding using named axes.

  A :class:`NamedSharding` is a pair of a :class:`Mesh` of devices and
  :class:`PartitionSpec` which describes how to shard an array across that
  mesh.

  A :class:`Mesh` is a multidimensional NumPy array of JAX devices,
  where each axis of the mesh has a name, e.g. ``'x'`` or ``'y'``.

  A :class:`PartitionSpec` is a tuple, whose elements can be a ``None``,
  a mesh axis, or a tuple of mesh axes. Each element describes how an input
  dimension is partitioned across zero or more mesh dimensions. For example,
  ``PartitionSpec('x', 'y')`` says that the first dimension of data
  is sharded across ``x`` axis of the mesh, and the second dimension is sharded
  across ``y`` axis of the mesh.

  The `Distributed arrays and automatic parallelization`_
  and `Explicit Sharding`_ tutorials have more details and diagrams that
  explain how :class:`Mesh` and :class:`PartitionSpec` are used.

  Args:
    mesh: A :class:`jax.sharding.Mesh` object.
    spec: A :class:`jax.sharding.PartitionSpec` object.

  Examples:

    >>> from jax.sharding import Mesh
    >>> from jax.sharding import PartitionSpec as P
    >>> mesh = Mesh(np.array(jax.devices()).reshape(2, 4), ('x', 'y'))
    >>> spec = P('x', 'y')
    >>> named_sharding = jax.sharding.NamedSharding(mesh, spec)

  .. _Distributed arrays and automatic parallelization: https://docs.jax.dev/en/latest/notebooks/Distributed_arrays_and_automatic_parallelization.html
  .. _Explicit Sharding:  https://docs.jax.dev/en/latest/notebooks/explicit-sharding.html
  %mesh_lib.Mesh | mesh_lib.AbstractMeshr   r   r:   
str | None_memory_kindtuple[int, ...] | Noner8   Nr6   c               |    || _         || _        || _        || _        t	        | j                   | j                         y r   )r   r:   r@   r8   check_pspec)r   r   r:   r7   r8   s        r   r   zNamedSharding.__init__   s4     DIDI#D2D		499%r   c                    | j                   dnd| j                    }| j                  dnd| j                   }t        | j                         }d| d| j                   | | dS )N z, memory_kind=z, logical_device_ids=zNamedSharding(mesh=z, spec=))r7   r8   strr   r:   )r   memldi	mesh_reprs       r   r4   zNamedSharding.__repr__   sw      ("t?O?O>P.QC))12"4#;#;"<= tyy>"I 7499+cU3%qIIr   c                j    t         | j                  | j                  | j                  | j                  ffS r   )r<   r   r:   r7   r8   r(   s    r   
__reduce__zNamedSharding.__reduce__   s1    $YY		4#3#3T5M5MNP Pr   c                    | j                   S r   )r@   r(   s    r   r7   zNamedSharding.memory_kind   s    r   c                    t        | d      s<t        | j                  | j                  | j                  | j
                  f      | _        | j                  S )N_hash)hasattrhashr   r7   r:   r8   rO   r(   s    r   __hash__zNamedSharding.__hash__   sC    4!99d&&		43K3K
LNdj::r   c                ,   t        |t              sy| |u ry| j                  |j                  k7  s2| j                  |j                  k7  s| j                  |j                  k7  ry| j
                  |j
                  u xs | j
                  |j
                  k(  S NFT)
isinstancer9   r:   r7   r8   r   )r   others     r   __eq__zNamedSharding.__eq__   sy    e]+u}		UZZu000##u'@'@@99

"=dii5::&==r   c           
         t        |      t        | j                        k  rEt        |      dk(  rdnd}t        d|  dt        | j                         dt        |       d|       y )Nr   z, For scalars the PartitionSpec should be P()rE   z	Sharding z+ is only valid for values of rank at least z%, but was applied to a value of rank .)lenr:   
ValueError)r   
aval_shape	extra_msgs      r   check_compatible_avalz#NamedSharding.check_compatible_aval   sr    
:TYY'*o* B02 dVF^A_Qyk+, , (r   c                .    | j                   j                  S r   )r   sizer(   s    r   num_deviceszNamedSharding.num_devices   s    99>>r   c                    t        | j                  t        j                        rt	        d      | j                  j
                  S )Nz>device_set is not implemented for `jax.sharding.AbstractMesh`.)rU   r   mesh_libAbstractMeshr[   _flat_devices_setr(   s    r   
device_setzNamedSharding.device_set   s8    $))X223
JL L99&&&r   c                    t        | j                  t        j                        rt	        d      | j                  j
                  S )NzF_device_assignment is not implemented for `jax.sharding.AbstractMesh`.)rU   r   rc   rd   r[   r'   r(   s    r   r)   z NamedSharding._device_assignment   s8    $))X223 7 8 899(((r   c                    t        | j                  t        j                        rt	        d      | j
                  j                  S )NzHis_fully_addressable is not implemented for `jax.sharding.AbstractMesh`.)rU   r   rc   rd   r[   _internal_device_listis_fully_addressabler(   s    r   rj   z"NamedSharding.is_fully_addressable   s<    $))X223 6 7 7 %%:::r   c                N    t        | j                  t        j                        ryyrT   )rU   r   rc   rd   r(   s    r   _is_concretezNamedSharding._is_concrete   s    $))X223r   c                    t        | j                  t        j                        rt	        d      | j                  j
                  S )NzGaddressable_devices is not implemented for `jax.sharding.AbstractMesh`.)rU   r   rc   rd   r[   _local_devices_setr(   s    r   addressable_devicesz!NamedSharding.addressable_devices   s:    $))X223 6 7 7 99'''r   c                    | j                   j                  dk(  ryt        | j                        }| j                   j                  }d}|D ]
  }|||   z  } |dk(  S )N   T)r   r`   get_array_mappingr:   shape)r   array_mappingr   num_partitionsnames        r   is_fully_replicatedz!NamedSharding.is_fully_replicated   s^    yy~~%dii0MJN )
4((n)Qr   c                    t        d t        | j                        D              }t        | j                  j                        || j                  j
                  z  | j                  j                  z  z
  S )Nc              3  H   K   | ]  }||t         j                  ur|  y wr   )r   UNCONSTRAINED).0ss     r   	<genexpr>z0NamedSharding.replicated_axes.<locals>.<genexpr>   s,      C=Qm&A&AA 	
Cs    ")	frozensetflatten_specr:   r   
axis_names	unreducedreduced)r   	flat_specs     r   replicated_axeszNamedSharding.replicated_axes   sd     C		*C CI TYY))*DII'''$))*;*;;= =r   c                &    | j                  |      S )N)r7   )update)r   kinds     r   with_memory_kindzNamedSharding.with_memory_kind   s    ;;4;((r   c           	     $   |j                  d| j                        }t        |t              st        | }t	        |j                  d| j
                        ||j                  d| j                        |j                  d| j                              S )Nr:   r   r7   r8   )r   r:   r7   r8   )popr:   rU   r   r9   r   r7   r8   )r   kwargsr:   s      r   r   zNamedSharding.update   s{    ::fdii(DdM*D!dZZ		*JJ}d.>.>?"JJ'<'+'?'?A	B Br   c                    t        | |      S r   )"named_sharding_to_xla_hlo_sharding)r   num_dimensionss     r   _to_xla_hlo_shardingz"NamedSharding._to_xla_hlo_sharding   s    -dNCCr   c                   t        |      D cg c]  }t        g d       }}t        | j                        D ]D  \  }}|t        j
                  u rd||   _        #|&t        |t              r|n|f}|||   _	        F t        | j                  j                  || j                  | j                  j                        S c c}w )NFr   T)r   r   r;   unreduced_axes)r   r    	enumerater:   r   rz   r   rU   tupler   r!   r   r"   r8   r   )r   r   r$   r   idim_specs         r   r%   zNamedSharding._to_sdy_sharding  s    #N35 U3 5M 5 + )8	]00	0#'a )(E:8 (a) tyy44"/'+'?'?#'99#6#68 85s   C)r   r>   r:   r   r7   r?   )r+   r?   )r\   Shaper+   None)r+   r*   )r+   zset[Device])r+   XLADeviceAssignment)r+   bool)r+   zfrozenset[MeshAxisName])r   rG   r+   r9   )r+   r9   r   r*   r+   zxc.HloSharding)r   r*   r+   r!   )r,   r-   r.   __doc____annotations__r	   r   r4   rL   r/   r7   rR   rW   r^   ra   rf   r)   rj   rl   ro   	functoolscached_propertyrw   r   r   r   r   r%   r0   r   r   r9   r9   V   sk   "H 	.--- !%$&7&?L&& &JP     	> 	>,   ' ' ) ) ; ;  
 ( (   = =)	BD8r   r9   zjax.shardingc                ~    g }| D ]5  }t        |t              r|j                  |       %|j                  |       7 |S r   )rU   r   extendappend)r:   outr|   s      r   r   r     s=    
# a!U	jjm	jjm	
 
*r   c                    t        | t        t        f      r| S t        j                         }t        |       D ];  \  }}||t        j                  u rt        |t              r|n|f}|D ]  }|||<   	 = |S r   )	rU   r   r2   collectionsOrderedDictr   r   rz   r   )axis_resourcesdr   r   axiss        r   rr   rr      s     '7 89!>* ga|t}:::dE*4D ag	 
(r   c                  6    e Zd ZU ded<   ded<   d	dZd Zd Zy)
r    zSequence[str]r   r   r   c                    t         j                  j                  | j                  D cg c]!  }t         j                  j                  |      # c}| j
                         S c c}w )N)	is_closed)r   DimensionShardingAttrgetr   AxisRefAttrr   )r   r   s     r   buildzSdyDim.build3  sN    $$((/3yy9t		T	"9ll" ) $ $9s   &A"c                *    d| j                          dS )NzSdyDim(rF   _custom_reprr(   s    r   r4   zSdyDim.__repr__8  s    T&&()++r   c                    dj                  d | j                  D              }d}| j                  r| j                  rdnd}d| | dS )N, c              3  (   K   | ]
  }d | d   yw)'Nr0   )r{   as     r   r}   z&SdyDim._custom_repr.<locals>.<genexpr><  s     6qAaS(6s   rE   z, ??{})joinr   r   )r   	axes_repr	open_reprs      r   r   zSdyDim._custom_repr;  sF    		6DII66II||99%#i	{9+R((r   N)r+   zsdy.DimensionShardingAttr)r,   r-   r.   r   r   r4   r   r0   r   r   r    r    .  s    -$
,)r   r    c                :      sy|J t         fd|D              S )Nr0   c              3  2   K   | ]  \  }}|v s|  y wr   r0   )r{   nr$   r   s      r   r}   z_get_axes.<locals>.<genexpr>H  s     5TQ19q5s   )r   )r   r   s   ` r   	_get_axesr   B  s(    				 
5Z5	55r   T)kw_onlyc                  d    e Zd ZU ded<   ded<   dZded<   dZd	ed
<    e       Zded<   ddZd Z	y)r!   z"tuple[tuple[str, int], ...] | Noner   zSequence[SdyDim]r   NrA   r;   r0   ztuple[str, ...]r   zfrozenset[str]r   c                >   | j                    t        j                  j                  g       }n~| j                  g nt        | j                        }t        j                  j                  | j                   D cg c]%  \  }}t        j                  j                  ||      ' c}}|      }t        | j                  | j                         }t        | j                  | j                         }t        j                  j                  || j                  D cg c]  }|j                          c}|D cg c]!  }t        j                  j                  |      # c}|D cg c]!  }t        j                  j                  |      # c}      S c c}}w c c}w c c}w c c}w )N)r   r   )r   r   MeshAttrr   r;   listMeshAxisAttrr   r   r   TensorShardingAttrr   r   r   )	r   	mesh_attrrI   rv   r`   r   r   dim_shardingr   s	            r   r   zSdyArray.buildR  s;   ,,""2&i**2R$))* 
,,"">Boo
N
d3d+
N
i   4 4dooFOt22DOODN!!%%262D2DE,			E?NOt,,T2O>LMd++D1M	 & O O O 	FOMs   8*F

F
0&F&Fc                    dj                  d | j                  D              }| j                  d| j                   nd}| j                  rd| j                   nd}d| d| | dS )	Nr   c              3  <   K   | ]  }|j                           y wr   r   )r{   r   s     r   r}   z$SdyArray.__repr__.<locals>.<genexpr>e  s      "6"6s   z, device_ids=rE   z, replicated_axes=z
SdyArray([]rF   )r   r   r;   r   )r   dim_sharding_reprdevice_id_reprrars       r   r4   zSdyArray.__repr__d  s    		 "6"&"4"4"6 6 00< &d&=&=%>?BD  ""   4 456(* )*!N+;C5BBr   )r+   zsdy.TensorShardingAttr)
r,   r-   r.   r   r;   r   r~   r   r   r4   r0   r   r   r!   r!   J  s<    00!!/3,3%'/?'#,;...O$Cr   r!   c                |   j                   rg g }}| j                  D ]C  }|j                  t        |j                  d             |j                  |j                         E t        j                        t        |      z
  }t        fd|D              }t        | j                  || j                  |      S | S )NTr   c              3  v   K   | ]0  }j                   |   t        j                  j                  k(  r| 2 y wr   )_name_to_typerc   r   Explicit)r{   rr   s     r   r}   z5modify_sdy_sharding_wrt_axis_types.<locals>.<genexpr>w  s9      T!#11!48I8I8R8RR  Ts   69)r   r   r;   r   )_any_axis_autor   r   r    r   r   setr   r   r!   r   r;   )sdy_shardingr   r   	used_axesr   remaining_axesr   s    `     r   "modify_sdy_sharding_wrt_axis_typesr   p  s    	!29M'' 6qvvt<=qvv )C	N:N T~ T TO|66"/'3'F'F$35 5 
r   i   F)max_sizetrace_context_in_keyc                   | j                   j                  }t        | j                        }t	        | j                   j
                        D ci c]  \  }}||
 }}}i }t        | j                   j                        }|rS| j                   j
                  }	|D ]8  }
t        j                  j                  j                  ||	j                  |
      <   : | j                  j                  }|rS| j                   j
                  }	|D ]8  }t        j                  j                  j                  ||	j                  |      <   : g }t	        |j                               D ]   \  }\  }}||vs|j!                  ||f       " t#        |      t#        |      k(  r |st        j$                  j'                         S g }dg|z  }t)        |j                         d       D ])  \  }}||xx   ||   z  cc<   |j!                  ||          + g }|r#t+        j,                  t.              }t+        j,                  d       }|D ch c]  }|d   	 c}j1                  t3        |j5                                     sJ |D ]Z  \  }}|j7                  |t        j                  j                  j8                        }||   j!                  |       ||xx   |z  cc<   \ t)        |j                         d       D ];  \  }}|j!                  |       |j!                  ||          |j;                  |       = |}| j                   j<                  }| j>                  #t        j$                  jA                  ||||      S t        j$                  jC                  tE        jF                  | j>                        jI                  |      jI                  |      jK                  |      jI                  |      |      S c c}}w c c}w )	Nrq   c                    | d   S Nrq   r0   xs    r   <lambda>z4named_sharding_to_xla_hlo_sharding.<locals>.<lambda>  s
    qt r   )keyc                      yr   r0   r0   r   r   r   z4named_sharding_to_xla_hlo_sharding.<locals>.<lambda>  s    r   r   c                     | d   j                   S )Nr   )valuer   s    r   r   z4named_sharding_to_xla_hlo_sharding.<locals>.<lambda>  s    qtzz r   )dimsreshape_dimstranspose_permsubgroup_types)r   )&r   rs   rr   r:   r   r   r~   manual_axesxc
OpShardingTypeMANUALindexr   	UNREDUCEDitemsr   rZ   HloSharding	replicatesortedr   defaultdictr   
issupersetr   keysr   
REPLICATEDr   
axis_sizesr8   	iota_tilesubgroup_with_device_orderingnpasarrayreshape	transpose)r   r   r   rt   r   rv   mesh_axis_posspecial_axesr   r   manual_axisr   ureplicated_mesh_axes	axis_nameaxis_valmesh_permutationnew_mesh_shapeposlast_tile_dimsaxes_by_typesize_by_typer   r`   tyr   r   r   s                               r   r   r     s    yy*#DII.-*3DII4H4H*IJwq$47J-J,$))//0+%%J" N46MM4F4F4M4Ml:##K01N 99&&.%%J G*,--*<*<*F*Fl:##A&'G "+J,<,<,>"? 1a	)X%!!1h-01 		#j/1,>>##%%3'.---/^D 1idC3:d++M$/01 .)4)@)@)FL**95L./QAaD/::3|?P?P?R;STTT' 4Ar}}11<<=b2a 2$ <--/5IJ $DBL,-d#$* 
$%%,	%>>##=M% $ ' ' >>77


4++,	ww|,YY7G-H	~ 8 7 7E KB 0s   O-<O3c                ~   | s
t               S d}t        j                  t              }| j	                         D ]!  \  }}||   j                  |       ||kD  s |}# g }t        |dz         D ]H  }||   }|r.|j                  t        |      dk(  r|d   n
t        |             8|j                  d        J t        | S )Nrq   r   )	r   r   r   r   r   r   r   rZ   r   )rt   	max_indexreverse_mapr   r   
partitionsr   s          r   array_mapping_to_axis_resourcesr    s    	?)''-+"((* kdEd#yi *Q aq>D3t9>QuT{C 

	##r      c                N    t        |d|        t        | |       t        | |       y )NzNamedSharding spec)_check_unique_resources_check_mesh_resource_axis_check_mesh_unreduced)r   r:   _manual_axess      r   rC   rC     s#    $ 4d;D$'d#r   c                  $     e Zd Z fdZd Z xZS )DuplicateSpecErrorc                N    t         |   |       || _        || _        || _        y r   )superr   messager   pspec)r   r   r   r!  	__class__s       r   r   zDuplicateSpecError.__init__  s%    	GWDLDIDJr   c                    | j                    S r   )r   r(   s    r   __str__zDuplicateSpecError.__str__  s    ll^r   )r,   r-   r.   r   r$  __classcell__)r"  s   @r   r  r    s    r   r  c           
     r   i }d}| D ]T  }|t         j                  u s|t        |t              r|n|f}|D ]#  }|j	                  |d      }|dkD  rd}|dz   ||<   % V |rR|j                         D 	cg c]  \  }}	|	dkD  s| }
}}	t        d| d|  dt        j                  |
       ||       y c c}	}w )	NFr   Trq   z	A single zP specification can map every mesh axis to at most one positional dimension, but z has duplicate entries for )r   r   r!  )	r   rz   rU   r   r   r   r  rc   	show_axes)r!  arg_namer   resource_counts	duplicater   resourcecountr   cmultiple_usess              r   r  r    s    -//) ,aM'''195!tA ,!!(A.e		"'!)oh	,	, #2#8#8#:D41aa!eQDMD
z "338' :&&}568     Ds   3B3B3c                     |D ]j  t        t              st         fdD              r*t         fdD              s?t	        d| d ddj                   fdD               d       y )	Nc              3  b   K   | ]&  }j                   d       j                   |   k(   ( yw)r   N)r   )r{   pr   r:   s     r   r}   z,check_pspec_mix_axis_type.<locals>.<genexpr>	  s8       Q(D,>,>q,AA s   ,/c              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wr   )r   r   Manualr{   r1  r   s     r   r}   z,check_pspec_mix_axis_type.<locals>.<genexpr>  s%     D!T"hoo5Ds   *-zcTuple subset of `PartitionSpec` cannot contain `Manual` mixed with `Auto` or `Explicit`. Got pspec z and subset z with axis types: (r   c              3  N   K   | ]  }t        j                  |           y wr   )rG   r   r4  s     r   r}   z,check_pspec_mix_axis_type.<locals>.<genexpr>  s!     D!3t11!45Ds   "%rF   )rU   r   allanyr[   r   )r   r!  r:   s   ` @r   check_pspec_mix_axis_typer8    s     
Id$	  
	DtD	D55:G <v DtDDEQHI 	I
Ir   c                   |D ]w  }|t         j                  u s|t        |t              r|n|f}|D ]F  }|| j                  vst        d| d| dt        | j                  j                                d       y t        | |       t        j                  | j                  vr-t         j                  |v rt        | d| j                         y y )NzResource axis: z of z is not found in mesh: rY   z[ cannot contain `P.UNCONSTRAINED` when no mesh axis_types are `Auto`. Got mesh axis_types: )r   rz   rU   r   r   r[   rs   r   r8  r   Auto
axis_types)r   r!  r1  r   s       r   r  r    s     BaM'''195!tA B	
$//	!aSUG ,%%*4::??+<%=$>aAB 	BB	B D%(mm4??*!!U*
' (	*+ + + +r   c           	        |j                   D ]i  }|| j                  vrt        d| d| j                  d|      | j                  |   t        j
                  k(  sPt        d|j                    d|         |j                  D ]i  }|| j                  vrt        d| d| j                  d|      | j                  |   t        j
                  k(  sPt        d|j                   d|         y )NzUnreduced axes z! is not found in mesh.axis_names=z. Got pspec=zhUnreduced axes can only refer to mesh axes that are of type `Explicit` or `Manual`. Got unreduced axes: z and mesh: zReduced axes zdReduced axes can only refer to mesh axes that are of type `Explicit` or `Manual`. Got reduced axes: )r   r   r[   r   r   r:  r   )r   r!  r  s      r   r  r  %  s+   ?? 	aA3@/A B  !-::?//9J K6 	 == 	a!>doo-? @  !-88= G6 	r   )r   z'PartitionSpec | AUTO | UnspecifiedValuer+   ArrayMappingOrAutoOrUnspecified)r   r!   r   )rt   ArrayMappingr   )r!  r   r(  rG   r+   r   )A
__future__r   collections.abcr   r   dataclassesr   typingr   r   jax._src.utilr   r   r	   jax._src.libr
   r   jax._src.lib.mlir.dialectsr   jax._srcr   rc   jax._src.meshr   jax._src.partition_specr   r   	JShardingnumpyr   r   r*   r   DevicesliceIndexr   r   r2   UNSPECIFIEDMeshAxisNamer   r>  r=  r<   r9   Shardingr-   r   rr   	dataclassr    r   r!   r   r   r  r~   rC   	Exceptionr  r  r8  r  r  r0   r   r   <module>rS     s   # $     > > ) * % " 1 * c3h	eSjv& ) )      &&|S'89"'d<L(L"M ?
 r {8I&& {8 !{8z * ;$ ) ) )&6 t$ C  C % CJ  51I7I7"0I7 2I7X$& %0)2 $ 1$
  !% ,I+$r   