
    uki1\                   
   U d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dl
Z
d dlmZ d dlZd dlmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z  d dlm!Z! d dlm"Z" d dl#m$Z$ d dl#m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z7 d dl8m9Z9 d dl:m;Z;m<Z< d dl=m>Z>m?Z? d dl@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZJ d dlKmLZL d dlMmNZN d dlOmPZP d dlOmQZQ d d lOmRZS d d!lOmTZT d d"lOmUZU d d#lVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZa e?Zbe"j                  edcZdZee"j                  egcZgZh e!j                  ej       eZkeZl G d$ d%      Zm em       Znd& Zodend ep       d'd(	 	 	 	 	 	 	 dd)Zqdend*dd+Zrdd,Zs e	e!j                  d-.      d/d0	 	 	 	 	 	 	 	 	 	 	 dd1       Zuej                  d2        Zwd3 Zxd4 Zydd5Zz ej                  d6d7d8g      Z|d9 Z}dd:Z~ G d; d<      Z e       Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd=Z	 	 	 	 dd>Z	 	 	 	 	 	 	 	 	 	 	 	 dd?Z	 	 	 	 	 	 	 	 	 	 	 	 dd@Z	 	 	 	 	 	 ddAZddBZdC Z edD      Z	 	 ddEZej                  dF        Zej                  dG        ZeZeee&f   Z G dH dIej                        Z edJ      Z e"j$                  dKd/L      ddM       ZdN Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddOZeeSj,                  _        ddPZddQZ	 	 	 	 ddRZeej6                  ej8                  <   	 	 ddSZeej<                  ej8                  <   dT Zeej@                  e<   ddUZ	 	 	 	 ddVZ	 	 	 	 ddWZdX Z	 	 	 	 ddYZ	 	 ddZZ eQjN                  ee       d[ Zdd\Zdd]Zdd^Zd_ Zd` Zda Zdb Zeej`                  _        dc ZdddZde ZddfZdg Z	 	 	 	 ddhZdi ZddjZ G dk dle      Zdm Z G dn doe      Z	 	 	 	 	 	 ddpZdq Zdr Zds Zdt Z G du dvej                        Z G dw dxejL                        Zdy Zi Zdzed{<   d| Zeeej                  <   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd}ZeePj                  _        d~ ZddZeeUj                  _        	 	 ddZeeSj                  _        ddZeeUj                  _        ej                  d        Zej                  dd       ZddZ	 	 ddZ	 	 ddZeeUj                  e<   	 	 	 	 	 	 	 	 	 	 ddZeeSj                  e<   	 	 	 	 	 	 ddZd Z	 	 	 	 	 	 ddZ	 	 	 	 	 	 ddZ	 	 ddZeeSj                  e<    eLj                  e      d        Zy)    )annotations)CallableHashableSequenceSetN)partial)prod)AnyTypeVarUnion)ad_util)api)api_util)config)core)dispatch)dtypes)linear_util)sharding_impls)source_info_util)traceback_util)util)order_wrt_mesh)pvaryTracertypeof
shard_avalunshard_aval)AbstractMeshMeshBaseMeshAxisTypeuse_abstract_meshget_abstract_meshget_concrete_mesh)reshard)laxparallel)ir)hlosdy)NamedShardingPartitionSpec)	HashableFunctionHashablePartialunzip2as_hashable_functionpartition_listmerge_lists
split_list
subs_list2fun_name)	discharge)AbstractRef)batching)mlir)partial_eval)pxla)ad)broadcast_prefixkeystrprefix_errorsgenerate_key_pathstree_flattentree_leavestree_maptree_structuretree_unflattenKeyPath	PyTreeDefc                      e Zd Zd Zd Zy)InferFromArgsc                     y)Nzjax.sharding.Infer selfs    M/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/shard_map.py__repr__zInferFromArgs.__repr__J   s        c                    t         dfS NrL   )_get_default_inferrM   s    rO   
__reduce__zInferFromArgs.__reduce__M   s    ##rQ   N)__name__
__module____qualname__rP   rU   rL   rQ   rO   rJ   rJ   I   s     $rQ   rJ   c                     t         S NInferrL   rQ   rO   rT   rT   R   s    	,rQ   T)in_specsmesh
axis_names	check_vmac              J    t        |||||      | fdS t        | fi S )a  Map a function over shards of data using a mesh of devices.

  See the docs at https://docs.jax.dev/en/latest/notebooks/shard_map.html.

  Args:
    f: callable to be mapped. Each application of ``f``, or "instance" of ``f``,
      takes as input a shard of the mapped-over arguments and produces a shard
      of the output.
    mesh: (optional, default None) a ``jax.sharding.Mesh`` representing the
      array of devices over which to shard the data and on which to execute
      instances of ``f``. The names of the ``Mesh`` can be used in collective
      communication operations in ``f``. If mesh is None, it will be inferred
      from the context which can be set via `jax.set_mesh` context
      manager.
    in_specs: (optional, default `Infer`) a pytree with
      ``jax.sharding.PartitionSpec`` instances as leaves, with a tree structure
      that is a tree prefix of the args tuple to be mapped over. Similar to
      ``jax.sharding.NamedSharding``, each ``PartitionSpec`` represents how the
      corresponding argument (or subtree of arguments) should be sharded along
      the named axes of ``mesh``. In each ``PartitionSpec``, mentioning a
      ``mesh`` axis name at a position expresses sharding the corresponding
      argument array axis along that positional axis; not mentioning an axis
      name expresses replication.
      If ``Infer``, all mesh axes must be of type
      `Explicit`, in which case the in_specs are inferred from the argument types.
      If ``None``, inputs will be treated as static.
    out_specs: a pytree with ``PartitionSpec`` instances as leaves, with a tree
      structure that is a tree prefix of the output of ``f``. Each
      ``PartitionSpec`` represents how the corresponding output shards should be
      concatenated. In each ``PartitionSpec``, mentioning a ``mesh`` axis name
      at a position expresses concatenation of that mesh axis's shards along the
      corresponding positional axis; not mentioning a ``mesh`` axis name
      expresses a promise that the output values are equal along that mesh axis,
      and that rather than concatenating only a single value should be produced.
    axis_names: (optional, default set()) set of axis names from ``mesh`` over
      which the function ``f`` is manual. If empty, ``f``, is manual
      over all mesh axes.
    check_vma: (optional) boolean (default True) representing whether to enable
      additional validity checks and automatic differentiation optimizations.
      The validity checks concern whether any mesh axis names not mentioned in
      ``out_specs`` are consistent with how the outputs of ``f`` are replicated.

  Returns:
    A callable representing a mapped version of ``f``, which accepts positional
    arguments corresponding to those of ``f`` and produces output corresponding
    to that of ``f``.
  )r^   r]   	out_specsr_   r`   c                    t        | fi S rZ   )
_shard_mapgkwargss    rO   <lambda>zshard_map.<locals>.<lambda>   s    Z,V, rQ   )dictrd   )frb   r]   r^   r_   r`   rg   s         @rO   	shard_maprk   W   s6    h TH	%<&Y,,	A	 	  rQ   )in_axesc              F    t        |||      | fdS t        | fi S )a)  Single axis shard_map that maps a function `f` one axis at a time.

  Args:
    f: Callable to be mapped. Each application of ``f``, or "instance" of ``f``,
      takes as input a shard of the mapped-over arguments and produces a shard
      of the output.
    in_axes: (optional) An integer, None, or sequence of values specifying which
      input array axes to map over. If not specified, `smap` will try to infer
      the axes from the arguments only under `Explicit` mode.
      An integer or ``None`` indicates which array axis to map over for all
      arguments (with ``None`` indicating not to map any axis), and a tuple
      indicates which axis to map for each corresponding positional argument.
      Axis integers must be in the range ``[-ndim, ndim)`` for each array,
      where ``ndim`` is the number of dimensions (axes) of the corresponding
      input array.
    out_axes: An integer, None, or (nested) standard Python container
      (tuple/list/dict) thereof indicating where the mapped axis should appear
      in the output.
    axis_name: ``mesh`` axis name over which the function ``f`` is manual.

  Returns:
    A callable representing a mapped version of ``f``, which accepts positional
    arguments corresponding to those of ``f`` and produces output corresponding
    to that of ``f``.
  )rl   out_axes	axis_namec                    t        | fi S rZ   _smapre   s    rO   rh   zsmap.<locals>.<lambda>   s    U1'' rQ   )ri   rr   )rj   rl   rn   ro   rg   s       @rO   smaprs      s.    4 (iH&Y''	q	F	rQ   c          	        t        |t        t        f      rt        d|       |-|t        ur%t        |t
        t        f      st        d| d      |t        ur*t        d t        |      D              st        d| d      t        d t        |      D              st        d| d      |t        u rt        nt        t        t        |      |d 	      }t        t        t        |      |d
 	      }t        | d |||hdd      S )Nz:smap axis_name should be a `str` or a `Hashable`, but got zsmap in_axes must be an int, None, jax.sharding.Infer, or a tuple of entries corresponding to the positional arguments passed to the function, but got .c              3  <   K   | ]  }t        |t                y wrZ   
isinstanceint.0ls     rO   	<genexpr>z_smap.<locals>.<genexpr>   s     ?QjC ?   zqsmap in_axes must be an int, None, jax.sharding.Infer, or (nested) container with those types as leaves, but got c              3  <   K   | ]  }t        |t                y wrZ   rw   rz   s     rO   r}   z_smap.<locals>.<genexpr>   s     ?AZ3?r~   z^smap out_axes must be an int, None, or (nested) container with those types as leaves, but got c                
    | d u S rZ   rL   xs    rO   rh   z_smap.<locals>.<lambda>   
    d rQ   is_leafc                
    | d u S rZ   rL   r   s    rO   rh   z_smap.<locals>.<lambda>   r   rQ   T)r^   r]   rb   r_   r`   rr   )rx   listtuple	TypeErrorr\   ry   allrC   rD   r   _axes_to_pspecrd   )rj   rl   rn   ro   r]   rb   s         rO   rr   rr      s7   	D%=)

DYKPR RgU2
WsEl
+
	%Ya	)* * U
?+g*>?
?
	::A!	EF F 
?X)>?	?
 ;;C*AG H H %'ew~y9724  w~y9824)	AD8y ){d$
H HrQ   zjax.shard_map)repro_api_nameFrq   c          	          t               st        d  dt                d      t        j                         t
        j                   fd              }|S )Nz>shard_map requires a callable for its first argument, but got 	 of type ru   c                    t              \  t        j                  t        j                  d| i             }t        |       \  }}t        |      \  }}t        j                  ||      \  }}	 t        | d       }t        u rst        fdD              r_|D 	cg c]  }	t        |	      j                   }
}	t        d |D              sJ |       |
D cg c]  }t        |j                          }}t#        d t%        |      D              \  }}|j                  j&                  ;t)        |      t)        |j                  j&                        k7  r|j+                         }t        j,                  |||d	      \  }}t/        ||||       rt1        ||      }t3        ||      }t5        j6                        t9        j:                         j<                  z
  }|k(  rjt        fd
D              rVt?        ||      D 	cg c]?  \  }	}t        |	      j                  j                   |k(  r|	ntA        |	tC        |            A }}	}	 tE        jF                  |g|||d}t[         |             S # t        $ r t        |       ^}} |d      d w xY wc c}	w c c}w c c}}	w # tH        $ rh}|jJ                  \  }tM              sEtO        tP        jR                   |       |      }tU        d |D              r|dz  }t        |      d Y d }~d }~wtV        $ rC}|jJ                  \  }tM              s tY         |       |      }t        |      d Y d }~d }~ww xY w)Nrk   
debug_infoc                
    | d u S rZ   rL   r   s    rO   rh   z-_shard_map.<locals>.wrapped.<locals>.<lambda>   s
    AI rQ   r   zshard_map in_specsc              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wrZ   _name_to_typer"   Explicitr{   ar^   s     rO   r}   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   '     K1Dq!X%6%66K   *-c              3  ,   K   | ]  }|t         u   y wrZ   r[   )r{   is     rO   r}   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s     3e3   c              3  .   K   | ]  \  }}|||f  y wrZ   rL   )r{   r   ss      rO   r}   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s$      (:41a+,= *+A (:s   Fc              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wrZ   r   r   s     rO   r}   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   r   r   r^   r]   out_specs_thunkr`   manual_axesc              3  J   K   | ]  }|t         uxr |j                     y wrZ   )no_failshape)r{   fails     rO   r}   z._shard_map.<locals>.wrapped.<locals>.<genexpr>  s#     H$t7"54::~5Hs   !#z In particular, for rank 0 outputs which are not constant over the mesh, add at least one (singleton) axis to them so that they can be concatenated using out_specs.)._shmap_checkslu	wrap_initr   r   rB   _broadcast_out_specsflatten_fun_nokwargsr>   
ValueErrorr@   r\   r   r   sharding_manual_specspecr0   	enumerate	arg_nameslenwith_unknown_namesargnums_partial_check_specs_vs_args_implicit_pvary_on_output_implicit_unreduced_on_output	frozensetr_   r   get_axis_envexplicit_mesh_axis_nameszipr&   r,   shard_map_pbind
_SpecErrorargscallable_spec_rank_errorSpecErrorTypeoutany	_RepError_inout_vma_errorrF   )r   fun	args_flatin_treer   out_treein_specs_flate_r   arg_sr   dyn_argnumsmesh_axis_names_wo_vmapout_flatfailsmsgrr   r_   r`   rj   r]   r^   rb   s                    rO   wrappedz_shard_map.<locals>.wrapped   sp    %j(Iu6D*
,,	h))+q$CEC%d+Iw/Y
KC11#w?MC.&
D"57m 	EK
KK+45avay!!5e53]33B]B3GLM!|J=MmM!' (:9];S (: ":K  ,KC 8 899""$c--c;	5QNCD'8[-"$ %c?;c)#?c 	$//"T%6%6%8%Q%QQ :-K
KK  #9m<>q! q	**//141=q#9:; >i >(!!
"" $})Y "h$ (*h//g  .Hd+ea!"#-. 6M(>  (vvfei }00!XZERH%HH
 C D# o4' !  (vvfei q$
IuEo4' !(sD   9J
 (J.#J3AJ8J> 
!J+>	M9AL**M969M44M9)r   r   typer   wrapsr   api_boundary)rj   r^   r]   rb   r_   r`   rr   r   s   ``````` rO   rd   rd      sn    
 
!
  c47)16 7 7 ::a=@0 @0  @0B 
.rQ   c                    | |i |}t        |      r# |       }t        t        j                  ||       n|}	 t	        ||      }|j                  t        |             |S # t
        $ r t        ||      ^}	}
 |	d      d w xY w)Nzshard_map out_specs)	r   _check_specsr   r   r>   r   r@   storer   )_fun_storerb   r_   r   rg   ans
out_specs_out_specs_flatr   r   s              rO   r   r     s    df#iJ""J
;J-%j#6N
 	,,u^$%	* 
 -*c*EA
!
",-s   A" "!Bc                :    |
t               S t        d g|z  | gz    S rZ   )P)ro   axiss     rO   r   r   0  s&    	\3J	
TFTMYK'	((rQ   c                &     !t                 j                  rAt        d      t               }|j                  s  j                  |k7  rt        d| d        t	         t
        t        f      st        d  dt                d      t         j                        t        j                         j                  z
  }t	        |t        t        f      st        dt        |             t	        |t              rt        |      }|s|}|j                  |      st        d| d	|       |t         u rVt#         fd
|D              sBdj%                   fd|D              }|rd| d| }t        |      d|d| }t        |      |t         ur9|7t'        t(        j*                  ||       t-        t(        j*                   ||       t/        |      s7t'        t(        j0                  ||       t-        t(        j0                   ||        |fS )NzWThe context mesh cannot be empty. Use `jax.set_mesh(mesh)` to enter into a mesh contextzThe context mesh z+ should match the mesh passed to shard_map ztshard_map requires a `jax.sharding.Mesh` or a `jax.sharding.AbstractMesh` instance for its second argument, but got r   ru   zU`axis_names` argument of shard_map should be of type `frozenset` or `set`. Got type: z"jax.shard_map requires axis_names=z# to be a subset of mesh.axis_names=c              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wrZ   r   r   s     rO   r}   z _shmap_checks.<locals>.<genexpr>[  s'     MQd  #x'8'88Mr   , c              3  N   K   | ]  }t        j                  |           y wrZ   )strr   r   s     rO   r}   z _shmap_checks.<locals>.<genexpr>\  s!     J!3t11!45Js   "%z)in_axes was not specified when axis_name=z was of type zzshard_map in_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but it was `None` when axis_names=z are of type )r$   emptyr   abstract_meshrx   r    r   r   r   r   r_   r   r   r   setissubsetr\   r   joinr   r   input_check_unreducedr   r   )	r^   r_   r]   rb   rr   ctx_meshr   
axis_typesr   s	   `        rO   r   r   6  s6   	\Dzz?@ @ !"H>>d00H<hZ (v    
D4.	/
 004vYtDzl!M N N
  4#4#4#6#O#OO  
JC 0	1
	!*-.	01 1 
C :&J	(J			4	5

,ZL 923	56 6 %
M*M
MJzJJJ8 E \#c C.]:,8c C.Ux3$$h
;](($
HE	)	""Iz:]&&j)D	z	rQ   c                   g }|D ]  }||j                  |       t        |t              rq|D cg c]
  }|| v r|nd  }}|r|d   |j                          |r|d   d |v rt	        d|       |j                  t        |      dk(  rd n
t        |             |j                  || v r|nd         t        t        j                  || |       t        ||j                  |j                  dS c c}w )NzInvalid spec: r   	unreducedreduced)appendrx   r   popr   r   r   r   r   r   r   r   )r   r   r^   r   r   ptemps          rO   r   r   o  s    
# 2ay	jjm	Au	789!1#a-9d9T"X%
 T"X%	>$011	jjTaU4[9	jja;&D12 =&&k4@	
C4>>4<<	@@ :s   C2r   r   r   c                   | t         j                  k(  rdnd}t        j                        |k(  }t	        |      \  }}|D ]  }|j
                  s|j                  s|s"t        d| dj                   d| d| d	      t        fd|j
                  D              st        d	| d
| d       t        fd|j                  D              rt        d| d
| d        y )Ninr   z(unreduced/reduced can only be passed to zB_specs when shard_map is in full manual mode. Got mesh axis names z, manual_axes: z	, specs: z<. Please file a bug at https://github.com/jax-ml/jax/issues.c              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wrZ   r   r{   ur^   s     rO   r}   z#_check_unreduced.<locals>.<genexpr>  s'     Oat!!!$(9(99Or   zunreduced in z_specs zi can only be used when the mesh passed to shard_map contains axis names all of type `Explicit`. Got mesh c              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wrZ   r   r   s     rO   r}   z#_check_unreduced.<locals>.<genexpr>  s'     Mat!!!$(9(99Mr   zreduced in )
r   r   r   r_   rB   r   r   NotImplementedErrorr   r   )	
error_typer^   r   specsprefixfull_manual
specs_flatr   r   s	    `       rO   r   r     s   !4!444%&$//*k9+u%-*a a;;qyy4VH =oook])A3 GAAB B
 O1;;OO& ,f  M199MMxwqc *f rQ   c                  
 | t         j                  k(  r|t        d      fd
t        
fdt	        |      D              ry | t         j                  k(  rdnd}t        |      D cg c]@  \  }}t        |t              s+d| dt        |       d| d	t        |      j                   d
	B }}}|st        |      D ]\  \  }}|D ]R  }t        |t              s|fn|}|D ]6  }	|	|	vs|j                  d| dt        |       dt        |	              8 T ^ t        d| d ddj                  |      z   d| dz         t        d| ddj                  |      z   d| dz         c c}}w )Nzshard_map in_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but it was None.
Instead of `in_specs=None`, did you mean `in_specs=P()`, where `P = jax.sharding.PartitionSpec`?c                    t        | t              sy| D ](  }t        |t              s|fn|}|D ]  }||vs  y * y)NFT)rx   r-   r   )r   namesnamer   s      rO   
check_specz _check_specs.<locals>.check_spec  sU    a' (6uhEe $K 7
 rQ   c              3  .   K   | ]  } |        y wrZ   rL   )r{   r   r  s     rO   r}   z_check_specs.<locals>.<genexpr>  s     31A3s   r   r   z  _specs is r   r   z refers to z
shard_map z8_specs argument must refer to an axis marked as manual (z	), but:



z

Check the z"_specs values passed to shard_map.zR_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but:

)r   r   r   r   rC   rA   rx   r   r?   r   rV   r   r   reprr   r   )r  r  r   r  keyr   msgsr   r	  r
  r  s     `       @rO   r   r     s   =&&&5=
	23 3 	3E 233
!4!444%&*51
Kc1Aq9I vhfVC[MaS	$q':J:J9K2
N 
K$ 
K	$U+ QQ Q% *5% 8e 	QD$k"9KK"VHF6#;-{4:,OP	QQQ 
VH (M	6
++d
	H>@	@A A
 	6( 9 :D(<>>	? ?
Ks   +AE2c                      e Zd Zd Zy)NoFailc                     y)NzNoFail()rL   rM   s    rO   rP   zNoFail.__repr__  s    rQ   N)rV   rW   rX   rP   rL   rQ   rO   r  r    s    rQ   r  c           	       	 t        t        j                  |      }t        ||      D 	cg c]%  \  }}	t	        |      |	j
                  k  s|	nt        ' }
}}	t        d |
D              r5t        |||
      }
t        t        j                  | |||
      }t        |      t        t        t        |            }t        ||      D 		cg c]0  \  	}t        	fd|j                         D              r	nt        2 }
}	}t        d |
D              r't        |||
      }
t!        | |||
      }t        |      y c c}	}w c c}}	w )Nc              3  ,   K   | ]  }|t         u  y wrZ   r   r{   rj   s     rO   r}   z'_check_specs_vs_args.<locals>.<genexpr>       (a'	(r   c              3  l   K   | ]+  \  }}j                   |   t        fd |D              z   - yw)c              3  <   K   | ]  }j                   |     y wrZ   r   r{   nr^   s     rO   r}   z1_check_specs_vs_args.<locals>.<genexpr>.<genexpr>  s     $?qTZZ]$?   N)r   r	   )r{   dnsr   r^   s      rO   r}   z'_check_specs_vs_args.<locals>.<genexpr>  s5      /q" 771:$?B$? ?? /s   14c              3  ,   K   | ]  }|t         u  y wrZ   r  r  s     rO   r}   z'_check_specs_vs_args.<locals>.<genexpr>  r  r   )mapr   shaped_abstractifyr   r   ndimr   r   _expand_failr   r   r   r   r   _spec_to_namesitems_spec_divisibility_error)rj   r^   r   r]   r   r   xsin_avalsr   r   r   r   in_names_flatr	  s    `       `    rO   r   r     s<    (("-(-2
4a q6QVV#!
0 
4$ 
4(4((d3D
=..7Hd
KC
S/NM:;- h6
8 
8a  / %/ /!4;< 
8$ 
8 	(4((d3D
"1dGXt
DC
S/ )
4
8s   *D;	5Ec                `    t         g| j                  z  }t        ||      D ]
  \  }}|||<    |S rZ   )r   
num_leavesr   )r   r   r   fail_r   rj   s         rO   r'  r'    s>     -49w7I7I+I%+t$ daE!H	,rQ   c                N   t        |      }| t        j                  k(  rd\  }}t        ||      }nd| d}}g }	t	        |||      D ]h  \  \  }
}\  }}d}| t        j                  k(  r|^}}t        d |j                  j                  j                         D              \  }}|j                  t        |      k\  s4||j                     j                  t        j                  j                  k(  r,d| d|j                  t        |      z
  dz    d	| d
|d    d	}n||j                     }d| d| d| d}|	j                  d| dt!        |
       d| dt        |       d| t!        |       | d|j#                          d|j$                   d|j$                   dt        |       d       k |	sJ t        |	      dk(  r	|	d   dd  g}	d| d| d| ddj'                  |	      z   dz   d | d!| d"| d#| d$	z   }t)        d% t	        |||      D              r	|d&| d'z  }|S )(N)r   zthe passed argsr   z(*args) c              3     K   | ]K  \  }}|j                   t        j                  j                  t        j                  j                  fvr||f M y wrZ   kindinspect	ParameterKEYWORD_ONLYVAR_KEYWORDr{   r
  params      rO   r}   z#_spec_rank_error.<locals>.<genexpr>  P      #@'D%ZZ 1 1 > > ' 1 1 = = ? ? -#@   AA, where args is the index     component of 's varargs parameter 'r   ', is bound to 's parameter 'z* r  r  z which has length z, but z has shape z, which has rank z (and z < )r      #shard_map applied to the function 'z' was given an zG_specs entry which is too long to be compatible with the corresponding zput value from the function:

r  zEntries in zQ_specs must be of length no greater than the number of axes in the corresponding z=put value.

Either revise the spec to be shorter, or modify 'z' so that its zputs have sufficient rank.c              3  B   K   | ]  \  }\  }}|j                      y wrZ   )r&  )r{   r   avals      rO   r}   z#_spec_rank_error.<locals>.<genexpr>  s     K<1iq$TYYKs   z0

For scalar values (rank 0), consider using an z>_specs entry of `P()`, where `P = jax.sharding.PartitionSpec`.)util_fun_namer   r   _try_infer_args_iter_pathsr0   	signature
parametersr)  idxr   r5  r6  r7  VAR_POSITIONALr   r?   	str_shortr&  r   r   )r  rj   treer  r   r6   r  basebar  spec_keyr   fail_keyrJ  extraarg_keyr   param_namesparams
param_namer   s                        rO   r   r     s    1(=&&&*LFD	D	!BXJg.DF	$,7eU,K G(x(4E](((R^kg" #@+-<<+B+B+H+H+J#@ @k6 ++V
$


"
"g&7&7&F&F
FyKK#f+-12 3j 6{26GrK GKK(
yhZ @)l".KK
VHF6(+,D6Ht9+V&!"5'T^^5E4F G))F499+ST1	FG#G, 
+Y!^T!WQR[MT.xj8  !A
C ;;t
  &
& vh 006x 8==EJ GH6
8	
8# 	K;tUE+JKK@ IF F GC	*rQ   c                   t        | |      }t        | dt        |             }g }t        |||      D ]  \  \  }}	\  }
}d}||
^}}t	        d |j
                  j                  j                         D              \  }}|j                  t        |      k\  s4||j                     j                  t        j                  j                  k(  r,d| d|j                  t        |      z
  dz    d| d|d	    d
	}n||j                     }d| d| d| d
}t        |	      }|j                         D ]  \  }}|j                  |   t!        fd|D              z  s+t        |      dkD  rd| nd|d    d}t        |      dkD  rdnd}t!        fd|D              }|j#                  dt%        |
       d|j'                          | dt%        |       d|	 d| d|j                  |    d| d| d| d| d|j                  |             |sJ t        |      dk(  r	|d   dd  g}d | d!t)        j                  j+                                d"j,                   d#d$j/                  |      z   d$z   d%| d&z   }|S )'NrV   r2  c              3     K   | ]K  \  }}|j                   t        j                  j                  t        j                  j                  fvr||f M y wrZ   r4  r:  s      rO   r}   z+_spec_divisibility_error.<locals>.<genexpr>"  r<  r=  r>  r?  r@  rA  rB  r   rC  rD  rE  c              3  <   K   | ]  }j                   |     y wrZ   r  r  s     rO   r}   z+_spec_divisibility_error.<locals>.<genexpr>1  s     8djjm8r   zaxes zaxis 'r   'ztotal c              3  <   K   | ]  }j                   |     y wrZ   r  r  s     rO   r}   z+_spec_divisibility_error.<locals>.<genexpr>4  s     ,A$**Q-,r   z* the passed argsz
 of shape z corresponds to in_specsz
 of value z, which maps array axis z
 (of size z
) to mesh z (of zsize z), but z does not evenly divide rG  rH  z' was given argument arrays with axis sizes that are not evenly divisible by the corresponding mesh axis sizes:

The mesh given has shape  with corresponding axis names .

r  a  Array arguments' axis sizes must be evenly divisible by the mesh axis or axes indicated by the corresponding elements of the argument's in_specs entry. Consider checking that in_specs are correct, and if so consider changing the mesh axis sizes or else padding the input and adapting 'z' appropriately.)rL  getattrr   rM  r0   rN  rO  r)  rP  r   r5  r6  r7  rQ  r(  r   r	   r   r?   rR  r   valuesr_   r   )rj   r^   rS  r  r   rU  r6   r  rV  r   rW  rJ  rX  rY  r   rZ  r[  r\  r	  r!  r"  r   totalszr   s    `                       rO   r*  r*    s    q$"Q
CF+(	$,7eU,K  (x(4E	~kg" #@+-<<+B+B+H+H+J#@ @k6 ++V
$


"
"g&7&7&F&F
FyKK#f+-12 3j 6{26GrK GKK(
yhZ @)l".4 E 
 2	A8R88	8"2w{rd|&Aq0AGaKR,,,x 01DNN<L;MeW U&&,X&6%7z$ H%%&Cz$**Q-
fE%bT4Lzz!}o		 
 % : 
+Y!^T!WQR[MT.xj 9% &+4::+<+<+>%?$@ A%%)__$5U	
<
 ;;t

  &
&, -5:5E	
G

G# 
*rQ   c                V   t        | dt        |             }g }t        |||      D ]  \  \  }}\  }	}
t        ||      }t	        |      dkD  rdj                  t        t        t        |t        |                        }dj                  t        t        t        ||
                  }dj                  t        t        t        ||D cg c]	  }||
v s| c}                  }|j                  dt        |       d| d| d| d| d	       |\  }|j                  dt        |       d| d
| d        |sJ t	        |      dk(  r	|d   dd  g}d| dt        |j                  j                                d|j                   ddj                  |      z   dz   dz   }|S c c}w )NrV   r@  ,z* out_specsr  zU which implies that the corresponding output value is only varying across mesh axes {z} and not {z4}, but it was inferred to be possibly varying over {}zS which implies that the corresponding output value is replicated across mesh axis 'z0', but could not infer replication over any axesr   rG  rH  z}' was given out_specs which require replication which can't be statically inferred given the mesh:

The mesh given has shape rb  rc  r  zCheck if these output values are meant to be replicated over those mesh axes. If not, consider revising the corresponding out_specs entries. If so, consider disabling the check by passing the check_vma=False argument to `jax.shard_map`.)rd  r   rM  _unmentionedr   r   r$  r   _spec_to_vmar   r?   r   r   re  r_   )rj   r^   rS  r  r   r6   r  rV  r   rW  vmaunmentionedneed_vmagot_vmar  diff	need_rep_r   s                     rO   r   r   J  s   Q
CF+(	$+6tUE+J K'x3tT*K
;!#c>$T8J#KLMhS.s";<=gXXc#~
K4q184 6 7 8d
kkx()dV 4jdV ,%%,IR12 ji
kkx()dV 4[HJKK$ 
+Y!^T!WQR[MT.xj 9% &+4::+<+<+>%?$@ A%%)__$5U	
<
 ;;t

  &
&
8	
8# 
*1 5s   	F&F&c                `    t        |      }| j                  D cg c]	  }||vs| c}S c c}w rZ   )rl  r_   )r^   r   vma_setr  s       rO   rk  rk  n  s+    '__	9(8!	99	9s   	++c                    t        |dg|j                  z        }	  t        j                  |       j                  | S # t
        t        f$ r Y y w xY wNF)rF   r/  r6  rN  r   r   r   )rj   rS  
dummy_argss      rO   rL  rL  s  sS    dUGdoo$=>*$7Q$$j11
Z	  s   !> AATc                    t        | |      }t        |      }t        t        |      t        |            }d }t        |||      }t	        ||      D 	
cg c]  \  }\  }	}
||
t
        ur||	|
ff c}
}	}S c c}
}	}w )Nc                |    | d u xs7 t        |       t        u xr$ t        |       dk(  xr t        | d         t        u S )NrG  r@  )r   r   r   r   r   s    rO   rh   z_iter_paths.<locals>.<lambda>  s7    19TQ5 0 TSVq[ TT!A$ZST_ rQ   r   )rF   rA   rE   r>   r   r   )rS  r  r   failuresfailures_augspecs_leaf	specs_augr   rW  	fail_datas              rO   rM  rM  {  s    D%((#H-,./1CE1JK&	T$vx>)L)
9 
9)A,AXy]y7 x#
$ 
9 9 
9s   A9c                     | |i |}t        | |             D cg c]9  \  }}t        |t        t        |      t	        |      j
                  z
              ; c}}S c c}}w rZ   )r   r   r   rl  r   rm  )rj   r   r   rg   r   osps          rO   r   r     s]    (8_%67
9a 5b)F1IMM9:
; 
9 9 
9s   >Ac                n    | |i |}g }t        | |             D ]  \  }}t        |      }|j                  |j                  j                  j                  z
  x}	rFt        |j                  j                  |	      }
|j                  t        j                  ||
             |j                  |        |S rZ   )
r   r   r   r   r   r   r^   r   lax_parallelvary_unreduced_cast)rj   r   r   rg   r   new_out_flatr  r  o_avalr   axess              rO   r   r     s    (,8_./ eaAYF\\FOO$8$8$B$BBCyCFOO00)<d,::1dCD! 
rQ   c                  "    e Zd ZdZd Zd Zd Zy)ShardMapPrimitiveTc                &     | j                   |i |S rZ   )
_true_bind)rN   r   r[  s      rO   r   zShardMapPrimitive.bind  s    4??D+F++rQ   c                <    |^}} |j                   t        ||fi |S rZ   )process_shard_mapr   )rN   tracefun_and_argsr[  r   r   s         rO   bind_with_tracez!ShardMapPrimitive.bind_with_trace  s'    JC$"5"";TDVDDrQ   c                N   t        |      }|j                  d      }t        |t        j                        sJ t        j                  t        j                  t        j                  |j                        |d      }|j                  d      t        fd      |d<   |g|fS )Njaxprr   rL   rb   c                      S rZ   rL   )r  s   rO   rh   z3ShardMapPrimitive.get_bind_params.<locals>.<lambda>  s    T rQ   closurer   )ri   r   rx   r   Jaxprr   hashable_partialr   
eval_jaxprr   r.   )rN   r[  
new_paramsr  subfunr  s        @rO   get_bind_paramsz!ShardMapPrimitive.get_bind_params  s    fJNN7#EeTZZ(((  
T__1A1ABE2OF>>+&D$4\4$PJ !8ZrQ   N)rV   rW   rX   multiple_resultsr   r  r  rL   rQ   rO   r  r    s    ,E
 rQ   r  rk      )max_sizetrace_context_in_keyc                |    | j                   j                  |D ci c]  }|t        j                   c}      S c c}w rZ   )r   update_axis_typesr"   Manual)r^   r   r  s      rO   _as_manual_meshr    s5    				-	-#./aq(///
1 1/s   9c                    t        j                  | j                  j                         D cg c]  \  }}||v r||f c}}      S c c}}w rZ   )r   extend_axis_env_ndr   r)  )r^   r   kvs       rO   _extend_axis_envr    sK    		 	 TZZ5E5E5G "7TQ%&+%5 $%a& "7 
8 8 "7s   A
c               |   t        j                         }	t        | j                  |	      }
t	        |
|      }t        ||      }|D cg c]  }|j                   }}t	        t        t        |||      ||      }t        ||      5  t        |      5  t        j                  |      5  t        j                  ||      \  }}}d d d        d d d        d d d        t         |              |r?j                  D cg c]  }|j                  j                    }}t#        | |       |       t	        t$        |      }t'         |       |      D cg c]  \  }}t%        t)        ||||             }}}t+               ft-              z  t/        |      z   }t        ||      5  t        |      5  t        j                  |      5  t        j0                        }d d d        d d d        d d d        t3        ||t/         |             ||      }t5        j6                  |j8                  |j:                        }t	        |
|      }| j=                  g |||||||	      S c c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}w c c}}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nsource_infor^   r]   rb   r  r`   r   )r   currentr   to_jaxpr_tracerr$  r  rJ  r   r  r#   r   
_check_vmapetrace_to_jaxpr_dynamic_check_namesoutvarsrm  _check_vmas_check_shapedarrayr   r   r   r   r   convert_constvars_jaxprri   r   filter_named_axis_effectseffectsr_   emit_eqn)r  primrj   
in_tracersr^   r]   r   r`   r   r  r  
inner_meshtr,  	in_avals_r  
out_avals_constsr  out_vma	out_avalsr   rJ  in_specs_stagedr[  effsconst_tracerss                              rO   _shard_map_stagingr    s   
 !((*+E11{K/?J/*t[1*()aff)()'*dKCX){+ H->z-J H)$H " 9 9!Y GE:vH H H  *-#(==1aqvvzz1G1o'1$j1)!$_%6	!BD4 ",tYd"KL D) DSFS[(5?:/{+ .->z-J .)$.&&u-E. . . TO 12%#>& 
	'	't	G$ov.-	5-5*5y${
, ,- *H H H H H H
 2D. . . . . .s   I#JI5(I(I5J<J!J$J20J&JJ&$J2(I2-I55I?	:JJJ#J&&J/	+J22J;c           	     v    t        |       D ci c]  \  }}||t        |t              r|n|f c}}S c c}}w rZ   )r   rx   r   )r   r   r	  s      rO   r(  r(    sE    #D/
@aU-> j.UUH
< 
@ @ 
@s   "5c                >    t        | t        j                        sJ | S rZ   )rx   r   ShapedArray)rJ  s    rO   r  r    s    	D$**	++	+	+rQ   c                   	 t        |t        j                        sJ |j                  |j                  j
                  j                  k7  r t        d| d|j                  d             |j                  |j                  j
                  j                  k7  r t        d| d|j                  d             t        |      	t         	fdt        |j                        D              }t         |      }|j                  j                  |      }|rt        |      n	t!               }||j"                  z  }|j                  |||      S )	Nzin_specs containing unreduced zM passed to shard_map should be equal to the unreduced present on the in_aval Tzin_specs containing reduced zK passed to shard_map should be equal to the reduced present on the in_aval c           	   3  r   K   | ].  \  }}|t        fd j                  |d      D              z   0 yw)c              3  <   K   | ]  }j                   |     y wrZ   r  r  s     rO   r}   z0_shard_shaped_array.<locals>.<genexpr>.<genexpr>  s     Gtzz!}Gr   rL   Nr	   getr{   r   rg  r^   r	  s      rO   r}   z&_shard_shaped_array.<locals>.<genexpr>  s8      82 $Geii26FGGG 8   47)r^   r   r   rm  )rx   r   r  r   r   r   r   rR  r   r(  r   r   r   r  updaterl  r   rm  )
r^   r   r`   r   rJ  	new_shapemanual_meshnew_shardingrm  r	  s
   `        @rO   _shard_shaped_arrayr    s1   	D$**	++	+	^^t}}))333

( /NN4 !	#$ $ 
\\T]]''///

&tf -88<t8L7M	OP P 
% 8!*4::!68 8)k2+%%;%7,'TY[#dhh#	9|	EErQ   c                    t        |t        j                        sJ |j                  |j                  j
                  j                  k7  r t        d| d|j                  d             |j                  |j                  j
                  j                  k7  r t        d| d|j                  d             t        |      t         fdt        |j                        D              }|j                  |j                        }|j                  dk(  rt               }ng }t!        ||j                  j
                        D ]  \  }}|r|s|j#                  |       |r|s|j#                  |       2|s|s|j#                  d        H|r|sJ t        |t              r|n|f}t        |t              r|n|f}|j#                  ||z           t%        ||j                  |j                  d}t'               j(                  r j*                  n	t'               }	t-        |	|      }
t/        |	j0                        |rt3        fd|j4                  D              n	t3               }|j7                  ||
|	      S )
Nzfout_specs passed to shard_map should be equal to the unreduced present on the out_aval. Got out_specs=z and out_aval=Tzdout_specs passed to shard_map should be equal to the reduced present on the out_aval. Got out_specs=c           	   3  r   K   | ].  \  }}|t        fd j                  |d      D              z   0 yw)c              3  <   K   | ]  }j                   |     y wrZ   r  r  s     rO   r}   z2_unshard_shaped_array.<locals>.<genexpr>.<genexpr>  s     FdjjmFr   rL   Nr  r  s      rO   r}   z(_unshard_shaped_array.<locals>.<genexpr>  s8      82 FUYYq"5EFFF 8r  r   r   c              3  ,   K   | ]  }|v s|  y wrZ   rL   )r{   r  r   s     rO   r}   z(_unshard_shaped_array.<locals>.<genexpr>,  s     ;!{*:1;s   	r  )rx   r   r  r   r   r   r   rR  r   r(  r   r   r   _normalized_spec_for_avalr&  r   r   r   r-   r$   r   r   r,   r   r   r   rm  r  )r^   r`   r   rJ  r  
names_specout_specname_saval_snew_meshr  rm  r   r	  s   `           @@rO   _unshard_shaped_arrayr    s,   	D$**	++	+	^^t}}))333
	337& 9^^D)*	,- - 
\\T]]''///
	++/& 1^^D)*	,- - 
% 8!*4::!68 8)--dii8*	YY!^sHHj$--*<*<= )	&f&  %fe46)%fe46)() h$..%)\\3H$5$7$=$=d  ! x2,H(()+ 
;dhh;	;$; 	9|	EErQ   c               V   t        |j                  ||      D ]T  \  }}	}
t        ||||
|	j                        }t	        j
                  |j                  |      rAt	        j                  d       t        ||      5  t        j                  |      5  t	        j                  |       d d d        d d d        |rc|j                  D cg c]  }|j                  j                   }}t        ||      D ]'  \  }}t        |||      rt	        j                  d       |j                  D 	cg c]  }	|	j                   }}	t        t        t         ||      ||      }t	        j"                  |j$                  |j&                        }||fS # 1 sw Y   xY w# 1 sw Y   xY wc c}w c c}	w )NzLshard_map argument avals not compatible with jaxpr binder avals and in_specsz7shard_map can't prove output is sufficiently replicated)r   invarsr   rJ  r   
typecompatJaxprTypeErrorr  r   r  check_jaxprr  rm  _valid_repeatsr$  r   r   r  r  r_   )r   r  r^   r]   rb   r`   r   in_atomsr  r   in_specsharded_avalr  rm  r  out_avals_shardedr  r  s                     rO   _shard_map_typecheckr  3  s    5<<8< CmaGdKGQVVLL??166<0 !B C CC
 k* F,=,=i,H U #(==1aqvvzz1G1Wi0 GXD#x0!!EG 	GG (-}}5!qvv55',i8)#%)		'	't	G$	D    2
 6s0   ;FF	'FF!*F&	F	FFc                    t        t        | |            t        | j                        z
  }t        fd|D              ryy)Nc              3  &   K   | ]  }|v  
 y wrZ   rL   )r{   r   rm  s     rO   r}   z!_valid_repeats.<locals>.<genexpr>M  s     acs   FT)r   rk  r   r   )r^   rm  r   ums    `  rO   r  r  K  s8    
<d#$s4+;+;'<<"2	rQ   c                z   t        | ||      }t        j                  |j                  t        j                        r+t        j                  ||      }t        j                  |      }|j                  |j                        }t        |      t        |j                        k  r|j                  D ]	  }d|_         |S )NT)_make_scoped_manual_shardingr   
issubdtypedtypeextendedr   physical_shardingr   physical_aval_to_sdy_shardingr&  r   r_   dim_shardingsis_open)ctxr^   r   r   aval_inr"  sdy_shardingdim_shardings           rO   _shardy_shard_map_shardingr   S  s     $Ct4"w}}foo6		)	)'2	6B  )G$$W\\2,DOO,,$22 "!l"	rQ   c                N    t        | |t                     }|j                  d      S Nr   )r  r   r  )r  r^   r"  s      rO   _get_token_shardingr  a  s%     $Cqs3"			Q	rQ   c                Z    t        | t              rt               }|j                  s|S | S | S rZ   )rx   r   r%   r   )r^   concrete_meshs     rO   _get_spmdaxis_ctx_meshr  h  s,    l#%'M - 3 3===	+rQ   c                     j                   j                  }|j                  D 	cg c]  }	|	j                   }
}	t	        |t
        j                        r||j                  z
  }n|}t        j                  t        |      |      } j                   j                  |      } j                  j                         D cg c]  } j                  j                  |       }}t        |      }t        ||      }t        |D cg c]  }|j                   |    c}      dk(  rt#        ||      5  t%        j&                  |      5  t)        j*                  || j,                  t)        j.                  t1         j                  j                         |            dg| j2                   j4                  d\  }} j7                  |       d d d        d d d        S t9        t;        t=        t>         ||      | j@                              }tC        jD                  |      }tG        jH                  |      \  }}t        |      }tK         fd|D              }|D cg c],  }t?         ||tM               tC        jN                  |            . }}t         j2                        }tQ         |      g||z   z  |z   |z   }t        jR                  |      jU                         }t9        t;        t=        t>         ||      | jV                              }tQ         |      g|z  |z   }t        jR                  |      jU                         }tX        jZ                  j                         g|z  t9        t;        t(        j\                   jV                              z   }g  j2                  |||}t_        j`                  |t)        jb                  |      ||t^        jd                  j                  tf        jh                  j                  |D  cg c]!  } tf        jj                  j                  |       # c}                   }!t)        j\                  tC        jl                  dto        jp                                     g|z  }"tX        jZ                  j                         g|z  }#t;        t(        j\                  |      }$t;        t(        j\                  |
      }%tf        jr                  ju                  |!jv                  g |"|#|$|%      }&tg        jx                  |&      5  t#        ||      5  t%        j&                  |      5  tG        jz                  |&j|                  |||g      \  }'}(}})t1        |||      D *+ci c]  \  }}*}+t        |      |*f|+ },}*}}+t)        j*                  || j,                  t)        j.                  t1         j                  j                         |(            dg|)|'|,d\  }-}t_        j                  g |j                         D .	cg c]  \  }.}	|		 c}	}.|-D /cg c]  }/tg        j                  |/       c}/       t        |j                               }|j                  t)        j.                  t1         j                  j                         |!j                  d |                   } j7                  |       d d d        d d d        d d d        |!j                  |d  S c c}	w c c}w c c}w # 1 sw Y   :xY w# 1 sw Y   S xY wc c}w c c} w c c}+}*}w c c}	}.w c c}/w # 1 sw Y   mxY w# 1 sw Y   qxY w# 1 sw Y   uxY w)Naxis_contextr@  rL   )dim_var_valuesconst_loweringc              3  h   K   | ])  \  }}t        j                  |j                  |        + yw))r  rJ  N)r:   ir_constantr  )r{   crJ  r  s      rO   r}   z-_shard_map_lowering_shardy.<locals>.<genexpr>  s5      +
!T q););$GG+s   /2)Emodule_contextr	  r  rJ  rx   r   SPMDAxisContextr   r  replace	tokens_inr  r  r   r   r	   r   r  r   r  r:   jaxpr_subcomp
name_stackTokenSetr   r
  r  set_tokens_outr   r$  r   r   avals_inr   jaxpr_const_argsr   r0   r   r   r   r  SdyArrayListbuild	avals_outr*   	TokenTypeaval_to_ir_typer+   ManualComputationOpflatten_ir_valuesManualAxesAttrr)   	ArrayAttr
StringAttrr  r   default_int_dtypeBlockcreate_at_startbodyInsertionPointr4   	argumentsidReturnOpr)  Valueupdate_tokensresults)0r  in_nodesr  r^   r]   rb   r   r`   axis_ctxr  r  shardy_manual_axesnew_axis_contextsub_ctxefftokens
num_tokensr   	out_nodes
tokens_outin_shardingsconst_args_and_avals
const_argsconst_avalsnum_const_argsconst_arg_valuesr  const_args_shardingsnum_dim_varsout_shardingsoutput_typesr   r   manual_computation_opdim_var_typestoken_typesconst_arg_typesin_typesblockr
  token_arg_valuesin_argsrJ  cablock_const_lowering
out_nodes_r   r   s0   `                                               rO   _shard_map_lowering_shardyrM  o  s    ,,($||,!qvv,),.889 %x';';;$#33T"K1&&4D&E'.1mm.C.C.EFsCMMc"F&F6{*t%78+	+
.Q4::a=
./14	$	, %f.?.?	.J %"00
5#..
--CMM113V<
=
- - ++++-i 
$% % 	',c4
E
CLL"#, ..u5 KK(<=*kz?. +)+ + 
 !dKdkk!nM  S''(,3%&*|*CD)*   ,,\:@@B,	',c4
E
S]]$%- #t$&(235BC- --m<BBD-==$$&'*4s4//?@A, 
F3	E	E)9	EH	E$11D**40,	
,,

+FQBMM--a0F
GIJ
 ''
r633568 9;GH-""$%
2+,,k:/%%y1(
((
"
"  AAAoAAC% 	%  #"24"E #)$#BF//,
NCCE?N$&6 z;8HI AtR 
Ar  "//c#--//13CDE
- - &+-J
 LL (5
8H8H8J)K1!)K (5)3(5 6!"((1+ 6 7Z'')*J))$--!6!>!>{
!K9M +N OJz"'# # #* 
	&	&z{	33{ - G /% % % 0 G *L 6# # # # # #s   [3>"[8[=.\B\
\1\&\!]&]<:\86\&B\8\-"
\8,\3B\8]]\	\\&\88]=]]		]]c                  t         j                  j                  rt        | |||||||      S |j                  D cg c]  }|j
                   }	}|j                  D 
cg c]  }
|
j
                   }}
t        t        t        | ||      || j                  |	|      }t        j                  t        |      |      }| j                  j                  |      }t!        ||      5  t        j"                  |      5  t%        j&                  dt)        j*                  |      d ||	|| j,                  g|| j.                  | j0                  t        t2        ||	      t        t2        ||      d\  }}d d d        d d d        | j5                         t        t        t6        | ||      ||| j8                        S c c}w c c}
w # 1 sw Y   UxY w# 1 sw Y   YxY w)Nr  
shmap_body)r
  r  r   result_names)r   use_shardy_partitionervaluerM  r  rJ  r  r$  r   
_xla_shardr  r   r  r  r  r  r  r  r:   call_loweringr  close_jaxprr  r
  r  _pspec_mhlo_attrsr  _xla_unshardr  )r  r  r^   r]   rb   r`   r   r.  r  r  r   r  	in_nodes_r1  r2  rL  r7  s                    rO   _shard_map_loweringrY    s    ""((%XudHiiQ Q  %||,!qvv,), %.1.*.'*c4=x,,	85)#33T"K1&&4D&E'k* DF,=,=i,H D!//bnnU+T7ICMMD$-D ))))'9=*IzBDJ
D D Z 	W\3k:I

4 4! -.D D D Ds0   F8F=#G9A5G.GG	GGc                   | j                   j                  }|j                  }t        |t        j
                        r8|j                  |j                  D ci c]  }|t        j                   c}      }t        ||      S c c}w rZ   )r  r	  r   rx   r   r  r  r   r"   r  r,   )r  r^   r   r/  r   s        rO   r  r    sm    ,,(			$.889!!%-%9%9:HOO	:<D	tT	"" 	;s   Bc                   t        |j                  j                         D cg c]  \  }}||v s| c}}      dk(  r|S t        | ||      }	t	        j
                  |j                  t        j                        r+t        j                  ||	      }	t        j                  |      }|	j                  |j                        j                         }
t        |      t        |j                         k  rt#        t%        |j                              n	t#               }t'        j(                  | |||
|      }t+        j,                  ||t#        |j.                        z  |      }t'        j0                  | ||||      S c c}}w Nr@  )unspecified_dims)r	   r   r)  r  r   r  r  r  r   r  r   r  _to_xla_hlo_shardingr&  to_protor   r_   r   ranger:   wrap_with_sharding_opr<   manual_protor   wrap_with_full_to_shard_op)r  r^   r   r   r  aval_outr   r  sizer"  shard_protounspecifiedsxrb  s                 rO   rS  rS    s6   	tzz//1
FGAtQ+5E4
FG1LH#Ct4"w}}foo6		)	)'2	6B  )G''5>>@+$s4??';; U7<<()AD !!#q';3>@"""{S!1!122D:,		(	(b(L)4
6 6 Gs
   E9
E9
c                   t        |j                  j                         D cg c]  \  }}||v s| c}}      dk(  r|S t        | ||      }	t	        j
                  |j                  t        j                        r+t        j                  ||	      }	t        j                  |      }t        |      t        |j                        k  rt        t        |j                               n	t               }
t	        j
                  |j                  t        j                        rt        j                  |      }t#        j$                  ||t        |j&                        z  |      }t)        j*                  | ||||
      }|	j-                  |j                         j/                         }t)        j0                  | ||||
      S c c}}w r\  )r	   r   r)  r  r   r  r  r  r   r  r   r  r   r_   r   r`  r&  r<   rb  r   r:   ra  r^  r_  wrap_with_shard_to_full_op)r  r^   r   r   r  rd  r   r  re  r"  rg  rb  rh  rf  s                 rO   rW  rW    s^   	tzz//1
FGAtQ+5E4
FG1LH#Ct4"x~~v7		)	)(B	7B!!(+H$s4??';; U7<<()AD w}}foo6  )G""{S!1!122D:,!!#q'<3>@"''6??A+		(	(b(K)4
6 6 Gs
   F<
F<
c                    t        |t        j                        r=t        |       }t	        t        |j                  t        |j                                    S y)Nr2  )	rx   r   r  r(  r   r$  r  r`  r&  )r   rJ  r	  s      rO   rV  rV    s>    d&&'4 Es599eDII./00	rQ   c           
        | D ]  }t        |d      st        |j                  t              s+|j                  j                  j
                  |j
                  k7  r_t        j                  |      }t        d|j                  j                  j
                   d|j
                   d|j                                |j                  j                  } t        |t              rt        d      t        |t              sJ |S )Nr   zMesh shape of the input z4 does not match the mesh shape passed to shard_map  z for shape zyPlease pass `jax.Array`s with a `NamedSharding` as input to `shard_map` when passing `AbstractMesh` to the mesh argument.)hasattrrx   r   r,   r^   shape_tupler   r%  r   rR  r   r    )r   r^   r   rJ  s       rO   get_mesh_from_argsro  $  s     aq**QZZ"G	
	$	$(8(8	8&&q)&qzz'B'B&C D  !T^^-=,>@A 	A ZZ__d l#
	IJ J 
D$			+rQ   c                ,    t        t        | |            S rZ   r   r   )r^   rm  s     rO   _vma_to_specrr  5  s    	
>$$	%%rQ   c                &    t        d | D              S )Nc              3  Z   K   | ]#  }|t        |t              r|n|fD ]  }|  % y wrZ   )rx   r   )r{   r   r   s      rO   r}   z_spec_to_vma.<locals>.<genexpr>9  s>      Cam",Q"6QQDC  C1 Cs   +!+)r   )r   s    rO   rl  rl  8  s    	 Cd C 
C CrQ   c          
        ~t        t              r&t               }	|	j                  s|	nt	        |      t               }
t        t        t        ||
      ||      }t        t        |      }t        ||||      \  }}|D cg c]9  }t        j                  |j                  d   dt        j                  |            ; }}t         |       |       |r't!         |       |       t#        fd|D              }n't#        fdt%        t'        |            D              } |       }t        t        t(        |      |||      S c c}w )Nr   c              3  6   K   | ]  }t        |        y wrZ   )rr  )r{   rr^   s     rO   r}   z"_shard_map_impl.<locals>.<genexpr>L  s     >|D!,>s   c              3  H   K   | ]  }t        t                      y wrZ   rq  )r{   r   r   r^   s     rO   r}   z"_shard_map_impl.<locals>.<genexpr>N  s%      5 k:; 5   ")rx   r   r%   r   ro  r$   r$  r   _unmatch_specrl  
_run_shmapr   mapped_avalr   get_avalr  r  r   r`  r   _match_spec)r  r  r   r   r^   r]   r   r`   r   r  cur_meshin_vmaoutsr  r   r  
src_pspecs
dst_pspecss       `   `         rO   _shard_map_implr  <  s.   
l#%'M - 3 3=DdD)D (	W]D)X{Kt
$|X&&S$T69M-$JNOQt
At}}Q/?@O)O ),o'1>g>>J 5 %c'l 35 5J *	W[$	;?T
+ + Ps   	>Ec           
         |j                   rJ t        |||      }t        t        t        |      ||      }t        ||      }t        j                  |      5  t        ||      5  t        |      5  t        j                  |      5   | j                  | }	t        t        |j                  |	            \  }
}d d d        d d d        d d d        d d d        
fS # 1 sw Y   %xY w# 1 sw Y   )xY w# 1 sw Y   -xY w# 1 sw Y   
fS xY wrZ   )r   ShardMapTracer$  r   ShardMapTracerr  r   set_current_tracer  r#   r   r  call_wrappedr0   to_val_vma_pair)rj   r^   r   r   vmasr`   r  r  r  r   r  r  s               rO   r{  r{  U  s    	
k9
5%7>514>*t[1*u% <'7k'J <*%<'-'8'8'C<
!..*
%C3u44c:;MD'< < < < 
w	< < < < < < < 
wsT   D%C51C)2C	9C)C5	DC&"C))C2.C55C>	:DDc                |   t        j                         5  t        j                  d      5  t	        | j
                        5   t        j                  t        t        | ||            |      cd d d        cd d d        cd d d        S # 1 sw Y   nxY wd d d        n# 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wrv  )	r   eval_contextr   disable_jitr#   r   jitr/   	_unmatch2r^   prev_manualr   r   s       rO   _unmatch_spec2r  `  s     KS__U3 K$,,-KG377?9dKFGJK K K K K K K K K K K@   B2B+B,	B5	B2BB	B2B&	"B22B;c                    t        t        | |      g| }t        |      }t        t        | ||z              } t        d ||      |      S )Nc                    | S rZ   rL   r   s    rO   rh   z_unmatch2.<locals>.<lambda>i      Q rQ   r]   rb   )r   r   rl  rk   )r^   r  r   r   srcnewly_manualdsts          rO   r  r  e  sN    	.{
+3d3#d#,	.{\9
:;#	<;	<Q	??rQ   c                |   t        j                         5  t        j                  d      5  t	        | j
                        5   t        j                  t        t        | ||            |      cd d d        cd d d        cd d d        S # 1 sw Y   nxY wd d d        n# 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wrv  )	r   r  r   r  r#   r   r  r/   _match2r  s       rO   _match_spec2r  k  s     IS__U3 I$,,-IE377?7D+tDEaHI I I I I I I I I I Ir  c                    t        |      }t        t        | ||z              }t        t        | |      g| } t        d ||      |      S )Nc                    | S rZ   rL   r   s    rO   rh   z_match2.<locals>.<lambda>t  r  rQ   r  )rl  r   r   rk   )r^   r  r   r   r  r  r  s          rO   r  r  p  sN    d#,	.{\9
:;#	.{
+3d3#	<;	<Q	??rQ   c                j   t        j                         5  t        j                  d      5  t	        |      5   t        j
                  t        t        | |||            |      cd d d        cd d d        cd d d        S # 1 sw Y   nxY wd d d        n# 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wrv  )r   r  r   r  r#   r  r/   _unmatch)r^   r`   context_meshr   r  r   s         rO   rz  rz  w  s     4S__U3 4,'41377?8T9g#.0 11244 4 4 4 4 4 4 4 4 4 4s?   B)B,A?#	B,	B)?BB	B)B	B))B2c           	         |r!t        |      }t        t        | |            }nt        | j                        }d} t	        t
        | |f|||      |      S )NFr^   r]   rb   r`   r_   )rl  r   r   r_   rk   _add_singleton)r^   r`   r  r   r   	used_axesr  s          rO   r  r  ~  s_    W%I
N4+
,C
DOO
CI
O>z I+
OOP
R RrQ   c                    t        | |      D cg c]'  \  }}|rt        |      |j                  kD  r|nt        ) }}}t	        d |D              rt        |      y c c}}w )Nc              3  ,   K   | ]  }|t         u  y wrZ   r  r  s     rO   r}   z_check_names.<locals>.<genexpr>  r  r   )r   r   r&  r   r   r   )r  avalsr  r   r   s        rO   r  r    sc    5%(
*b! c"g&!G
3 
*$ 
*(4((
T
 )
*s   ,Ac                      e Zd Zy)r   NrV   rW   rX   rL   rQ   rO   r   r         rQ   r   c           	         t        ||      D cg c]  \  }}t        | ||      s|nt         }}}t        d |D              rt	        |      y c c}}w )Nc              3  ,   K   | ]  }|t         u  y wrZ   r  r  s     rO   r}   z_check_vmas.<locals>.<genexpr>  r  r   )r   r  r   r   r   )r^   r  r  r  rm  r   s         rO   r  r    s\    UD)
+b# $D#r2#
? 
+$ 
+(4((
D/ )
+s   Ac                      e Zd Zy)r   Nr  rL   rQ   rO   r   r     r  rQ   r   c           
        t        t        | ||||      }t        j                         5  t	        j
                  d      5  t        | j                        |k(  r9 t	        j                  |t        | |            |      cd d d        cd d d        S  t	        j                  |      |      cd d d        cd d d        S # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY w)NF)r@  )
r/   _matchr   r  r   r  r   r_   r  r,   )r^   r`   r   	src_pspec	dst_pspecr   fns          rO   r~  r~    s    vtYY "" COOE2 
4??{*FSWWR}T9'EFqI   3772;q>      s.   C?B>>	CB>+	C>C	CCc           	     8     t        t        | ||||      |      S )Nr  )rk   _rem_singleton)r^   r`   r   r  r  r   s         rO   r  r    s)    
+>y&))
++,
. .rQ   c                0    t        j                  | dg      S r  )r'   squeezer   s    rO   r  r    s    ckk!aS11rQ   c                0    t        j                  | dg      S r  )r'   expand_dimsr   s    rO   r  r    s    cooa!55rQ   c           	     ~   t         j                  j                  st         j                  j                  sy t	        |       D cg c]   }t        |dg       D ]  }|j                   " }}}	 t        j                  d|       y c c}}w # t        j                  $ r}t        d|j                   d      d d }~ww xY w)Naddressable_shardsrk   zInvalid value (z%) encountered in sharded computation.)r   
debug_nansrR  
debug_infsrC   rd  datar   check_specialr   InternalFloatingPointErrorFloatingPointErrorty)r  r~  r   bufsr   s        rO   _maybe_check_specialr    s    				 	 ):):)@)@&(. 
<T4!5r:
< && 
<& 
<$ 
<f;-
< 
	,	, f
qttf4YZ
[aeefs   %B-B
 
B<B77B<c                  l     e Zd ZU dZded<   ded<   ded<    fdZd	 Zd
 Zd Zd Z	d Z
d Zd Z xZS )r  )r^   r   checkameshr    r^   frozenset[AxisName]r   boolr  c                n    t         |           || _        || _        || _        |j
                  | _        y rZ   )super__init__r^   r   r  r   r  )rN   r^   r   r  	__class__s       rO   r  zShardMapTrace.__init__  s1    	GDI"DDJ##DJrQ   c                &   t        |t              r|j                  |j                  fS t        |t              rt        d|       t        | j                  | j                  | j                  | j                  t               |      }|t               fS )Nz*Shouldn't have any non-shard_map tracers: )rx   r  valrm  r   	Exceptionrz  r^   r  r  r   r   r   )rN   r  val_s      rO   r  zShardMapTrace.to_val_vma_pair  st    #~&WWcgg	C	 B3%HII499djj$**d>N>N3%d9;rQ   c           
        t        t        | j                  |            \  }}| j                  r |j                  d |D        i |\  }}t        |      t        u rt        |      n|}t        d |      }t        t        t        t        | j                        |            }	t        t        t        | j                        |      }
n5t               }t        t        | j                  | j                              x}	}
t         j#                  |      }|r || j                  g|i |}nt%        t&        || j                  t        |j)                               | j                  | j                  |	|
      }t+        j,                         5  t/        j0                  d      5  t3        j4                  d      5  t3        j6                  d      5  t9        | j:                        5   t/        j<                  |      | }d d d        d d d        d d d        d d d        d d d        t?               |j@                  rBtC        |t        t        f      r|n|gtE        |      z  }t        t        tF        |       ||      S tG        | ||      S # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nc              3  2   K   | ]  }t        |        y wrZ   )r   r{   r  s     rO   r}   z2ShardMapTrace.process_primitive.<locals>.<genexpr>  s     )E&))Es   c                    | j                   S rZ   )rm  )r   s    rO   rh   z1ShardMapTrace.process_primitive.<locals>.<lambda>  s
    155 rQ   F)$r0   r$  r  r  abstract_evalr   r   r   rD   r   rr  r^   r   r   r   r   eager_rulesr  r/   _prim_applierr)  r   r  r   r  r   r  r  r#   r  r  r  r  rx   r   r  )rN   r  tracersr[  in_valsr  r  r   r  r]   rb   
eager_ruleout_valsrj   s                 rO   process_primitivezShardMapTrace.process_primitive  s8   S!5!5w?@OGVzz'T'')EW)EPPli&*9o&=%	"9i)4gGL$))<fEFh7<;WEig~dii9I9IJKKh&JDII::6:h

tzz5+@$))


Hi1a  (!7 (9J9J59Q (e$(&7

&C(3771:w'( ( ( ( ( 8$&wu>i#h-/ .BB$22( ( ( ( ( ( ( ( ( (sl   J:!J.7J"J	#J
<J	J"J.J:
JJ	JJ""J+'J..J7	3J::Kc	           
        t        |t              r|| j                  k7  rt        ~|| j                  k7  rt        ~t        t        | j                  |            \  }	}
t        | j                  || j                  z  | j                        }t        |
|      D cg c]  \  }}|t        |      z   }}}t        |	|      D cg c]'  \  }}t        | j                  | j                  ||      ) }}}t        t        t        |      ||      }t        | j                  || j                  z        }t!        j"                  |      5  t%        | j                  |      5  t'        |      5   |j(                  | }t        t        |j                  |            \  }}d d d        d d d        d d d         |       }t        |      D cg c]'  \  }}t+        | j                  | j                  ||      ) }}}t        |      D cg c]  \  }}|t        |      z
   }}}t        t        t        |       ||      S c c}}w c c}}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}}w c c}}w rZ   )rx   r    r^   r  r  r0   r$  r  r  r   r   rl  r  r   r  r  r   r  r  r#   r  r  )rN   r  r   r   r^   r]   r   r`   r   r  in_vmasr  rm  r   in_vmas_r   r   in_vals_r  r  r   	out_vals_	out_vmas_rb   r  r  out_vmass                              rO   r  zShardMapTrace.process_shard_map  s:    $		)ODJJoc$"6"6=>GW$))[43C3C%CTZZPE474JK&#ql1o%KHK"7H57At tyy$*:*:D!D 7H 7W^U3XxHJ K$:J:J,JKJ

 
 
' E)9$))[)Q E
J
'ECj)c#C(=(=s$CDiE E E  !I"9i8:At TYY(8(8$B :H :69)Y6OP71dL&&PHPw~t,hAA L7E E E E E E
:PsT   H?,II#I+2II%I#,I/	I5III 	I##I,c                     t        d| d      )NzEager evaluation of `z` inside a `shard_map` isn't yet supported. Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/jax-ml/jax/issues !r  )rN   call_primitiver   r  r[  s        rO   process_callzShardMapTrace.process_call  s!    

/ 01 	12 2rQ   c                    t        d      )NzEager evaluation of `pmap` inside a `shard_map` isn't yet supported.Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/jax-ml/jax/issues !r  )rN   map_primitiver   r  r[  s        rO   process_mapzShardMapTrace.process_map  s    
	FG GrQ   c                   ~~~t        t        | j                  |            \  }}t        || j                  | j
                  ||| j                        \  }}	t        t        t        |       |	|      S rZ   )	r0   r$  r  r{  r^   r   r  r   r  )
rN   r  r   jvpr  symbolic_zerosr  r  r  r  s
             rO   process_custom_jvp_callz%ShardMapTrace.process_custom_jvp_call  sd    c>S!5!5w?@OGV"3		43C3CW#)4::7Hgw~t,gx@@rQ   c                   |rd}t        |      ~~~~~t        t        | j                  |            \  }	}
t	        || j
                  | j                  |	|
| j                        \  }}t        t        t        |       ||      S )Nzcustom_vjp symbolic_zeros support with shard_map is not implemented; please open an issue at https://github.com/jax-ml/jax/issues)
r  r0   r$  r  r{  r^   r   r  r   r  )rN   r  r   fwdbwdr  	out_treesr  r   r  r  r  r  s                rO   process_custom_vjp_callz%ShardMapTrace.process_custom_vjp_call  s    4c  $$c3	>S!5!5w?@OGV"3		43C3CW#)4::7Hgw~t,gx@@rQ   )rV   rW   rX   	__slots____annotations__r  r  r  r  r  r  r  r  __classcell__)r  s   @rO   r  r    sD    7)*""	+$3:B82GAArQ   r  c                  J    e Zd ZU ded<   ded<   d Zed        Zd Zd
dZeZ	y	)r  r  rm  JaxTyper  c                    || _         t        |t              rt        |      }t        |t              sJ || _        || _        y rZ   )_tracerx   r   r   rm  r  )rN   r  rm  r  s       rO   r  zShardMapTracer.__init__,  s:    DK#scNcc9%%%DHDHrQ   c                2    t        j                   j                        } j                  j                  r j
                  n j                  j                  }t         fd|D              }t        j                  |d|      }t        t         j                  j                   j                  j                        |j                  j                        }t        j                  j                   r j
                  n	t#               }|j%                  ||      S )Nc              3  d   K   | ]'  }j                   j                  j                  |    ) y wrZ   )r  r^   r   )r{   r  rN   s     rO   r}   z&ShardMapTracer.aval.<locals>.<genexpr>8  s&     7a  &&q)7s   -0r   )r   rm  )r   r}  r  r  r  rm  r   r	   r|  r,   r  r  r   r   r   r  rR  r   r  )rN   rJ  rm  re  r   r  s   `     rO   rJ  zShardMapTracer.aval4  s    =="Dkk''$((T[[-D-DC7377D


4D
)C ))4;;+B+BCL ''--$((9;C::|:55rQ   c                r   | j                   j                  r| j                  t               k(  rit	        j
                         5  t        | j                   j                        5  t	        j                  | j                  d         cd d d        cd d d        S y # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wr  )
r  r  rm  r   r   r  r#   r  to_concrete_valuer  rM   s    rO   r  z ShardMapTracer.to_concrete_value@  s    {{TXX4 3 1$++2C2C D 3%%dhhqk23 3 3 3 3 3 3 3s$    B-""B	B-B!	B--B6c                   t        | j                  j                  j                        | j                  z
  }t        | t        |            } t        j                         5  t        | j                  j                        5  t        | j                        }d d d        d d d        | j                  j                  }ddj                  t        t        |j                               ddj                  fdt!        t#        j$                  |j&                              D              S # 1 sw Y   xY w# 1 sw Y   xY w)N(r   z,)
c           
   3  H   K   | ]  \  \  }}}d | d d| d| d	  yw)zOn z at mesh coordinates z = z:
r  NrL   )r{   rP  devicerG  r_   s       rO   r}   z)ShardMapTracer.__str__.<locals>.<genexpr>N  s?      O MS&5 fX*:,c#c%KOry  )r   r  r^   r_   rm  r   r   r   r  r#   r  r   r  r   r$  r   r   npndenumeratedevices)rN   pb_namesblocksr^   r_   s       @rO   __str__zShardMapTracer.__str__G  s    4;;##../$((:HuX'D				 /0A0AB DHH~f ;;DTYYs389:"=J99 O$'t||(Df$MO O O	   s$   ! D7D+D7+D4	0D77E N)returnr   )
rV   rW   rX   r  r  propertyrJ  r  r  rP   rL   rQ   rO   r  r  (  s8    , 	6 	6	O (rQ   r  c           	     v      fd}t        |      t        u rt        |      n|} t        ||||||      | S )Nc                 v     j                   t        t        |       i t              }t	        t
        |      S rZ   )r   r$  r  ri   rD   r  )r   r  
params_tupr  s     rO   applyz_prim_applier.<locals>.applyV  s1    499c.$/D4
3CDDND))rQ   r  )r   r   r   rk   )	r  r`   r  r  r   r]   rb   r   r  s	   ` `      rO   r  r  T  sG    * "&iE!9d9oy)
+5}x&))
+,0
2 2rQ   zdict[core.Primitive, Callable]r  c               :    ~ ~~|D ]  }|t        d|        |S )Nzadevice_put with explicit device not allowed within shard_map-decorated functions, but got device )r   )r^   srcsr	  copy_semanticsr+  r  s         rO   _device_put_eager_ruler  `  sG    
D. Rf HHNxQ R RR 
)rQ   c	           	        t        t        | j                  |            \  }	}
| j                  j                  | j                  j
                  |D ch c]  }t        |      D ]  }|  }}}t        j                  j                  st              |z  rt        d      t        ||
      D cg c]0  \  }}|t        j                  u r|nt        j                   ||      2 }}}| j                  j"                  t%        fdD              z  }t        j&                  | j                  j(                  || j                  j                  | j                  j
                        }n|D ch c]  }t        |      D ]  }|  }}}t              |z  rt        d      t        ||
      D cg c]0  \  }}|t        j                  u r|nt        j                   ||d       2 }}}| j                  }nSt        ||
      D cg c]0  \  }}|t        j                  u r|nt        j                   ||d       2 }}}| j                  }t        j*                  || j,                  |t/        |
            \  }t1              fd       }t3        ||||      }t5        j6                  | j8                        5   |j:                  |g|	i |}d d d        t=        t        j>                  | tA        jB                               }t        |              S c c}}w c c}}w c c}}w c c}}w c c}}w # 1 sw Y   gxY w)Nz7vmap spmd_axis_name cannot appear in shard_map in_specsc              3  <   K   | ]  }j                   |     y wrZ   r  r  s     rO   r}   z#_shard_map_batch.<locals>.<genexpr>{  s     +RaDJJqM+Rr   zCvmapped away explicit mesh axis cannot appear in shard_map in_specsr  c                 4    t                               S rZ   )_batch_out_specs)explicit_mesh_axisout_dimsr   spmd_axis_names   rO   new_out_specs_thunkz-_shard_map_batch.<locals>.new_out_specs_thunk  s    N,>
+-/ /rQ   r   r  )"r0   r$  to_batch_info	axis_data	spmd_namer  rl  r   disable_vmap_shmap_errorrR  r   r   r   r9   
not_mappedr<   
batch_specre  r	   AxisDatar
  batch_subtracetagr   r1   ri   r   r  parent_tracer   r   BatchTracerr   r  )r  r  r   r  r^   r]   r   r`   r   r  in_dimsr   r  usedr  r!  new_in_specsnew_sizenew_axis_datar  r  r  make_tracerr  r  r  s       ` `                @@@rO   _shard_map_batchr0  l  s   
 C 3 3Z@A'7??,,.99"?$L,>?qA?A?D?**00S5H45OPQQ 7+-B 8&&&DOOB>,RR-L - ##t+R>+R'RRH%%h(A(A**,M %"?$L,>?qA?A?D?
% , - - 7+-B 8&&&DOOB4,HH-L - OOM "%Xw!79A x222BAt8TT 9L 9OOM))	599mU7^5-#x 0/ 1/ $79 +-* e001 6tyy5w5*5H6,,e$4$<$<$>@+	[(HJ	//O @- @-
96 6s*   L(:5L.9L4=5L:5M 	MMc           	        | |D ch c]  }t        |      D ]  }|  }}}t        j                  j                  st	        |       |z  rt        d      t        ||      D cg c]0  \  }}|t        j                  u r|nt        j                  |||       2 c}}S ||D ch c]  }t        |      D ]  }|  }}}t	        |      |z  rt        d      t        ||      D cg c]0  \  }}|t        j                  u r|nt        j                  ||d       2 c}}S t        ||      D cg c]0  \  }}|t        j                  u r|nt        j                  ||d       2 c}}S c c}}w c c}}w c c}}w c c}}w c c}}w )Nz8vmap spmd_axis_name cannot appear in shard_map out_specszDvmapped away explicit mesh axis cannot appear in shard_map out_specs)rl  r   r"  rR  r   r   r   r9   r#  r<   r$  )	r!  r  dimsrb   r   r  r+  r  r!  s	            rO   r  r    s   #@$\$-?@A@A@D@**00S^d5JQRRY-/A x***BAy0QQ / /%#@$\$-?@A@A@D@
% - . . Y-/A x***BAt0LL / / Y-/A x***BAt0LL / / A/ A//s   E+&5E1&E7*5E=25Fc	           	     2   |j                         }t        t        | j                  |            \  }	}
|
D cg c]  }t	        |      t
        j                  u }}|
D cg c]!  }t	        |      t
        j                  ur|nd # }
}t        |	|
f      \  }}t        j                  || j                        }t        j                  |      \  }t        ||      D cg c]
  \  }}|s	| }}}t              fd       }t        |g |||||      }t        j                  ||      \  }}|j                  | j                   |ft#        |      z   |      }t%         |       |      \  }}t        ||      D cg c]?  \  }}|6t        j                  t'        j(                  |      j+                               n|A }}}t        ||      D cg c]  \  }}t        j,                  | ||       c}}S c c}w c c}w c c}}w c c}}w c c}}w )Nr  c                 L            } g | d t        |               D        S )Nc              3  ,   K   | ]  \  }}|s	|  y wrZ   rL   r{   axnzs      rO   r}   z>_shard_map_jvp.<locals>.new_out_specs_thunk.<locals>.<genexpr>  s     Jfb"rrJ   
)r   )out_axr   which_nz_outs    rO   r  z+_shard_map_jvp.<locals>.new_out_specs_thunk  s*    FKVKJFLN(CJKKrQ   r   )r   r0   r$  to_primal_tangent_pairr   r=   ZerorB   jvp_subtracer'  nonzero_tangent_outputsr   r1   ri   	traceabler  r(  r   rF   r   r}  to_tangent_aval	JVPTracer)r  r   rj   r  r^   r]   r   r`   r   primalstangentsr  which_nzr   r   f_jvpr  r8  tangent_in_specsr  r[  r   result
primal_outtangent_outr   r;  s         `                   @rO   _shard_map_jvprK    s   !S!=!=wGH'8=EF47"'')F(F=EF47"'')at3F(F23-$
//!UYY
'%2259%'*8X'>EVR"bEE0L 1L T$Bh$B1A$B 3y')& LL0/%&&u'9'9E8eDk;QSYZ&*8:v>*k [9;Q AB	q)99;<qP ;+ ;03J0L	M1",,ua
#	MM' GF F;	Ms$   "G=!&H 
H+HAH!Hc	           
     X  23 t        | j                  |      }|D 	cg c]  }	|	j                   }
}	t        j                  |
      \  }}}t        j
                  ||      \  }}t        t        t        |      ||      }t        j                  || j                  |j                  d      }t        |      }t        j                  |g |g |      \  }3t        |      2t              23fd       }t        g |||      }|j!                  | j"                  |j%                         g||      } 3       \  }}}}}}t'        d t)        ||      D              }t+        |t-        |      |z
  g      \  }}|j.                  rJ t        j
                  |              \  }} |       } t1        |||||      }!t3        |      }"g }#t)        ||      D ]  \  }$}%|$|#j5                  ||$          |%|#j5                  | |%          4r:t7        |"      j8                  }&|#j5                  t;        t=        |&                   p|#j5                  t;        2              g |#t;               ft-        |      z  z   g |z   }t        | j>                  |!      }'t        | j                  |      }(|D 	cg c]  }	|	jA                         r|	 })}	|jB                  D *cg c]  }*|*jD                   }+}*t        |tG        |      |jI                  |j                  j%                               |      },t        t        tJ              ||+      }-|-D .cg c]6  }.t        jL                  | t        jN                  jQ                  |.      d       8 }/}.tS        jT                  |jV                  jX                        }0t        jZ                  | g |'|(|)|/||,|0t]        j^                               }1|/D ]	  }	|1|	_0         tc        ||/|      S c c}	w c c}	w c c}*w c c}.w )NFr  c            	                \  } } }}} } t        j                  | 
             \  } }r-|D cg c]!  }t        t        	|j                              # }}nt              gt        |      z  }g ||S c c}w rZ   )r  r2   r   r   rm  r   )r   
out_knowns	res_avalsout_known_specsr   	res_specs	all_namesauxr`   r^   r   s         rO   known_out_specsz0_shard_map_partial_eval.<locals>.known_out_specs  s    (+%Aq*iA**:7HIA;DEa1^D!%%01EiEY<.3y>1i)_)y)) Fs   &A<r   c              3  6   K   | ]  \  }}|d u xr |d u   y wrZ   rL   r{   f1f2s      rO   r}   z*_shard_map_partial_eval.<locals>.<genexpr>  s$     Mfb"d
)rTz)M   r   r  )2r$  r  pvalr  partition_pvalsr2   r   r   trace_to_subjaxpr_nounits_fwd2r'  r   _promote_scalar_residualspartial_eval_wrapper_nounits2_all_newly_manual_mesh_namesr1   ri   r  r(  r   sumr   r4   r   	constvarsr5   iterr   nextrm  r   r   new_instantiated_constis_knownr  rJ  r   r  r   JaxprTracer
PartialValunknownr   r  r  r_   new_eqn_reciper   r  reciper3   )4r  r   rj   r  r^   r]   r   r`   r   r  in_pvals	in_knownsr,  	in_constsunk_in_specsknown_in_specsin_avals_shardedf_knownrT  known_paramsr   in_fwdout_fwdrN  rO  r  envnum_res
out_constsnon_fwd_resunk_out_specsr   known_out_specs_resres_avals_iterrQ  rW  rX  res_vmar  env_tracersunk_arg_tracersr  r  
unk_paramsr  r   out_tracersr  eqnrR  rS  s4       ` ``                                          @@rO   _shard_map_partial_evalr    s    %%w/'%&aff&(&#%#5#5h#? )Xy!#!2!29h!G,T;	J%x1''599allEJ!"!119,*,.,'3*4=)0* 1* 4*<N*<&5"-/, 	##E$6$6%,%?%?%A$NI$N$0	2# 8;u4&':y%MFG8LMM'&sSX-?,@A*k__	&&z?3DE-$&67Iz;G#	?.)FG$ 
'fb"	~~b)*	'+,	~&**>$89:9&
' 9!##c(!225E|5EE,e22C8-E))3/+ '<1qzz|Q</<',}}5!qvv55#M2--53C3C3V3V3X-Y'[B* ',i8-#%) $% r}}'<'<Q'?F %+ %		'	't	G$
%!Q=!Q;!Q!Q%{J 0 8 8 :	<# &a3&	Zj	99E 'f =5%s   PPP+P"";P'c	           
     D  ./01 t        t        | j                  |            \  }	}
t        d |
D              }|j	                         }t        j                  || j                  ||j                        \  }0t        |0      }t        |      .t        0      .0fd       }t        |||      }|j                  | j                  |g|	|      } 0       \  }}/}}}t        d t!        ||      D              }|d | }||d  }t#        |||	||      }t!        |||      D cg c]"  \  }}}t%        |dd      dk(  xr
 |d u xr |d u $ }}}}t'        |      5  t)        t+        |            5  t-        j.                        5  t1        /|      /d d d        d d d        d d d                }t!        |||      D cg c]  \  }}}||| }}}}t3        |      } g }!t!        ||      D ]  \  }}||!j5                  ||          ||!j5                  ||          4r:t7        |       j8                  }"|!j5                  t;        t=        |"                   p|!j5                  t;        .              g |!t;               ft?        |      z  d t!        ||      D        }#t        d	 t!                |      D              1t        1      1fd
       }$t        |#|$|      }%/fd}&t!        |
|      D '(cg c]
  \  }'}(|(s	|' })}'}(|j                  | j@                  tC        jD                  |&/j                        g|||)|%      }*t3        |*      }+t!        ||      D (,cg c]1  \  }(},|(rt7        |+      nt
        jF                  jI                  |,      3 }-}(},t        tK        t
        jL                  |       |||-      S c c}}}w # 1 sw Y   JxY w# 1 sw Y   OxY w# 1 sw Y   TxY wc c}}}w c c}(}'w c c},}(w )Nc              3  R   K   | ]  }t        |      t        j                  u ! y wrZ   )r   r=   r=  r  s     rO   r}   z'_shard_map_linearize.<locals>.<genexpr>  s     :Aa':s   %'r  c            	     .           \  } }}}}}t        | ||      D cg c]  \  }}}||| } }}}        }r-| D cg c]!  }t        t        |j                              # }	}nt        
      gt	        |       z  }	g |	|S c c}}}w c c}w rZ   )r   r   r   rm  r   )rO  r   rs  rt  rw  rW  rX  rb   r   rQ  rR  r`   linearize_outs_thunkr^   r   s             rO   fwd_out_specs_thunkz1_shard_map_linearize.<locals>.fwd_out_specs_thunk"  s    *>*@'Iq!Q$'	67$C / /yq"bJ2:  /I /!I;DEa1^D!%%01EiEY<.3y>1i#Y###/ Fs   B&Br   c              3  6   K   | ]  \  }}|d u xr |d u   y wrZ   rL   rV  s      rO   r}   z'_shard_map_linearize.<locals>.<genexpr>5  $     Q&"bB$J-2:-QrY  r   rL   c              3  ,   K   | ]  \  }}|s	|  y wrZ   rL   r6  s      rO   r}   z'_shard_map_linearize.<locals>.<genexpr>P  s     A62rbBAr9  c              3  *   K   | ]  \  }}|r|  y wrZ   rL   r6  s      rO   r}   z'_shard_map_linearize.<locals>.<genexpr>Q  s      #62r!  #s   c                      S rZ   rL   )tangent_out_specss   rO   tangent_out_specs_thunkz5_shard_map_linearize.<locals>.tangent_out_specs_thunkS  s	    rQ   c                 2    t        j                  dg|  S rS   )r   r  )r   	lin_jaxprs    rO   	f_tangentz'_shard_map_linearize.<locals>.f_tangent[  s    ??9b0400rQ   r   )'r0   r$  r<  r   r   r=   linearize_subtracer'  r   _promote_scalar_residuals_linr_  r1   ri   r  r(  r`  r   r5   rd  r  r#   r  r   r  _promote_scalar_residuals_jaxprrb  r   rc  rm  r   r   r   tangent_tracer   r   r=  from_primal_valuer   maybe_linearize_tracer)2r  r   rj   r  r^   r]   r   r`   r   rC  rD  nzs_inf_primalr  
fwd_paramsall_fwd_resultsrO  nzs_outru  rs  rt  num_res_outrx  primals_out	residualsrJ  rW  rX  args_to_promoterb   rw  
res_avals2r|  rQ  r}  r,  r  tangent_paramsr  r  r8  nz_tangents_innz_tangents_outnz_tangents_out_iterprimaltangents_outrR  r  r  r  s2       ` ``                                      @@@@rO   _shard_map_linearizer    sb    S!=!=wGH'8:::&!#%#8#8FALL$* ( *85IJ(*4=) 45	$ 6	$ ()Y*  //8.g.
</8L8N5)WifgQC<PQQ+-+-+';L)),Y)HJ J%r2 T7B/25S"*StS J/ J{+ L/$<=L)$L 0	?KIL L L )#&y&'#B . .iaR:"*  .* .
#.)FG$ 
'fb"	~x|$	y}%	~&**>$89:9&
'C9 CvC0 CAS6%:AC, #S1BG-L # # 12 3,8O{4.
1 &)6%:A'1bbAA.A//	||I)*>*>? ***(*+9;/ o.$'$=? b& 13$+,8Q8QRX8YY ?, ?	WR..6Wl	[[[JL L L L L L
.< B?sZ   0'O!'P=O5O( O5(PP
PP6P(O2-O55O?	:PPc                     | |i |} |       \  }}}}}}t        d t        ||      D              }|d | }	||d  }
|	D cg c]'  }t        |dd      st        j                  |d      n|) }	}g |	|
S c c}w )Nc              3  6   K   | ]  \  }}|d u xr |d u   y wrZ   rL   rV  s      rO   r}   z0_promote_scalar_residuals_lin.<locals>.<genexpr>m  r  rY  r   rL   r@  )r`  r   rd  r'   	broadcast)rj   r  r   rg   r   r   rs  rt  r  r  rC  r   s               rO   r  r  i  s    	46# 4 6!Q1fgQC<PQQ+,;)'!# .5Q-Ds}}Q%!K #) #	)	g	#s   ,A6c                d    | |i |\  }\  }}}}}t        |||j                        D 	
cg c])  \  }	}
}|	d u xr |
d u xr |j                  j                   + }}
}	}t	        ||      }|D cg c]'  }t        |dd      st        j                  |d      n|) }}||||||ffS c c}}
}	w c c}w )Nr   rL   r  )r   ra  rJ  r   r  rd  r'   r  )rj   r   rg   r  in_fwdsout_fwds	out_pvalsrw  ru  rW  rX  r  whichr   s                 rO   r]  r]  t  s    ;<d;Mf;M8%	8'8Y
C5??CE Er2q :9"*9QVV\\)99 E% E
)%
7%#% /6a".Ea&1L %* %	(Iz3?	??E%s   .B&/,B-c           	          fd}t         j                        D cg c]4  \  }}|r!t        j                  dd|j                        n|j                  6 }}}g | j
                  D cg c]  }|j                   c}}t        j                  t        j                  | j                        |      \   }} S c c}}w c c}w )Nc                     t        | t        j                        g      \  }}t        |      D cg c]  \  }}|rt	        |      n| }}}t        j                  |g| S c c}}w rZ   )r4   r   ra  r   r  r   r  )res_and_argsr{  r   r   wr  r  s        rO   r   z,_promote_scalar_residuals_jaxpr.<locals>.fun  sd    <#eoo*>)?@IC58e_
ETQ>!q(
EC
E??5#--- Fs   A'r@  r   r   )r   ra  r   unmapped_avalrJ  r  r  r  r   r   r   )r  r  r   r  r  rO  r,  r   s   ``      rO   r  r  ~  s    .
 u681a 45t!!!Q/!&&@ 8) 8:y:ELL9qAFF9:())ll35#3#34h@+%A	,89s   9C/Cc                ^    t        |      }t        | |      D cg c]  }||vr|
 c}S c c}w rZ   )rl  _all_mesh_names_except_spmd)r^   r   r   name_setr  s        rO   _unmentioned2r    s<     $(0{C 
 h  
    
 s   *c               V   d }t        ||       D 	
cg c]  \  }	}
t        |
      t        j                  u r,t        j                  t	        |	|
j
                              nds&t        j                  |
      t        j                  k(  r|
n: ||
t        t        j                  j                  t        |	                         } }	}
t        |      D 	
cg c]N  \  }	}
t        |
      t        j                  ur|
n+t        j                  t	        |	|
j
                              P }}	}
t        | t!        |      f      \  }}fd}dj"                  j$                   d|_        t)        j*                  |j"                        }t        j,                  |      \  }t/        j0                  ||      \  }}t        ||       D 
cg c]5  \  }}
t        |
      t        j                  urt3        j4                  |      7 c}
}t        |      D 
cg c]#  \  }}
t        |
      t        j                  us"|% c}
}z   }fd}	 t7        j8                  |g|t!        |      |d}tS         |             S c c}
}	w c c}
}	w c c}
}w c c}
}w # t:        t<        f$ r}t?        d       	 tA        jB                  d	      5  t7        j8                  |g|t!        |      |d}d d d        n# 1 sw Y   nxY wt/        jD                  |       n# t:        t<        f$ r}|d d }~ww xY wY d }~d }~wtF        $ rR}|jH                  \  }tK        |      s.tM        | |       tO         |             |      }tQ        |      d Y d }~#d }~ww xY w)
Nc                    |dk7  r| |z  S | S )Nr@  rL   )r   ys     rO   rh   z&_shard_map_transpose.<locals>.<lambda>  s    aA Q rQ   c                   t        t        j                  |      }t        ||      \  }}t	        j
                  t	        j                        |d      \  }}}} t        j                  |      | }t        j                  |j                  ddg |||       t        |      d  }	t        |      \  }}
t        |
|	      D cg c]y  \  }}t        |      t        j                  u r+t        j                  t        ||j                               n-r|n)t#        j$                  |t'        t)        |                  { }	}}|D cg c]  }t+        j,                  |       }}t/        |||	      S c c}}w c c}w )NFrL   )r$  r=   is_undefined_primalr2   r  partial_eval_jaxpr_nounitsrU  r   jaxpr_as_funbackward_passr  r   r   r   r=  r   rJ  r  psumr   r  r   zero_from_primalr3   )out_ctsr   in_undefr{  undefsjaxpr_knownjaxpr_unknownr   res_reshapedin_ctsin_ct_specsr  r   rw  	res_zerosr`   r]   r  r   r^   s                  rO   fun_trans_callablez0_shard_map_transpose.<locals>.fun_trans_callable  sk   2))40H 40KC')'D'D
ux(0$K114$$[137LUB(@,(@(@'	,F $Hh7NA{ {F35 "a aBGG# ggl4B?@.7$$QmD"k.R(STU 5F 5 7::))!,:I:xF335 ;s   A>E4	E:z
transpose(rF  r   c                 F    t        d t                       D              S )Nc              3  P   K   | ]  \  }}|rt        j                  |         y wrZ   )r   primal_spec_to_cotangent_spec)r{   r  r8  s      rO   r}   zD_shard_map_transpose.<locals>.new_out_specs_thunk.<locals>.<genexpr>  s+      BRb 33B7 Bs   $&)r   r   )r]   
nz_arg_ctss   rO   r  z1_shard_map_transpose.<locals>.new_out_specs_thunk  s*     B"8Z\:B B BrQ   r   zsInvalid nan value encountered in the backward pass of a shard_map function. Calling the de-optimized backward pass.T)*r   r   r=   r=  r   rJ  r   r  float0r	   r$  r   r  r  UndefinedPrimalrB   r   r   	func_namerV   r   r   nonzero_outputsr   r   r   r  r   r   r  ZeroDivisionErrorprintr   r  _raise_no_nan_in_deoptimizedr   r   r   r   r   r   rF   )r  r  r^   r]   rb   r`   r   r   mb_divr  r   all_argsr   r  	fun_transfun_trans_flatr   r   r,  r  r   r   r   e2r   r   r  s    ``` ``                   @rO   _shard_map_transposer    sg    /&
 y'*	 "a 
aBGG	 	ggj{Ir166BC&/6<<?fmm3S!T#djjnnmD"k.RSTUV'  8T*
,b! Awb000!


Zk9b!&&Q
RS 
,$ 
, #GU4[#9:(G4 4$ #-U-=-=-G-G,H Jll-%:J:JK),,Y7)Z%::9gN.( y'*F1ad1gRWW.D ))!, F4(NTQDG2;M;M,MqNO 
B&!!!(,u\7J+y!H2 

H	--K
,:FN /	0 /	 > ?/ ??4  %%%%,05;N/9#%% % % ++A.  12 D /	 &VVFEI
T8:t,?,A'BEKcsO% &s   B6J!AJ:J%#J#	J#&J) )N(8M
L+'L	 	L+	L	L+M
+M:L==MM

N(AN##N(c                   |j                   d   |j                   d   }}|j                   d   |j                   d   }}t        ||      5  t        j                  |      5  t	        t        ||            5  t        j                  |||dd|       \  }}	}
}}d d d        d d d        d d d        t        j                        z
  }t        j                  |      |d  }t        |j                  |g      \  }}t        |      D ci c]  \  }}t        |      | }}}|D cg c]  }|j                  t        |             }}t        ||      D cg c]  \  }}|d u xr |d u  }}}|j                   d   }t        ||      5  t	        t        ||            5  t        j                  |      5  t        j                   |dg|z  |z         }t#        ||	      \  }}	d d d        d d d        d d d        t%        j&                  ||j(                        }t%        j&                  	|j(                        }	t+        ||j,                        \  }}t+        
|j                        \  }}t+        ||j,                        \  }}t+        |j                        \  }}t%        j.                         }g g }}t        |	j,                  d | |      D ]  \  } }!|!s	|r)t1        t3        || j4                  j6                              nt1        t9        ||            }"|j;                   |t=        |||"| j4                                     |j;                  |"        |rHt        ||      D  #cg c]0  \  } }#|#)t1        t3        || j4                  j6                              2 }$} }#n"t1        t9        ||            gt?        |      z  }$tA        ||tC        tD        jF                  |
      ||||$|tI        |j                   |      tI        |j                   |	      
      \  }%}&t        jJ                  |g |||jL                  |%|jN                  |jP                  |jR                        }'tU        |||||      }(t        jJ                  g |(|||jL                  |&|	jN                  |jP                  |jR                        })t        |)j,                        t        |	j,                        k(  sJ t        |j,                  |      D *+cg c]$  \  }*}+tW        |*      t$        jX                  u r|+s|*& },}*}+|,|D ch c]  }||	 c}D -cg c]  }-||-   	 c}-z  },|'|)|
||,|z   fS # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}}w c c}w c c}}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}#} w c c}+}*w c c}w c c}-w )Nr  r^   r`   r   FT)r  )-r[  r  r   r  r#   r  r  partial_eval_jaxpr_customr   r  _jaxpr_forwardingr4   r   r)  r  r   prune_jaxpr_outputs_add_reshapesr   remove_named_axis_effectsr_   r2   r  gensymr   r   rJ  rm  r_  r   r   r`  _pe_custom_paramsr$  opnot_ri   new_jaxpr_eqn	primitiver  r  r  r5   r   Var).saveableunks_ininst_inr  r  r^   r`   r   r  jaxpr_stagedunks_outinst_outrv  num_out_primalsrs  out_varsres_varsr   r  idx_maprt  rW  rX  r  	ins_knownr   out_binders_known
ins_stagedout_binders_stagednewvarr  staged_in_res_specsvarr  rnr  out_res_specs_knownparams_knownparams_staged	eqn_knownfull_res
eqn_stagedr   instnew_instrj   s.                                                 rO   _partial_eval_jaxpr_custom_ruler    sp   
 

7#SZZ%7%::k2CJJ}4M[){+ VV->->y-I V/$<=V 	$$UGWeUHU ;Kx7V V V ++,w6/,_-=>&!+"5"57HI(H"+H"56$!QRUAX6'6)12AW[[A2'236vw3G
HR2:$"*$
H%
H	F	${+ P/$<=P)$P ((tf6NQV6VWK -e[, OK	P P P
 ..{DOOL+//dooN,4,)Q'#++>Q #**5-!Z(3;;?!;;=&#%r )L''159 %fc1 nT388<<01 <T; OP 	vl4BIJK  $% ),Xw)?N%sA19 ^D#((,,?@ N N 	

&t[
9:<>A%jI 1wBGGX.&'.
3::[)4

,+O!Q, y*J,=*J	*J"}}lK<O<O"9) 4EyQ( 8( 8Z 8:L #} , 4 4cooswwP* 
Z	3|':':#;	;;	;"3::w7 3GAtaDHH$T  3( 3
-RAAMa-RS #SS(	J(Hy4H	HHgV V V V V V 72
HP P P P P P(N"3-RSs   	U6U)5!UU)U6V#!V	VV.V!3.V!V!)V.(5V;?)W0W8WWU&!U))U3	.U66V VV!!V+	&V..V8c           	        
 t         j                  d t                      D cg c]   \  }}|xr |j                  j                   " c}}
t        
      sfS j                  sj                  rJ  fd}j                  D cg c]  }|j                   }}t        j                  t        j                  |j                        |      \  }\    
fd}t        
j                  d t                      D cg c]4  \  }}|r!t        j                  dd|j                        n|j                  6 }	}}g |	j                  t               d  D cg c]  }|j                   c}}t        j                  t        j                  |j                        |      \  }\   fS c c}}w c c}w c c}}w c c}w )Nc                     t        j                  dg|  }t        |t        |      t	              z
  g      \  }}|D cg c]  }|j
                  st        |      n| }}g ||S c c}w rS   )r   r  r4   r   r`  r   r  )r   r   	out_knownr{  r   r  r  s        rO   knownz_add_reshapes.<locals>.known)  sq    
//+r
1D
1Cc#hU&;%<=NIs<?
@qAGG>!2
@C
@Y As    A*r   c                     t        | t              g      \  }}t        |      D cg c]  \  }}|rt        |      n| }}}t	        j
                  dg|| S c c}}w rS   )r4   r   r   r  r   r  )	r   res_insr   r  r{  r  r  which_s	         rO   stagedz_add_reshapes.<locals>.staged2  sd    4#e*.ID#58v5F
GTQ>!q(
GC
G??<8c8C88 Hs   Ar@  r   )r   r  r   rJ  r   r   ra  r  r  r   r   r   r   r  )r  r  r  r  r  r  r  r   r  rO  r  s   ```       @rO   r  r     s   
 E<#6#6{E
#CDF1 "affll"" F&	V[,66""<+A+AA	A
 *001aff1(100ll5[%;%;<hH+q"9
 v|':':;CJ'GHJ1a 45t!!!Q/!&&@ J) JNyNL,?,?E
,LMqAFFMN(11ll6l&=&=>J,2 
l	""1F 2JMs   %F9F?9G$G
c
                   t        | |d         \  }
}t        ||d         \  }}||z   }t        |      t        |d   j                        k(  sJ t        |t	        |
      t	        |            }t        ||	d         \  }}t        |      }t        ||      D cg c]  \  }}||
|   n|||   n
t        |        }}}||z   }t        ||	d         \  }}t        |	t	        |      t	        |            }||fS c c}}w )Nr]   rb   r  r  )r2   r   r  ri   r   rb  r   rc  )r  r  kept_outs_knownkept_outs_stagedrs  rt  r   r  r  r  in_specs_knownr   out_specs_knownnew_params_knownr  iter_stagedrW  rX  rQ  out_specs_stagednew_params_stageds                        rO   r  r  >  s?    %Wl:.FG.!%o|K7PQ!_#&99/	_	\'%:%B%B!C	CC	C,~1F$)/$:< &g}Z/HI!_()+ 0367/CE%+R &(^~b!&(nr"K ! E) E //&'7{9ST!=53I%*+;%<>	,	,,Es   #C=c                    t        j                         }|j                  t        fd| j                  D              S )Nc              3  2   K   | ]  }|vr|v r|  y wrZ   rL   )r{   r
  r   
spmd_namess     rO   r}   z._all_mesh_names_except_spmd.<locals>.<genexpr>[  s)      Bj(T[-@  Bs   )r   r   spmd_axis_namesr   r_   )r^   r   axis_envr  s    ` @rO   r  r  W  s=     (''*	 B B 
B BrQ   c                    t        j                         }t        |j                        t	               x}j
                  s|} t        |j                        nt        |j                        t        fd| j                  D              S )Nc              3  8   K   | ]  }|z  vr|v r|  y wrZ   rL   )r{   r
  already_manual_namesr   vmap_spmd_namess     rO   r}   z/_all_newly_manual_mesh_names.<locals>.<genexpr>h  s.      )1EEE;&  )s   )
r   r   r   r  r$   r   r   
axis_sizesr   r_   )r^   r   r  r   r"  r#  s    `  @@rO   r_  r_  ^  sx     (001/')
)(	0	0Dx334 x223	 ) ) 
) )rQ   c           	        t        |       s0t        j                  |      sdgt        |j                        z  d fS |j
                  d   }|j
                  d   }|j
                  d   }t        ||      5  t        j                  |      5  t        t        ||            5  t        j                  |j
                  d   |       \  }}d d d        d d d        d d d        t              st        |       sj                  s|d fS t        ||j
                  d         \  }}t        | |j
                  d         \  }}	t        |j
                  t        |      t        |	            }
t!        j"                  |j                  |j$                        }t        j&                  t)        |j                  |      D cg c]
  \  }}|s	| c}}t)        |j*                  |       D cg c]
  \  }}|s	| c}}|j,                  |
||j.                  |j0                        }||fS # 1 sw Y   ^xY w# 1 sw Y   cxY w# 1 sw Y   hxY wc c}}w c c}}w )	NFr^   r   r`   r  r]   rb   )r  r]   rb   )r   r  has_effectsr   r  r[  r  r   r  r#   r  	dce_jaxprr  r2   ri   r   r   r  r_   r  r   r  r  r  r  )used_outputsr  r^   r   r`   r  used_inputsr   r]   rb   r  r  r  r+  r   new_eqns                   rO   _shard_map_dcer+  p  s   	\	2>>##67S_$d**	F	$

=)+jj%){+ IV->->y-I I/$<=Icjj&9<HE;I I I 
[	#l"3EMM cjj.DEKAx!,

;0GHLAycjjh %i 02J))%--IDcjj+6?wq$$?ckk<8Awq$DAz4#''CG I I I I I I 	@AsT   5H>H1!'H$H1H>;
I
I
&
I1I$H.)H11H;	6H>>Ic          	     ^   t        ||      }	t        ||      5  t        |	      5  t        j                  |      5  t        j                  |d      \  }
}d d d        d d d        d d d        rt        ~t        ||j                        D cg c]!  \  }}t        |j                  t              r|# }}}t        
g ||      }t        j                  |      \  \  }}|j!                         \  }t        j"                  |g||||||d}t%        |t'        |j(                        g      \  }}t+        |      }| D cg c]  }t        |t              rt-        |      nd ! }}t-        |d       J ||fS # 1 sw Y   *xY w# 1 sw Y   /xY w# 1 sw Y   4xY wc c}}w c c}w )NrL   )r  rb   )r^   r]   r   r   r`   )r  r  r#   r   r  r7   discharge_stater  r   r  rx   rJ  r8   ri   r   r  re  r   r4   r   r  rb  rc  )r,  r  r  r^   r]   rb   r`   r   r   r  discharged_jaxprdischarged_constsr   invar	ref_specsr[  rj   params_discharged_out_specsout_and_ref_valsr  ref_vals	ref_vals_r   
new_invalss                            rO   _shard_map_discharger8    s    t[1*{+ O->z-J O)$O*3*C*CE2*N''O O O 11'*8U\\'B 7eUZZ5  7) 7&2JI2J	2JK&,,V4,#1w!..* %%A
AH+*iA ""2S5G4HI(H8n)!# $.a#=Y4G #* #	i		&&	&	X	'O O O O O O7#sE   FF
E=F
F&F$$F*=FF

F	FF!rZ   )
rb   Specsr]   zSpecs | None | InferFromArgsr^   Mesh | AbstractMesh | Noner_   Set[AxisName]r`   r  )ro   AxisName)rj   r   r^   r:  r]   r9  rb   zSpecs | Callable[[], Specs]r_   r;  r`   r  rr   r  r  r   )r   r   r  r   )r  r   r  r
   r  None)rj   r   r^   Mesh | AbstractMeshr   rH   r]   r9  r   Sequence[int]r   zSequence[P]r+  r   r  r=  )r   rH   r   r?  r   z#Sequence[core.ShapedArray | NoFail]r  list[core.ShapedArray | NoFail])r  r   rj   r   rS  rH   r  r9  r   r@  r  r   )rj   r   r^   r>  rS  rH   r  r9  r   r@  r  r   )rj   r   r^   r>  rS  rH   r  r9  r   zlist[set | NoFail]r  r   )r^   r>  r  list[AxisName])rS  rH   r  r9  r   zlist[T | NoFail]r  z1list[tuple[tuple[KeyPath, P], tuple[KeyPath, T]]])r   r   r  r   )r  zpe.DynamicJaxprTracer  core.Primitiverj   lu.WrappedFunr  zSequence[Any]r^   r    r`   r  r   r   r  zSequence[pe.DynamicJaxprTracer])r   r-   )rJ  core.AbstractValuer  zcore.ShapedArray)r^   r    r   r   rJ  rD  r  rD  )r^   r    rJ  rD  r  rD  )r^   r    rm  r;  r  r  )r  mlir.LoweringRuleContextr  zsharding_impls.SdyArray)r  rE  r  zir.Attribute)r  rE  r  
core.Jaxpr)r  rE  )rJ  rD  r  r   )r  r  )r^   r    r   r  r  r  )r  zSequence[core.ShapedArray]r  r=  )
r^   r    r  r-   r  r-   r   r  r  r  )r  zbatching.BatchTracer  rB  r   rC  r  Sequence[batching.BatchTracer]r^   r    r`   r  r   r   r  rG  )rj   rC  )r  zpe.JaxprTracerj   rC  )rj   r   )r  rF  r  Sequence[bool])r^   r    r   r  r  rA  )r  rF  )
r  zCallable[..., pe.RematCases_]r  rH  r  rH  r  core.JaxprEqnr  zStuple[core.JaxprEqn, core.JaxprEqn, Sequence[bool], Sequence[bool], list[core.Var]])r  rH  r  rF  r  rF  r  ztuple[core.Jaxpr, core.Jaxpr])r^   r    r   r  r  tuple[AxisName, ...])r^   r!   r   r  r  rJ  )r(  z
list[bool]r  rI  r  z'tuple[list[bool], core.JaxprEqn | None])
__future__r   collections.abcr   r   r   r   enum	functoolsr   r6  mathr	   operatorr  typingr
   r   r   numpyr  jax._srcr   r   r   r   r   r   r   r   r   r   r   r   r   jax._src.corer   r   r   r   r   r   jax._src.meshr   r    r!   r"   r#   r$   r%   jax._src.pjitr&   jax._src.laxr'   r(   r  jax._src.lib.mlirr)   jax._src.lib.mlir.dialectsr*   r+   jax._src.sharding_implsr,   r-   jax._src.utilr.   r/   r0   r1   r2   r3   r4   r5   r6   rK  jax._src.stater7   jax._src.state.typesr8   jax._src.interpretersr9   r:   r;   r  r<   r=   jax._src.tree_utilr>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   r   safe_mapr$  
unsafe_mapsafe_zipr   
unsafe_zipregister_exclusion__file__r9  r<  rJ   r\   rT   r   rk   rs   rr   r   rd   transformation_with_aux2r   r   r   r   Enumr   r   r   r  r   r   r'  r   r*  r   rk  rL  rx  rM  transformation2r   r   r  MaybeTracer	Primitiver  r   cacher  r  r  DynamicJaxprTracer  r(  r  r  shard_aval_handlersr  r  unshard_aval_handlersr  custom_typechecksr  r   r  r  rM  rY  register_loweringr  rS  rW  rV  ro  rr  rl  r  	EvalTracer{  r  r  r  r  rz  r  r  r  r   r  r   r~  r  r  r  r  Tracer  r  r  r  r  r  device_put_pr0  
BatchTracer  rK  JVPTracer  
JaxprTracer  LinearizeTracer  r]  r  r  r  primitive_transposesr  partial_eval_jaxpr_custom_rulesr  r  r  r_  r+  	dce_rulesregister_discharge_ruler8  rL   rQ   rO   <module>r|     s   # = =      & &         & # % #  ( I I. . . " 6   / @N N N % , * & 4 & $O O O O --Z--Z ! ! !( + 	$ $ 	
8!7<15*3+ $	8!48!.8! (8! 	8!vu >H8 		$	$_E #KK+FK(K59K K (0K FK\  &)6rA( 		/GU+;<2'?R  (*5>JO/: 	 (:5--"*-2;-DI-*-/2-^00*02;0DI0*0/20d"!"*<"AD"H:
 CL	9F	9 9 9 
 
 GVO$   *  , Su51 618 , ,'5 ,:G , ,(, , +/ , >G , '	 ,B *<   &@F$6F;MF( .A  )) *)F1)FT 0E  4++ ,* ';  { #	! 	!  a4	!a4a4H4)42   {$7 8#6$6("&C+. $3  	K
@I
@44!(4R 	 (-49@.
 2 5fqADJJ qAh*T[[ *X2 /1+ 0 &<H!! "
0000&400;H00.006:00 +/00 >G00 
(	00b )9   %/*N0 !/ F:,F:N #: O\` ';   #  @ @
 % I. *I.T (<   $:I+:I6D:I:I"/:I:Ix $  " "; /#)# *#/L#<-2B
B0B5IB)
)!4)9M)$ ? . +[  #"";/ 0rQ   