
    ukip*                      U d Z ddlmZ ddlZddlmZ ddlmZmZmZ ddl	Z	ddl
mZmZ ddl
Z
ddlZddlZddlZddlmZmZmZmZ ddlZddlZddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddlm"Z" ddlm#Z# ddlm$Z% ddlm&Z& ddlm'Z' ddlm(Z( ddlm)Z) ddlm*Z* ddlm+Z+ ddlm,Z, ddlmZ ddlm-Z- ddlm.Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5 dd l4m6Z6 dd!l4m7Z8 dd"l4m9Z9 dd#l:m;Z;m<Z<m=Z= dd$l>m?Z? dd%l>m@Z@ dd&l>mAZB dd'lCmDZD dd(lEmFZF dd)lGmHZH dd*lImJZK dd+lLmMZMmNZNmOZOmPZP dd,lQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmHZY dd-lZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZe dd.lfmgZgmhZh dd/limjZj  G d0 d1ek      Zleme[cZnZme\eocZoZp ej                  er      Zseeteueveeteuf   d2f   f   Zwe,j                  Zxe'j                  Zye'j                  Zze'j                  Z{e'j                  Z|e'j                  Z}ee{ezeyf   Z~e*j                  Zee|e}f   Ze'j                  Zd3 Ze)j                  	 d	 	 	 	 	 	 	 	 	 	 	 dd5       Zi Zd6ed7<    e-j                  d8d9:      d;        Zd< Zeeej                  j                  <   d= Ze1D ]  Zeee<   	 eee"j                  <   d> Ze j                   D ]  Zeee<   	 d? Ze"j$                  D ]  Zeee<   	 d@ Zeeej(                  <   	 	 d	 	 	 	 	 	 	 	 	 ddAZddBZi ZdCedD<   ddEZeee3<   	 	 	 	 	 	 	 	 ddFZed2eBj4                  j6                  f   Zi ZdGedH<   	 	 	 	 	 	 ddIZi ZdGedJ<   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddKZddLZ G dM dNe      Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddOZddPZ e-j                  dd9:      	 	 	 	 ddQ       Z edRdSdTg      Z G dU dVejN                        Z	 	 	 	 ddWZ	 	 	 	 	 	 ddXZ	 	 	 	 	 	 ddYZ G dZ d[ejX                        Ze%j                  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd\       Z e	j^                  d4]       G d^ d_             Z G d` dae      Z G db dce      Z e       Zddede<   ddfZddgZddhZddiZ	 	 	 	 	 	 	 	 ddjZe%jt                  dk        Z	 	 	 	 	 	 ddlZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddmZe)j                  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddo       Z	 	 ddpZeveef   Ze3eefiZdqedr<   	 	 	 	 ddsZ G dt dne+j                        ZdduZe	j^                   G dv dw             Z G dx dye+j                        Zdz Z G d{ d|      Z G d} d~      Z	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 d dZ G d d      Z ej                  d      ZeАj                  ZeАj                  e       d Zd Zd Z	 	 	 	 	 	 	 	 ddZd Zd Zd Zee8j                  e<    ee8j                  deeլ      e8j                  e<   ee8j                  e<   ee5j                  e<   ee5j                  e<   ee5j                  e<    ee5j                  eЫ      e5j                  e<   d Zd Zd ZddZd ZddZddZdd	 ddZ e9j                  ee       ddZddZddZddZ ej                  d      Zej                  d        Z	 	 	 	 	 	 d	dZ ee9j                  e      	 	 d
d       Z ej                  d      Zej                  d        Z ee9j                  e      	 	 dd       Z	 	 	 	 ddZeveeKeTeSf   e+j                  eedf   f   ZddZ	 	 	 	 	 	 ddZeeKeTf   Z	 	 	 	 ddZedd        Z  G d de      Zed	 	 	 	 dd       Zed	 	 	 	 dd       Zeddd       Z G d d      Z	 	 	 	 	 	 	 	 	 	 ddZed	 	 	 	 	 	 	 	 	 	 	 	 dd       Z e-j                  d8d9:      	 	 dd       Z	 	 	 	 ddZ	eddd       Z
d Zed	 	 dd       Zeee;e<df      Z G d de      Z e-j                  d8d9:      dd       Zd Z	 	 	 	 	 	 ddZ e-j                  dd9:      d        Zd ZddZe)j                  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd       Z	 	 	 	 	 	 ddZ G d de+j                        Z	 	 ddZ	 	 	 	 ddZ	 	 	 	 	 	 	 	 d dÄZ	 	 	 	 	 	 	 	 d!dĄZ	 	 	 	 d"dńZi Z	 	 	 	 	 	 d#dƄZeeeX<   	 	 	 	 	 	 d$dǄZeeeV<   dȄ Z 	 d%	 d&dɄZ!	 	 	 	 d'dʄZ"	 	 d(d˄Z#d̄ Z$d̈́ Z%edd΄        Z&dτ Z'dЄ Z(dф Z)d҄ Z*d)dӄZ+e	j^                   G dԄ dի             Z, G dք de      Z- e	j^                  d4d4ث       G dل dګ             Z.dۄ Z/ G d܄ de+j                        Z0dބ Z1	 	 d*d߄Z2d Z3d+dZ4	 	 	 	 	 	 	 	 d,dZ5d Z6y(-  z1Implementation of pmap and related functionality.    )annotationsN)
namedtuple)CallableSequenceIterable)partialcached_property)Any
NamedTupleUnioncast)api)array)compiler)config)core)dispatch)dtypes)effects)literals)
jaxpr_util)linear_util)op_shardings)sharding_specs)pjit)profiler)sharding_impls)stages)	tree_util)typing)util)
xla_bridge)array_types)ShapedArray)ad)batching)partial_eval)mlir)Layout
AutoLayoutFormat)_jax)jaxlib_extension_version)
xla_client)ir)hlo)PartitionSpec)Sharding)AbstractMeshMeshget_abstract_meshget_concrete_mesh)ArrayMappingAUTOUnspecifiedValuearray_mapping_to_axis_resourcesSingleDeviceShardingGSPMDShardingNamedShardingr1   )safe_mapsafe_zippartition_list	wrap_nametuple_updatetuple_deletedistributed_debug_logunzip2HashableFunctionweakref_lru_cachetuple_insert)AbstractRef	RefEffect)	ArrayLikec                      e Zd Zy)WeakRefListN)__name__
__module____qualname__     U/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/interpreters/pxla.pyrM   rM   Q   s    rR   rM   .c                    | S NrQ   )xs    rS   identityrW   k   s    rR   Tc                   t        |      dk(  r7|d   }|rt        j                  |      }t        t	        |         |g| ||      S t        j                  d       }t        t        || ||            D ]  \  }\  }}}	}
|rt        j                  |      }|t	        |         }|d   j                  |       |d   j                  |       |d   j                  |       |d   j                  |	       |d   j                  |
        d gt        |      z  }|j                         D ]5  \  }\  }}}}}t        |   ||||      }t        ||      D ]
  \  }}|||<    7 t        d |D              sJ |S )N   r   c                     g g g g g fS rU   rQ   rQ   rR   rS   <lambda>zshard_args.<locals>.<lambda>   s    RRR,@ rR            c              3  $   K   | ]  }|d u 
 y wrU   rQ   ).0results     rS   	<genexpr>zshard_args.<locals>.<genexpr>   s     6FV46   )lenr   canonicalize_valueshard_arg_handlerstypecollectionsdefaultdict	enumerater?   appenditemsall)	shardingslayoutscopy_semanticsargscanonicalizeargbatchesishardinglayoutcsbatchresultstindicesaslxcsoutsouts                        rS   
shard_argsr   n   s    	Y!^
q'C%%c*cd3i(#	7)79 9
 ##$@A'(1tY8): 	$a	$#x%%c*cDIE	!HOOA	!HOOC	!HOOH	!HOOF	!HOOB	 )-vD	'9'$+MMO  a	 '1aCa Aq#.D7D) 3gaj 
6g6	66	6	.rR   zrdict[Any, Callable[[Sequence[Any], Sequence[Any], Sequence[Any], Sequence[xc.ArrayCopySemantics]], Sequence[Any]]]rf   i   F)max_sizetrace_context_in_keyc           
        | |t        |t              ry|t        j                  u sK|j                  t
        j                  k(  s.t        j                  |j                  t
        j                        ryt        | t              ry|j                  d   }|j                  |j                        }	 t        | t        j                  |j                   j#                  |j                  ||                  S # t$        j&                  $ r:}|j(                  ^}}t        |t*              r|j-                  d      rY d }~y d }~ww xY w)NTFr   UNIMPLEMENTED)
isinstancer9   r   abstract_tokendtyper   float0
issubdtypeextendedr*   _device_assignmentshard_shapeshapeis_user_xla_layout_equalr)   from_pjrt_layoutclientget_default_layoutr,   JaxRuntimeErrorrq   str
startswith)curr_layoutrv   avaldr   emsg_s           rS   is_default_layoutr      s   H,
8EU0V
d!!!TZZ6==%@

FOO4Z(!!!$!$$TZZ0+ $HH''

KC	EF F 
		 ffGC!#s ?s   0AC4 4E/D<;D<<Ec                    t        d      )Nznumpy masked arrays are not supported as direct inputs to JAX functions. Use arr.filled() to convert the value to a standard numpy array.)
ValueError)xsrn   ro   rp   s       rS   _masked_array_errorr      s     V 	W WrR   c           
        g }t        | ||      D ],  \  }}}|j                  }|j                  t        j                  k(  r8t        j                  |j                  t        j                  t                    }t        j                  |      }	|0|j                  t        j                  |t        ||                   |j                  r|gt!        |      z  }
nDt#        |j%                  |j                        j'                               }|D cg c]  }||   	 }
}|j                  t)        |	||
|             / |S c c}w )Nr   )r?   _addressable_device_assignmentr   r   r   npzerosr   boolr   shaped_abstractifyrk   r   
device_putr+   is_fully_replicatedrd   tupleaddressable_devices_indices_mapvaluesbatched_device_put)r   rn   ro   rp   rz   rV   rv   rw   devicesr   shardsr|   ru   s                rS   _shard_np_arrayr      s	   '%b)W= Ja655Gww&--
((177"((4.
1a""1%DnnS^^Avfh'?@A		%	%s7|#@@IPPRS '(1!A$((nn'hHIJ 
. )s   Ec                j    t        |||| D cg c]  }t        j                  |       c}      S c c}w rU   )r   r   r   r   rn   ro   rp   rV   s        rS   _shard_python_scalarr      s.    	Iw*,-QRXXa[-
/ /-s   0c                    t        | D cg c]8  }t        j                  t        j                  ||j
                        d      : c}|||      S c c}w )Nr   T	weak_type)r   r   TypedNdArrayr   r   r   r   s        rS   _shard_typed_scalarr      sM    	 RXXaqww74H .
 s   =Ac                n    | D cg c]  }|j                   j                   }}t        ||||      S c c}w rU   )_refs_bufr   )r   rn   ro   rp   rV   bufss         rS   _shard_mutable_arrayr      s3     "	#1!'',,	#$	#	Iw	== 
$s   2c                T   t        j                  d       	 t        ||      D cg c]T  \  }}t        |t        j
                        r5t        j                  |j                        r|j                         |hk(  r|V }}}t        |      t        |      cxk(  rdkD  r2n n/t	        j
                  | |||d      t        j                  d       S t        j                  | ||t        |      ||      t        j                  d       S c c}}w # t        j                  d       w xY w)Nbatched_device_put_startr   T)	committed_skip_checksbatched_device_put_end)
enable_x64)r!   
test_eventr?   r   r   	ArrayImplr   is_single_device_shardingrv   r   rd   xcr   list)	r   rv   r   r   r   r   rV   r   r   s	            rS   r   r      s     //,-."2w/ %$!Q1eoo.221::>		s"  %D % 4yCGa__
$)$H
 	OO,-   xT']I,68 	OO,-% 	OO,-s#   D AD
?<D #D 
D D'c                    	 t        t        |         | ||      S # t        $ r}t        dt        |             |d }~ww xY w)Nz!No _shard_aval handler for type: )_shard_aval_handlersrg   KeyError	TypeError)sizeaxisr   errs       rS   _shard_avalr      sK    OT
+D$==	 O
7T
|D
E3NO    	A<Az>dict[type[core.AbstractValue], Callable[[int, int, Any], Any]]r   c                   	 |j                   |   | k7  rt        d|  d| d|j                          	 t        j
                  j                  r'|j                  t        |j                   |d            S |j                  t        |j                   |            S # t        $ r t        d|j                   d|       d w xY w)Nz
Axis size z does not match dimension z
 of shape zCannot split a zD value along axis rY   r   )
r   r   
IndexErrordimr   pmap_no_rank_reductionvalueupdaterB   rC   )r   r   rV   s      rS   _shard_abstract_arrayr     s    Swwt}D6)CD6 J  !y* + + 
 ""((88,qwwa889988,qww5866 
 S
quug-@G
HdRSs   0B &C c                    	 t        t        |          | ||      S # t        $ r}t        dt        |              |d}~ww xY w)a  Returns a function for handling the raw buffers of a single output aval.

  Args:
    aval: The local output AbstractValue.
    sharding_spec: Indicates how the output is sharded across devices, or None
      for non-array avals.
    indices: The pre-computed result of spec_to_indices, or None for non-array
      avals.

  Returns:
    A function for handling the Buffers that will eventually be produced
    for this output. The function will return an object suitable for returning
    to the user, e.g. an Array.
  !No pxla_result_handler for type: N)local_result_handlersrg   r   r   )r   rv   r|   r   s       rS   local_aval_to_result_handlerr     sQ    &C $t*.tXwGG	 C

+DJ<8:?BCCr   z1dict[type[core.AbstractValue], PxlaResultHandler]r   c                    	 t        t        |          | ||      S # t        $ r}t        dt        |              |d}~ww xY w)a  Returns a function for handling the raw buffers of a single output aval.

  Args:
    aval: The global output AbstractValue.
    out_axis_resources: A PartitionSpec specifying the sharding of outputs.
      Used for creating GSDAs.
    global_mesh: The global device mesh that generated this output. Used
      for creating GSDAs.

  Returns:
    A function for handling the Buffers that will eventually be produced
    for this output. The function will return an object suitable for returning
    to the user, e.g. an Array.
  r   N)global_result_handlersrg   r   r   )r   out_shardingr   r   s       rS   global_aval_to_result_handlerr   0  sQ    "C!$t*-dL)LL	 C

+DJ<8:?BCCr   r   c       
           	
 t         j                  j                  r 
st        	      s	 
fd}|g fS t	        t
        j                  |      }t         	
g| \  }}}t         j                  j                  r;t        dfd j                  fdfdt        t
        j                  |      fd|f       ||fS )Nc                 4    t        g| 
	d
S )N)
backend	axis_name	axis_sizeglobal_axis_sizer   namein_axesout_axes_thunkdonated_invarsis_explicit_global_axis_size)
_emap_impl)rq   r   r   r   r   r   funr   r   r   r   r   s    rS   _emap_apply_fnz*xla_pmap_impl_lazy.<locals>._emap_apply_fn^  s<     Sd SGy"+>N 'dG'5'55QS SrR   zRunning pmapped functionzpython functionr   zabstract argsfingerprint)r   disable_jitr   any
unsafe_mapr   abstractifyparallel_callabledistributed_debugrD   fmap)r   r   r   r   r   r   r   r   r   r   r   rq   r   abstract_argscompiled_funr   
const_argss   ```````````      rS   xla_pmap_impl_lazyr   N  s     
&s>/BS S 2T--t4-*;	7Iy*:GT~~"+4 &3+4',Z ##5t<,cee4$g.*C0@0@$,GH(+6	8
 
z	!!rR   c                :    t        | g|i |\  }} |g || S rU   )r   )r   rq   paramsr   r   s        rS   xla_pmap_implr  u  s.    /EdEfE,
		)z	)D	))rR   c                  "    e Zd ZU ded<   ded<   y)EmapInfo
str | Noner   Sequence[Any] | Noner   NrN   rO   rP   __annotations__rQ   rR   rS   r  r  y  s    rR   r  c       
        Z   t        d |	D              rt        d      |
rt        d      t        ||      }|D cg c]
  }|i n||i }}t        ||      }t	        j
                  ||fg      5  t        ||      D cg c]  \  }}t        |||       }}}t	        j                  |      5   | j                  | }d d d        t        |j                        }t        d |D              \  }}d d d         |       }t        j                  |      j                  }|dv rdnd}g }t!              t!        |      k(  sJ t        ||      D ]  \  }}}t#        j$                  d      5  |}t'        |t(        j*                        rd} t#        j,                  d	 d
|j/                  |      f||d n
t1        |      ||      t3        j4                  |      |      }|j7                  |       d d d         |S c c}w c c}}w # 1 sw Y   JxY w# 1 sw Y   $xY w# 1 sw Y   xY w)Nc              3      K   | ]  }|  y wrU   rQ   r`   r   s     rS   rb   z_emap_impl.<locals>.<genexpr>  s     #q#   z,Buffer donation not supported in eager pmap.z9Non-default global_axis_size not supported in eager pmap.c              3  L   K   | ]  }|j                   |j                  f  y wrU   val
shard_axesr`   r{   s     rS   rb   z_emap_impl.<locals>.<genexpr>  s     "NQAEE1<<#8"N   "$>   tpucudarocm)rY   rQ   Fc                    |S rU   rQ   )r   rV   s     rS   r[   z_emap_impl.<locals>.<lambda>  s    q rR   r   )r   out_axesr   r   donate_argnums)r   NotImplementedErrorr  MapTracer   extend_axis_env_ndzip	MapTracerset_current_tracecall_wrappedr   to_map_tracerrE   xbget_backendplatformrd   r   r   r   r   r   pmapgetr   r   arangerk   ) r   r   r   r   r   r   r   r   r   r   r   rq   	emap_infoin_axisr  tracers   r~   tracersansout_tracersoutvalsout_axes_srcr  r"  r  new_outvalsout_axis_srcout_axisoutvaldonate_argnums_r   s                                    rS   r   r   }  s@    	#N##
L
MM!
 , - - w()OVWGgoIw+??W*W
9i
(%I 678 O69$
6KLFCyQ'LGL				& 'Cg&c' e))3/K""N+"NNG\O (^^G$--(#'>>4B.+	\	c(m	++	+(+L(G(L $lHf		 &o	FEOO	,*CHH
l&&y12"?4W(* +-))I*>Hc   
= X M' 'O O sH   G<0H HH/H?3H,BH!HH	HH!H*	c                >     t         fdt               D              S )Nc              3  b   K   | ]%  \  }d nt        fdd | D              z
   ' y w)Nc              3  4   K   | ]  }|d uxr |k    y wrU   rQ   )r`   jru   s     rS   rb   z*_map_schedule.<locals>.<genexpr>.<genexpr>  s!     @1q}.Q.@s   )sum)r`   r   ru   idxs     @rS   rb   z _map_schedule.<locals>.<genexpr>  s@      +1a yt3@BQ@@@A +s   +/)r   rj   )r8  s   `rS   _map_scheduler9    s#    
 
 +$S>+ 
+ +rR   c                   g }t        t        t        |                  D ]  \  }t        fd|D              }t	        d |D              s-t        j                  | ||d|j                  |j                  d nt        |j                              } |j                  |        t        t        |            D ci c]  \  }}||
 }}}| |fS c c}}w )Nc              3  (   K   | ]	  }|     y wrU   rQ   )r`   arg_axisru   s     rS   rb   z_multi_pmap.<locals>.<genexpr>  s     9HHQK9s   c              3  $   K   | ]  }|d u 
 y wrU   rQ   )r`   r'  s     rS   rb   z_multi_pmap.<locals>.<genexpr>  s     
677$
6rc   r   )r   r   r  r   r   )
reversedr   rj   r   r   r   r#  r   r   rk   )	r   infonamesall_axes
used_namesr   r   ru   out_shard_axess	          ` rS   _multi_pmaprD    s     *$y/01 
ga999G

6g
66
((
,,<</4T$,,5GJa 
 ,5Xj5I+JK4D!GK.K	
N	 Ls   >CFakePrimitivemultiple_resultsbindc                  L     e Zd ZdZ fdZd Zd Zd Zd Zd Z	d Z
d	 Z xZS )
r  )r   r&  c                >    t         |           || _        || _        y rU   )super__init__r&  r   )selfr   r&  	__class__s      rS   rK  zMapTrace.__init__  s    	GDNDNrR   c                @    t        |t              r|S t        | |i       S rU   )r   r  )rL  r  s     rS   r  zMapTrace.to_map_tracer  s     #y!jtS"%%rR   c                   ddl m j                  u r | j                  di S j                  u r*t        fdt        j                               f      }n)t        fdt        j                               f      }t        | j                  |      }t        |D cg c]  }|j                  |j                  f c}      \  }}| j                  }t        j                         j!                         t        fd|D              }	t#        || j                  |	      \  }
}t        j$                         5  t'        j(                  d      5   |
| }d d d        d d d        j*                  rD cg c]  }t-        | ||       c}S t-        | |      S c c}w # 1 sw Y   JxY w# 1 sw Y   NxY wc c}w )Nr   )parallelc                8    j                  | d   d         S )Naxesaxis_index_groups)r   rS  )psum)rV   rP  r   s    rS   r[   z,MapTrace.process_primitive.<locals>.<lambda>  s'    HMM$%89 " ; rR   c                 (     j                   | i S rU   rG  )rq   r   	primitives    rS   r[   z,MapTrace.process_primitive.<locals>.<lambda>  s    )H)H rR   c              3  \   K   | ]#  }t        t        |j                               % y wrU   )r9  r   r$  )r`   r~   r@  s     rS   rb   z-MapTrace.process_primitive.<locals>.<genexpr>  s!     J!]3quue#45Js   ),FrQ   )jax._src.laxrP  axis_index_pprocess_axis_indexpsum_prF   r   rl   r   r  rE   r  r  r&  r   get_axis_env
axis_namesrD  eval_contextr   r   rF  r  )rL  rW  r)  r   r   r{   valsr  r?  rA  f_mappedrC  r,  r  r@  rP  s    ` `          @@rS   process_primitivezMapTrace.process_primitive  s   %H)))$T$$.v..HOO#
; eFLLN+
,	.a H%uV\\^'<=?a$$$g.GgFq||4FGD*>>D**,EJzJJH*1dnneXNHn				  cooe4  $g   !!>EFsic>2FFT7N33 G
        Gs0   +F%F6F*!F6G*F3	/F66F?c                    t         rU   )r  )rL  call_primitiver   r)  r   s        rS   process_callzMapTrace.process_call  s    
rR   c                   |d   t        d      t        j                  j                  s2t	        fdf      }t        d|      }| j                  |||      S |d   |d   |d   |d	   f\  }}t        d
 |D              \  }	}
t        |	||
      D cg c]<  \  }}}|2t        t        j                  |      |j                         |      i|n|> }
}}}t        t        t        |       |	|
      }t!        j"                  fg      5  t!        j$                  |       5   j&                  | }d d d        t        | j(                        }t        d |D              \  }}d d d        t        fdt         |             D              \  }}t        t        t        |       ||      S c c}}}w # 1 sw Y   xY w# 1 sw Y   _xY w)Nr   +Nested pmap with explicit devices argument.c                 0     j                   g| i |S rU   rV  )rq   kwargsr   map_primitives     rS   r[   z&MapTrace.process_map.<locals>.<lambda>  s    "4-"4"4S"J4"J6"J rR   TrF  rG  r   r   r   r   c              3  L   K   | ]  }|j                   |j                  f  y wrU   r  r  s     rS   rb   z'MapTrace.process_map.<locals>.<genexpr>
  s     Equuall3Er  c              3  L   K   | ]  }|j                   |j                  f  y wrU   r  r  s     rS   rb   z'MapTrace.process_map.<locals>.<genexpr>  s     GaQUUALL1Gr  c              3  D   K   | ]  \  }}}t        |||        y wrU   )_match_annot)r`   vr~   dstr   r   s       rS   rb   z'MapTrace.process_map.<locals>.<genexpr>  s.      Q(q!S 'y)Q3G Qs    )r   r   r   r   rF   rE  rb  rE   r  _annot_to_flatr   ndimr   r   r   r  r   r  r  r  r  )rL  rj  r   r)  r   rG  fake_primitiver   r   r`  r  rp  axr~   
in_tracersr*  r+  r   outaxesr   r   s    ``                @@rS   process_mapzMapTrace.process_map  s   i$DEE##
J#
 d %dFn##NGVDD5;K5Hy6"23VK5H5J1Iw	EWEED* #&dGZ"@B BaQ ^ nRWWQZRHNAN)*+ BJ B WY-tZ@J		 	 9i"8!9	: H!!$' ,c
+,**C0kG;GGlc7	H
  Q,/Wn>N,OQ QLCwy$'g66B
, ,H Hs+   AF:G,G<3GG
	GGc                   |rd}t        |      ~~~t        j                  |       5   |j                  | cd d d        S # 1 sw Y   y xY w)Nzcustom_jvp with symbolic_zeros=True not supported with eager pmap. Please open an issue at https://github.com/jax-ml/jax/issues !r  r   r  r  )rL  primr   jvpr)  symbolic_zerosr   s          rS   process_custom_jvp_callz MapTrace.process_custom_jvp_call  sR    Nc$$c>				% (Sw'( ( (s   AA
c                    |rd}t        |      ~~~~~t        j                  |       5   |j                  | cd d d        S # 1 sw Y   y xY w)Nzcustom_vjp with symbolic_zeros=True not supported with eager pmap. Please open an issue at https://github.com/jax-ml/jax/issues !rz  )	rL  rW  r   fwdbwdr)  	out_treesr}  r   s	            rS   process_custom_vjp_callz MapTrace.process_custom_vjp_call!  sV    Nc$$3Y				% (Sw'( ( (s   AAc                0   ddl m}m t        fdj                  f      }t        d|      }|j                  t        j                  t        j                         j                              }t        | |di      }| j                  ||fi       S )Nr   )laxrP  c                &    j                        S rU   )
axis_index)r   r   rP  s    rS   r[   z-MapTrace.process_axis_index.<locals>.<lambda>.  s    (%%i0 rR   Frk  )rY  r  rP  rF   r  rE  iotar   int32r   r]  r   r  rb  )rL  r   r  rG  rt  rangedummy_tracerrP  s    `     @rS   r[  zMapTrace.process_axis_index+  s    *0			i(*D #EENHHRXXt002<<YGHET59a.9L!!.</2FFrR   )rN   rO   rP   	__slots__rK  r  rb  re  rx  r~  r  r[  __classcell__)rM  s   @rS   r  r    s1    ()
&4272((GrR   r  c                j    |y t        |      }t        |       D cg c]	  }||vs| c}|   S c c}w rU   )setr  )rs  mapped_axes
annotationmapped_axes_ru   s        rS   rr  rr  5  s8    [!,4[	:A\$9!	::	FF	:s   	00c           	     p   t        |      }|j                  | d       }t        t        j                  |      |d u z   |j                         |      }t        j                         5  ||k(  r|}nt        |      t        |      cxk(  r	t        k(  r<n n9t        j                  |||      }t        t        j                  |      |||      }nL|D|Bt        j                  |||d       }|j                         D 	
ci c]  \  }	}
|	|
||
k  z    }}	}
nt        d d d        ||fS c c}
}	w # 1 sw Y   |fS xY wrU   )dictpoprr  r   rs  r   r   r_  rg   intr&   moveaxis	_moveaxis	broadcastrl   r  )r   r   r  shard_axis_srcdst_annotationshard_axis_outsrcrq  r1  nr   s              rS   ro  ro  ;  s&    '.9d+#rwws|sd{3^5J5J5L%	'# 
 
czf	cd3i	&3	&  c3/f ~sCHn	!!#y#t<f6D6J6J6LMda1q>)MnM
  
	 N
  
	s   %BD):D#
D)#D))D5c                    d g| z  }|j                         D ]
  \  }}|||<    |j                  |      }|j                  |||k  z
  |       t        |      D ci c]  \  }}|	|| c}}S c c}}w rU   )rl   r  insertrj   )	rs  r  r  rq  lstkrp  r   ru   s	            rS   r  r  P  s{    %)FTM#  daCF	$**SC#I%!*3	Dga43C$'	DD	Ds   
A/&A/c                  8    e Zd ZddgZddZed        Zd Zd Zy)	r  r  r  c                     | _         | _        | _        t         fd j                  j	                         D              sJ y )Nc              3  P   K   | ]  }|j                   j                  k    y wrU   )r  rs  )r`   r  rL  s     rS   rb   z%MapTracer.__init__.<locals>.<genexpr>`  s     GssTXX]]"Gs   #&)_tracer  r  rm   r   )rL  r(  r  r  s   `   rS   rK  zMapTracer.__init__\  s9    DKDH DOGdoo.D.D.FGGGGrR   c                    t        j                  | j                        }t        | j                        }t        |j                               d d d   D ]&  }t        j                  |j                  |   ||      }( |S N)	r   r   r  r  r  sortedr   mapped_avalr   )rL  r   r  axis_idxs       rS   r   zMapTracer.avalb  sn    DHH%Ddoo&J:,,./"5 Ddjj2HdCdDKrR   c                    | S rU   rQ   rL  s    rS   
full_lowerzMapTracer.full_lowerj  s    KrR   c                    | j                   j                         D cg c]  \  }}| d|  }}}| j                   ddj                  |       dS c c}}w )N={,})r  rl   r  join)rL  r  rp  
named_axess       rS   __str__zMapTracer.__str__m  sW    )-)>)>)@AAQCq*AJAhhZr#((:./r22 Bs   AN)r(  r  r  dict[core.AxisName, int])	rN   rO   rP   r  rK  propertyr   r  r  rQ   rR   rS   r  r  Y  s0    l#)H  3rR   r  c                    t        | |||||| j                  |||
      \  }}}}}t        | |||||||	|
|d t        j                         |||||      }|j                         }t        |j                  |j                  |j                  g      S )N)r   r   r   r   r   r   avals)lowering_platformslowering_parametersclosed_jaxprr   replicasr   pci)
get_pmap_jaxprrN   lower_parallel_callabler(   LoweringParameterscompilerM   unsafe_callr   r   )r   backend_namer   r   r   r   r   r   r   r   r   r  r  
xc_backendr  r   r  pmap_computationpmap_executables                      rS   r   r   q  s     5C	<,<CLL'#5	521,
Hfc
 -	9i!17D~"E43J3J3Lh %,,./	o11?3N3N&113 
4 4rR   )frozenc                      e Zd ZU ded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ed        Zed        Zy)ParallelCallableInfor   r   	xc.Clientr   core.AxisNamer   r  r   r   Sequence[xc.Device] | Noner   Iterable[int | None]r   "Callable[[], Sequence[int | None]]r   Sequence[core.AbstractValue]r  c                    | j                   rW| j                   D cg c]0  }|j                  t        j                  | j                        k(  r|2 }}t	        |      dkD  sJ |S d }|S c c}w Nr   )r   process_indexr   r   rd   )rL  r   r   s      rS   local_devicesz"ParallelCallableInfo.local_devices  sp    || C12#3#3DLL#AA  Cc CX\\ J cJCs   5A(c                "    | j                         S rU   )r   r  s    rS   r  zParallelCallableInfo.out_axes  s      rR   N)rN   rO   rP   r  r	   r  r  rQ   rR   rS   r  r    s\    ).%%44	%%  ! !rR   r  c                  @    e Zd ZU ded<   ded<   ded<   ded<   ded<   y	)
	ShardInfor  sharded_avalsSequence[core.ShapedArray]out_sharded_avalsglobal_sharded_avalsr  num_local_shardsnum_global_shardsNr  rQ   rR   rS   r  r    s     --//44rR   r  c                  ,    e Zd ZU ded<   ded<   ded<   y)ReplicaInfor  jaxpr_replicasnum_local_replicasnum_global_replicasNr  rQ   rR   rS   r  r    s    rR   r  zset[core.Primitive]_initial_style_primitivesc                .    t         j                  |        y rU   )r  add)r{  s    rS    register_initial_style_primitiver    s    %rR   c                L    t        t        t        | j                        d      S )zThe number of replicas needed for a jaxpr.

  For a eqn, multiply the `axis_size` with the `jaxpr_replicas` of the
  subjaxprs. For a list of eqns, take the maximum number of replicas.
  rY   default)maxr   _eqn_replicaseqns)jaxprs    rS   _jaxpr_replicasr    s     
Zuzz2A	>>rR   c                    | j                   j                  d      }|r(| j                   j                  dd      t        |      z  S | j                  t        v rt        | j                         S y)N
call_jaxprr   rY   )r   r$  r  rW  r  !_initial_style_primitive_replicas)eqnr  s     rS   r  r    sU    zz~~l+*::>>+q)OJ,GGG
}}11,SZZ88rR   c                h    t        t        j                  t        |       j	                         d      S )NrY   r  )r  r   traverse_jaxpr_paramsr  r   )r   s    rS   r  r    s*    	T''@GGI
 rR   c                F    t        |       }||z  }||z  }t        |||      S rU   )r  r  )r  r   r   r  r  r  s         rS   find_replicasr    s5     #5). >1(>9	^%79L	MMrR   c                    ddl m t        fdt        ||      D              } | | } |       }t        fdt        ||      D              S )Nr   )r  c              3  T   K   | ]  \  }}||nj                  ||f       ! y wN)
dimensions)squeeze)r`   r'  rs   r  s      rS   rb   z)_change_argument_ranks.<locals>.<genexpr>  s5      
'3 _c#++cwj+"II   %(c              3  T   K   | ]  \  }}||nj                  ||f       ! y wr  )expand_dims)r`   rV   r   r  s      rS   rb   z)_change_argument_ranks.<locals>.<genexpr>  s5      
!T <aS__QD7_CCr  )rY  r  r   r  )r   r   r   rq   rz   r  r  s         @rS   _change_argument_ranksr    sX    	 gt, 
$ tH'(	 (+ 
 rR   c                l    t         fdt         j                   j                        D              }|}t        j
                  j                  r"t        | j                   j                        }n|}t        j                   j                   j                  fg      5  t        j                  d|j                  t        j                         5  t#        j$                  |j'                         |      \  }}}d d d        d d d        t)              t)         j*                        k(  s%J t)        |      t)         j*                        f       t-         j.                   j                        }|j0                  }|j2                  }	t5        |||||	      }
|||
fS # 1 sw Y   xY w# 1 sw Y   xY w)Nc              3  Z   K   | ]"  \  }}|t        j                  ||      n| $ y wrU   )r   r   )r`   r   r   r  s      rS   rb   z*stage_parallel_callable.<locals>.<genexpr>  s4      :
$ 150@k#--t,dJ:s   (+zIFinished tracing + transforming {fun_name} for pmap in {elapsed_time} secfun_nameevent)r   r?   r   r  r   r   r   r  r   r   r  r   r   r   log_elapsed_timerN   JAXPR_TRACE_EVENTpetrace_to_jaxpr_dynamicwith_unknown_namesrd   r  r  r   r  r  r  )r  r   r  orig_funr  r  constsr  r  r  r   s   `          rS   stage_parallel_callabler    s     : cii8: :- (""((
 ckk33E3E
FC
C0D0D EFG 3		"	"SX%?%?
A 3 *,)B)B

 
 
"M*3&e33 
	3s||#4	4 1	
c#,,/71 1	4 5#--1E1EF(0022&)+& 
&	((#3 33 3s$    1F*)F:F*F'	#F**F3c
                p   ||t        j                  |d         }
nt        j                  |      }
t        ||
|||||||		      }t	        j
                  ||fg      5  t        ||       \  }}}}d d d        t	        j                  |h      }t	        j                  |      }||
|fS # 1 sw Y   =xY wr  )	r   get_device_backendr!  r  r   r  r  remove_named_axis_effectsClosedJaxpr)r   r  r   r   r   r   r   r   r   r  r   r  r  r  r  r   r  s                    rS   r  r    s     \1##GAJ/Gnn\*G
GY	+;W~u	&# I 678 H&=c3&G#E68VH

(
(
<%!!%0,	w&#	55	H Hs   B,,B5PmapComputationc
                  t        j                         dk(  r|r||k7  rt        d| d| d      |j                  }|j                  j                  t        |j                              }|j                  rt        j                  |      }t        |      \  }}t        |      }d|z  |z   }d|z  |z   }t        |      |j                  z   }t        t        |      |j                  z   |j                   t        |      |j"                  z   |j$                  |j&                        }t)        j*                  ||t        |      t        |j,                        z         }d|z  |z   }n|j                  }g }d	}d
}d
}|st        j                  |      dkD  r|rd}nd}t.        j1                  t2        j4                        rt.        j7                  d|j                         t.        j7                  d|j"                         t.        j7                  d|j8                  |j:                         t.        j7                  d|       t.        j7                  d|j<                         t        j                  |      dkD  r|r|j$                  t        j>                  |      k7  ra|j$                  |k(  r$t        d|dt        j>                  |             t        d|j:                   dt        j>                  |             |r'|j@                  dkD  rt        d|j@                         tB        jD                  jF                  rt2        jH                  nt2        j4                  }t.        j1                  |      rBt.        jK                  |d| jL                  tO        |       |j&                  |	|j8                         tQ        jR                  |j8                  |f|f      }|D cg c]  }|d u  }}tU        jV                  t        |j"                        |jX                        }t[        d|      }|
xs |jX                  f} t        j\                  ||fg      5  t        t^        j`                  jc                  |j^                              }!|!rt        d      t        t^        j`                  je                  |j^                              }"tU        jf                  d|tT        jh                        5  tk        jl                  |||||!|| tQ        jn                  |      ||d d ||j                  jq                  t        |jr                              |j8                  |      }#d d d        d d d        tu        #jv                  t        |      | ||||"!|#jx                  |#jz                  |j                  j                  |#j|                        S c c}w # 1 sw Y   sxY w# 1 sw Y   wxY w)NrY   zSpecified axis_size z" doesn't match received axis_size .rU   F)r   r   r   FTzsharded_avals: %szglobal_sharded_avals: %sz(num_replicas: %d  num_local_replicas: %dzdevices: %szlocal_devices: %szOn multi-host platforms, the input to pmapped functions must have leading axis size equal to the number of local devices if no `devices` argument is specified. Got axis_size=, num_local_devices=zOn multi-host platforms, pmapped functions must run across all devices, i.e. num_replicas * num_partitions should equal the number of local devices. Got num_replicas=z, and num_local_devices=zOn multi-host platforms, pmapped functions that both have `devices` specified and contain an inner_pmap must specify an `axis_size` (or remove the `devices` argument). Got nested_replicas=z@Compiling %s (%d) for %d devices with args %s. (num_replicas=%d)r#  z(Ordered effects not supported in `pmap`.MFinished jaxpr to MLIR module conversion {fun_name} in {elapsed_time:.9f} secr  )num_const_argsin_avalsordered_effectsr   	platformsaxis_contextdonated_argsreplicated_argsarg_shardingsresult_shardings	arg_namesresult_namesnum_replicasr  )r  r  r  r   
tuple_argsunordered_effectsr  	keepalivehost_callbacksjaxpr_debug_infoshape_poly_state)?r   process_countr   r  _debug_infosafe_arg_namesrd   r  hoist_constants_as_argsr   jaxpr_const_argsrE   r   r  r   r  r  r  r  r  dataclassesreplacer  loggerisEnabledForloggingDEBUGdebugr  r  r  local_device_countr  r   log_compilesr   WARNINGlogrN   idr   AxisEnvr   should_tuple_argsr"  rA   r  r   r  	filter_infilter_not_inr  JAXPR_TO_MLIR_MODULE_EVENTr(   lower_jaxpr_to_moduleReplicaAxisContextsafe_result_pathsoutvarsr  moduler)  r*  r,  )$r   r   r   r   r   r   r   r   r   r  r  r  r  r   r  r   r  r  r$  const_args_and_avalsr   const_arg_avalsr  jaxpr_avalsno_nested_shardingmust_run_on_all_deviceslog_priorityaxis_envr   r!  r'  module_namer  r  r(  lowering_results$                                       rS   r  r  )  s|   0 	A">
i
'

/0 1Kq	"# # 

%..s<3H3H/IJ)00007"()=">J)N&0G.?N',*?*??Ko!5!55  o!<!<<!9!9	;F
 

c7$)/$:U399=M$MOC&2I''KJN!	%	 1$	 " #''
LL$f&:&:;
LL+V-H-HI
LL;--x/J/JL
LL(
LL$c&7&78w!#(?!6!6w!??)+<1: = 33G<=
?@ @  !334 5227;<		>? ? H33a7
	M 
	 	 !#$ $ %+$7$7$=$=7==,&
JJ|Q||RW''0L0LN
 ##""YL3C2EG(.56dTT\6/6))#f.I.I*J*1*:*:<*&$'+ 7W%5%5$7),< =>? 3)),*>*>?AOABB--l.B.BCE		"	"WH$G$G
I 3 22

')%88B%)((::3u}};MN331!3o336 
//j)#, 8 &:+<)8#2#<#<(7(F(F*6*<*<*H*H*9*J*J

L 
LA 73 33 3s,   WBW*A/WWW	WW c                    ||S t        |      t        u r7t        t        |j                  ||       |j
                  |j                        S t        |      )Nr   )rg   r  r$   rB   r   r   r   r   )r   r   r   s      rS   _pmap_unmap_shaped_arrayrR    sJ    	\$;DzS|DJJd;TZZ!%1 1orR   zdict[type, AvalMapHandlerPair]_pmap_aval_mapping_handlersc                    t         j                  j                  st        j                  | ||      S t
        j                  t        |      d      \  }}|
 || ||      S t        d| dt        |             )NNNzno unmapping handler for z	 of type )	r   r   r   r   unmapped_avalrS  r$  rg   r   )r   r   r   r   handlers        rS   _pmap_unmapped_avalrX    sr    		&	&	,	,dD$//*..tDz<H*!W4t$$
/vYtDzlK
LLrR   c                  b    e Zd ZU ded<   ded<   d
dZddZej                  ddd	 dd	       Zy)r  	ir.Module_hlozPmapExecutable | None_executablec                <    d | _         || _        || _        || _        y rU   )r\  r[  r   compile_args)rL  r0   r   r^  s       rS   rK  zPmapComputation.__init__  s     DDI DO$DrR   c                    | j                   S rU   r[  r  s    rS   	stablehlozPmapComputation.stablehlo      99rR   Ndevice_assignmentc                   | j                   |9t        j                  | j                  fi | j                  d|i}||| _         |S | j                   S )Ncompiler_options)r\  UnloadedPmapExecutablefrom_hlor[  r^  )rL  rf  rd  
executables       rS   r  zPmapComputation.compile  se     #3#?)22
))-((-+-j 
	!%rR   )r0   rZ  r   list[ArrayLike]returnrZ  rU   rl  PmapExecutable)	rN   rO   rP   r  rK  ra  r   annotate_functionr  rQ   rR   rS   r  r    s@    /$$% 	 	!	 	rR   c                4    t        | t              sJ |        | S rU   )r   r$   )r   s    rS   _cast_to_shaped_arrayrq    s    	D+	&,,	&	+rR   c                      e Zd ZU ded<   ded<   ded<   ded<   d	ed
<   ded<   ded<   ded<   ded<   ded<   ded<   d ZddZe	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd       Zy)rg  r
   compiledr  r   r  local_input_avalsSequence[JSharding]input_shardingsSequence[ShapedArray]local_output_avalsoutput_shardingslist[core.Effect]r(  r  Sequence[Any]r)  r*  core.DebugInfor+  c                $   g }t        | j                  | j                        D ]  \  }}t        |t        j
                        sJ |       t        |t        j                        sJ |       |j                  |j                  *t        j                  |j                  |j                        nd         t        | j                  | j                        }t!        | j                  d gt#        | j                        z  | j$                  j'                         |      }t)        | j$                  d| j*                  ||| j,                  | j.                  | j0                  t3        | j4                        t7        t9        t#        |                  d       }|S )Nzparallel computation)r?   rt  rv  r   r   PmapShardingr   r$   rk   sharding_specr   spec_to_indicesr   local_avals_to_results_handlerrx  ry  InputsHandlerrd   rs  r  ExecuteReplicatedr   r(  r  r)  r   r*  r  r  )rL  input_indicesr   spechandle_outshandle_argsexecute_funs          rS   build_execute_funz(UnloadedPmapExecutable.build_execute_fun
  sW   Mt55t7K7KL 7
dn99:@D@:d../55/+ 
(
(T5G5G
H1577 11H1H151F1FHK 4 4!%T-A-A)B B $ ; ; =}NK $DMM3I$(LL+{$($:$:$($8$8$..$()<)<$=$'c-.@(A$BDJK rR   c                    t        | j                  dd       }t        | j                  | j                  || j                  |       S )Nr   )getattrrs  rn  r  rt  )rL  r   s     rS   loadzUnloadedPmapExecutable.load  s=    $--=Kt--{& &rR   Nc                
   ~
|!|j                   rt        j                  |       } |j                  }|!|j                  t        j                  |j                        kD  rOd}t        |j                  |j                  t        j                  |j                        |j                              |j                  |j                  kD  r^t        t        j                  |j                              D cg c])  }t        j                  ||j                        D ]  }| + }}}nAt        j                  |j                        d |j                   }n|j                  t        |j                        k7  rdj!                  t#        t$        |j                              }|j                  |j&                  k(  r2t        d|j&                   dt        |j                         d| d      t        d|j                   d	t        |j                         d
|j&                   d| d	      |j                  t        |      k7  r%t        d|j                  dt        |      d      d}t)        j*                  |      j-                  |j                  |f      }t/        j0                  |j                  ||d|t/        j2                  |       |j                        }||_        t        j6                  |j                        }t)        j*                  |j8                  D cg c]  }|j6                  |k(  s| c}      }t;        |j<                  |j>                        D cg c]I  \  }}tA        jB                  |jD                  |j&                  tG        tH        |      jJ                  |      K }}}tM        ||      }t;        |jN                  |jP                        D cg c])  \  }}| tS        tU        |j&                  ||            n|+ }}}t;        |jN                  |jP                        D cg c];  \  }}tA        jB                  |jD                  |j&                  |jJ                  |      = }}}tM        ||      }tW        jX                  d|jZ                  tV        j\                        5  t_        ta        |j8                              }|J t/        jb                  |j                  | ||||      }d d d        te        |j                  |jf                  ||||||||	      ji                         S c c}}w c c}w c c}}w c c}}w c c}}w # 1 sw Y   ^xY w)Nzocompiling computation that requires {} logical devices, but only {} XLA devices are available (num_replicas={})r   z, ztLeading axis size of input to pmapped function must equal the number of local devices passed to pmap. Got axis_size=r  z$.
(Local devices available to pmap: )zpmapped function requires zR local devices to run due to nested pmapped or other parallel functions, but only z" are available.
(outer axis size: z#, local devices available to pmap: z#compiling computation that creates z shards, but z devices were specifiedrY   F)r&  num_partitionsrd  use_spmd_partitioningenv_options_overridesdetailed_loggingr   @Finished XLA compilation of {fun_name} in {elapsed_time:.9f} secr  )rs  r   rt  rv  rx  ry  r(  r  r)  r*  r+  )5uses_dim_varsr(   refine_polymorphic_shapesr   r  r   device_countr   r   formatr  r  r  r-  r  rd   r  r   r   r   r   r   reshaper   get_compile_optionsuse_detailed_loggingparameter_is_tupled_argumentsr  flatr?   r  r   r   pmap_sharding_specr  r   r$   r   _get_pmap_shardingr  r  rq  rX  r   r  r   BACKEND_COMPILE_EVENT_create_device_listr   compile_or_get_cachedrg  r  r  ) r0   r  r  r   r'  r(  r  r*  r)  r+  r  r,  rf  r   r   r  r   local_devices_strr  rd  compile_optionslocal_device_assignmentr   r'  input_sharding_specsin_shardingsr0  local_unmapped_avals	out_specsout_shardingsexecutable_devicesrs  s                                    rS   rh  zUnloadedPmapExecutable.from_hlo&  s    	#(8(F(F**3/ckkG		!	!BOOCKK$@	@9F$<$<$&OOCKK$@$,$@$@B C 	C 
	!	!F$;$;	; +00@0@0M*N I++M3;;GI  I1 I I ""3;;78P9P9PQ		 	 C(9(9$:	: IIc#s/@/@&AB""cmm3G3C8I8I4J3K L33D2EQHI I *6+B+B*C D%%():):%;$< =##&==/ 2()	,- - 
	!	!S\	1113w<A B 	B N$&HHW$5$=$=		%	%~6%82211%+#.!66s;O 5?O1$$S[[1M hh$))(Q__-M(  &f&:&:CKKH	J D' 	))''d#))7	4J J
 &&=&:<L 'v'?'?N	P D(  	x>	@%)	*P P '$$cll45 D( 	))''

H	N5I 5
 ''>	JM		"	"J!?!?
A . /
',,
-/+++//
++s-
,.h. "))$/&+'%)+ ,0462cIX(JP
5. .s3   3.T<U4U"AU".U6A U5AUU"rm  rU  )r0   rZ  r  r  r  r  r   r  r'  r   r(  rz  r  rz  r*  	list[Any]r)  r
   r+  r|  r  Sequence[str]r,  "mlir.ShapePolyLoweringState | None)rN   rO   rP   r  r  r  staticmethodrh  rQ   rR   rS   rg  rg    s    -11&&++''&&$$""*&  GK $}2(}2$}2 !}2  	}2
 #4}2 !2}2  )}2 }2 "0}2 (}2 "D}2 }2rR   rg  c                  \    e Zd Zg dZ	 	 ddZedd       Zd Zej                  d        Z
y)	rn  xla_executable_unsafe_callbuild_unsafe_callr   r  _unloaded_executablec                X    || _         d | _        || _        || _        || _        || _        y rU   r  )rL  r  r  r   r  unloaded_executables         rS   rK  zPmapExecutable.__init__  s3     )DD.D"DDM 3DrR   c                \    | j                   | j                         | _         | j                   S rU   r  r  r  s    rS   r  zPmapExecutable.unsafe_call  +     002drR   c                    | j                   S rU   r  r  s    rS   xla_extension_executablez'PmapExecutable.xla_extension_executable      rR   c                    t        t        j                  |      }t        | j                  || j
                  j                          | j                  | S rU   )r   r   r   check_arg_avals_for_callr  r  r+  r  )rL  rq   	arg_avalss      rS   callzPmapExecutable.call  sI     D$$d+IT]]I!66GGI4T""rR   N)r  rg  rl  zCallable[..., Any])rN   rO   rP   r  rK  r  r  r  r   ro  r  rQ   rR   rS   rn  rn    sJ    B)4$:4   # #rR   rn  c                T    |D cg c]  }t        j                  | |       c}S c c}w rU   )r   r~  )r   specsr  s      rS   r  r    s#    AF	G.
%
%gt
4	GG	Gs   %c                  (    e Zd ZdZ	 	 ddZd Zd Zy)r  )rW  r  
in_layoutsr  r  Nc           	         t        t        ||t        j                  j                  gt        |      z        | _        || _        || _        || _	        || _
        y rU   )r   r   r   ArrayCopySemanticsREUSE_INPUTrd   rW  r  r  r  r  )rL  r  r  r  r  s        rS   rK  zInputsHandler.__init__  sS    L*				*	*+c,.??ADL %D DO&D&DrR   c                $    | j                  |      S rU   )rW  )rL  input_bufferss     rS   __call__zInputsHandler.__call__  s    <<&&rR   c           	     p    d| j                    d| j                   d| j                   d| j                   d	S )NzInputsHandler(
in_shardings=z,
in_layouts=z,
local_devices=z,
input_indices=r  )r  r  r  r  r  s    rS   r  zInputsHandler.__str__  sP     --. ///* +!//0 1!//0	3 4rR   rU  )rN   rO   rP   r  rK  r  r  rQ   rR   rS   r  r    s!     ) >B!''4rR   r  c                      e Zd ZdZd Zd Zy)ResultsHandlerhandlersr  	out_avalsc                .    || _         || _        || _        y rU   r  )rL  r  r  r  s       rS   rK  zResultsHandler.__init__  s    DM&DDNrR   c                l    t        | j                  |      D cg c]  \  }} ||       c}}S c c}}w rU   )r?   r  )rL  out_bufshr   s       rS   r  zResultsHandler.__call__  s)    #+DMM8#DE4AdGEEEs   0N)rN   rO   rP   r  rK  r  rQ   rR   rS   r  r    s     9)
FrR   r  c           
     ,   t        ||       D cg c]7  \  }}t        |j                  |j                        j	                               9 }}}t        | ||      D cg c]  \  }}}t        |||       }}}}t        |||       S c c}}w c c}}}w rU   )r?   r   devices_indices_mapr   r   r   r  )unmapped_local_out_avalslocal_shardingsr~   r   out_indicesidcsr  s          rS   r  r    s     !):R SUQ q,,TZZ8??AB U+ U $$<o{[ 
$4 #4D1(  
/3K	LLUs   <B	 Bc           	     z    t        | |      D cg c]  \  }}t        |||       }}}t        |||       S c c}}w rU   )r?   r   r  )global_out_avalsrn   r   global_avalr~   r  s         rS   global_avals_to_results_handlerr     sM     %%5yA
+q $KI>(  
)-=	>>	s   7c                  r    e Zd ZdZg dZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	dZd Zd Zej                  d        Z
y)
r  zIThe logic to shard inputs, execute a replicated model, returning outputs.)r  r   r   
in_handlerout_handlerhas_unordered_effectsr  r)  has_host_callbacks_local_deviceskept_var_idxmutpgle_profiler__weakref__Nc                    || _         || _        || _        || _        || _        t        |      | _        || _        | j                   j                         | _	        || _
        |	| _        |
| _        || _        || _        y rU   )r  r   r   r  r  r   r  r  r  r  r)  r  r  r  r  )rL  r  r   r   r  r  r(  r  r)  r  r  r  r  s                rS   rK  zExecuteReplicated.__init__  s|     )DDIDL DO"D!%&7!8D*D--;;=DDN0D$DDH&DrR   c                    | j                   rQ| j                   D cg c]6  }t        j                  j                  || j                        j
                  8 }}g ||}|S c c}w rU   )r  r   runtime_tokensget_token_inputr  r   )rL  
input_bufsefftokenss       rS   _add_tokens_to_inputsz'ExecuteReplicated._add_tokens_to_inputs'  sk     )) 
!
!
1
1#t7J7J
K
P
Pf  *V)j)js   ;A c                   t        | j                        D ]4  \  }}t        j                  j	                  ||j                  |             6 t        | j                  |      D ]  \  }}t        |      dkD  sJ t        |      dk(  r7t        j                  j                  |t        j                  |d                \g }|D ]P  }t        |j                  t        j                        sJ |j!                  |j                  j"                  d          R t%        t'        |d      t)        d            }	t+        j,                  d|	|      }
t        j                  j                  |t        j                  |
              y )Nr   rY   rV   )r   )rj   r  r   r  set_output_runtime_token	get_tokenr  r  rd   set_token_resultr   Tokenr   rv   r   r;   rk   r   r=   r4   Pr   $make_array_from_single_device_arrays)rL  
token_bufssharded_tokenru   devicer  	token_buftoken_devicestokenr~   global_token_arrays              rS   _handle_token_bufsz$ExecuteReplicated._handle_token_bufs0  sL    t223 .	666
-))!,.. d22J? 
Y^a	Y1	00djj16NO 	EEENNN,O,OP
PP


u~~@@C
D	E $}c2AcF;"GG!Y
 	00./	

rR   c                   t         j                  j                  rt        d| j                   d      t        |      D cg c]  \  }}|| j                  v s| }}}| j                  rg || j                  j                  }| j                  |      }t        j                  j                  | j                        5  | j                  s| j                  s| j                   r| j#                  |      }| j$                  j'                  |d      }t(        dk\  r,|j+                  d gt-        | j                        z  d      }n$|j/                  t-        | j                              }|j1                         }| j3                  ||       n| j$                  j'                  |      }t(        dk\  st5        j6                         s| j8                  j:                  }t5        j6                         rLt=        j>                  t4        j@                  | j                        }	|D 
cg c]  }
|
jC                  |	       }}
|j+                  |      }nH|jE                         }|D ]"  }t5        jF                  | j                  |       $ | j9                  |      }| j                  ;| j                  jI                         r!t-        |      d	kD  r|d	   jK                          d d d        | j                  S g }tM        | j                  jN                        D ]A  \  }}|)||   jP                  jR                  jU                  |       1|jY                  |       C |S c c}}w c c}
w # 1 sw Y   xY w# tV        $ r Y hw xY w)
NzJAX tried to execute function z+, but the no_execution config option is setT)with_tokensi  c                    | S rU   rQ   )r   s    rS   r[   z,ExecuteReplicated.__call__.<locals>.<lambda>Z  s    " rR   F)strictr   )-r   no_executionr   RuntimeErrorr   rj   r  r  in_mutr  r   PGLEProfilerr(  r  r  r  r  r  r  execute_shardedr-   consume_with_handlersrd   ,disassemble_prefix_into_single_device_arraysconsume_tokenr  r   needs_check_specialr  r  	functoolsr   check_special_arraypre_wrap%disassemble_into_single_device_arrayscheck_special
is_runningblock_until_readyr  out_mutr   r   _replace_withAttributeErrorrk   )rL  rq   ru   rV   r  rz   result_token_bufssharded_runtime_tokenr  special_checkr  r   
out_arraysarraysout_os                   rS   r  zExecuteReplicated.__call__H  s     &tyyk 2   $D/D$!QQ$2C2C-CADDDxx&t&dhhoo&d&J				$	$T%7%7	8  #


$"<"<$$//
;
%%55jd5S#s*%;;oD$8$8 99% < I
 &RR$&&')
 ' 5 5 7 13HI%%55jA	!S	(0L0L0N##,,'')#++**DII7-9ABAajj/B(B++H5BBD
  	4F

 
 F
3	4z*



(T-?-?-J-J-L#hlA  "A #D xxjddhh&&, $!Q=Aw}}!!//2 ++a. ka E4 C- #  #R  
%
%s>   MM E#M'#M";B#M'(M3"M''M03	M?>M?rU   )r  r  r  r  r(  rz  r  rz  r)  r
   r  r   r  set[int]r  MutationData | Noner  profiler.PGLEProfiler | None)rN   rO   rP   __doc__r  rK  r  r  r   ro  r  rQ   rR   rS   r  r    s    Q6) >B'*'"3' !2' ?B' $(	' 8@	'
 (' ;'*
0 5 5rR   r  xla_pmapc                   t        | |d         \  }}t        | |d         \  }	}t        ||d         \  }}
|
dg|z  z   }
t        |t        |	      t        |
      t        |            }t        ||d         \  }}dg|z  |z   }t        ||d         \  }}dg|z  |z   }t        ||d         \  }}t        |t        |      t        |      t        |            }||fS )Nr   r   r  r   )r   r  r   F)r@   r  r   )unks_ininst_inkept_outs_knownkept_outs_stagednum_resparams_knownparams_stageddonated_invars_knownr   in_axes_knownout_axes_knownnew_params_knowndonated_invars_stagedin_axes_stagedout_axes_stagednew_params_stageds                   rS   (_pmap_partial_eval_custom_params_updaterr6    s    +7LAQ4RS#G\)-DE-$_l:6NO!^!QC'M1.,m0D#(#8)./C)DF
 ,G]CS5TU! 'G+.CC$WmI.FG!^3=>1.%&6j8QR!_=%2G$)/$:*/0E*FH 
,	,,rR   c                6    t        j                  | d   d|      S )Nr   r   )r   rV  )r,  r   s     rS   #_pmap_partial_eval_custom_res_makerr8    s    			L5q$	??rR   c           	        t        |       s0t        j                  |      sdgt        |j                        z  d fS |j
                  d   }t        j                  ||j
                  d   fg      5  t        j                  |j
                  d   |       \  }}d d d        t        |j
                  d         \  }}t        ||j
                  d         \  }}t        | |j
                  d         \  }}t        |j
                  t        |      t        |      t        |            }	t        |      st        |       s|j                  s|d fS t        j                  |j                  |h      }
t        j                  t        |j                  |      D cg c]
  \  }}|s	| c}}t        |j                   |       D cg c]
  \  }}|s	| c}}|j"                  |	|
|j$                        }||fS # 1 sw Y   `xY wc c}}w c c}}w )	NFr   r   r  r   r   r  )r  r   r   r  )r   r	  has_effectsrd   invarsr   r   r  	dce_jaxprr@   r  r   r   filter_named_axis_effectsnew_jaxpr_eqnr  rF  rW  source_info)used_outputsr  r   	new_jaxprused_inputsr   r   r   r  
new_paramseffsrp  usednew_eqns                 rS   _pmap_dce_rulerG    s   	\	2>>##67S_$d**jj%)CJJ7I,J KLM R\\#**\*BLQI{R$[#**=M2NO!^k3::i+@A*!W|SZZ
-CD+!XCJJ9#(#8!'NU8_F* 
[	#l"3I<M<M)))*;*;i[IDcjj+6?wq$$?ckk<8Awq$DAz4:G !R R 	@As$   0'G5
H
"H

HH5G?c                    | d   }|s|rd|z  }nCt        |      t        |      k(  sJ t        ||      D cg c]
  \  }}|s	| }}}dg|z  |z   }t        | t        |            S c c}}w )Nr   r  Fr   )rd   r  r  r   )r   kept_inputsnum_new_inputsr   r   kepts         rS   $_xla_call_partial_eval_update_paramsrM    s     *+.	.N{s>2222'*>;'GPGAt4aPNPW~->N	fU>%:	;; Qs   
A)A)c                    | d   }t        ||      D cg c]
  \  }}|s	| }}}g ||}t        | |      S c c}}w )Nr   rI  r  r  )r   nz_tangentsr   r   nzdonated_tangentsnew_donated_invarss          rS   xla_call_jvp_update_paramsrT    sS    *+.%(%ELEAraLL;;*:;	f%7	88 Ms   
;;c                v    | d   }g d t        |      D        d t        ||      D        }t        | |      S )Nr   c              3      K   | ]  }d   yw)FNrQ   )r`   r   s     rS   rb   z4_xla_call_linearize_update_params.<locals>.<genexpr>  s     ;e;r  c              3  ,   K   | ]  \  }}|s	|  y wrU   rQ   )r`   r   rQ  s      rS   rb   z4_xla_call_linearize_update_params.<locals>.<genexpr>  s     QEArbaQs   
rI  )r  r  r  )r   rK  rP  donated_invars_prevr   s        rS   !_xla_call_linearize_update_paramsrY    sL    /0S;U>%:; SQS)<k%JQS.	f^	44rR   c                    | d   }t        ||      D cg c]
  \  }}|r	| }}}|D cg c]  }|sd	 }}t        | g ||      S c c}}w c c}w )Nr   FrI  rO  )	r   undef_primalsnonzero_ctsr   r   udonated_primalsrQ  donated_cotangentss	            rS   !_xla_call_transpose_update_paramsr`    se    *+.#&~}#EO41aQQO/O(3:"r::	f%L%L9K%L	MM P:s   
AAAAr  )res_avalc                   t        j                  t        j                  | j                  t        j                  | j                        z  t        j                              }t        j                  t        j                  | j                  d   t        j                              }t        j                  t        j                  t        j                         |      |      S r  )r(   ir_constantr   r   nrepsmathprodsizesuint32r0   	remainderdivide
replica_id)rN  divmods      rS   _unravel_index_hlorn    s    hhx~~8>>!::BIIF	H#(.."4bii@A#	szz#.."2C8#	>>rR   c                   | t         j                  u r|S t        | t         j                        r9t	        j
                  | j                  t        j                        rt        j                  | j                        } t        | j                        }t        j                  t        j                  dt        j                              }|gt!        |      z  }|j#                  |t%        |             |j'                         }|j#                  |d       t)        j*                  ||t        j,                  |            }t)        j.                  t        j0                  |       |      S t3        |       )NrQ   r   rY   )r   r   r   r$   r   r   r   r   physical_element_avalr   r   r(   rc  r   r   rh  rd   r  rn  copyr0   dynamic_slicedense_int_arrayr  aval_to_ir_typer   )	r   rN  rV   r'  dimszeroidxsdims_unsqueezeddynamic_slice_results	            rS   
_hlo_shardrz    s   	T   H$(()V__5''

3d

DBHHRryy9:D6CIDKK+H56iikO7A&,,	4%%o68;;t++D13GHH
D/rR   c                    	 t        fdt        | j                        D              S # t        $ r t	        d       d w xY w)Nc              3  4   K   | ]  \  }}|k(  s|  y wrU   rQ   )r`   ru   r   r   s      rS   rb   z_axis_read.<locals>.<genexpr>  s     OWQTY=NqOs   zunbound axis name: )r  rj   r@  r   	NameError)rN  r   s    `rS   
_axis_readr~    sI    AO	(.. 9OOO	 A
))5
6D@As	   &* Ac                &   t        |t        t        f      s|f}t        t        t	        t
        |       |            }t        | j                  t        j                  | j                              \  }}|rJ | j                  |fz   }t        ||      S rU   )r   r   r   r   r   r~  divmodrd  re  rf  rg  _axis_groups)rN  r   	mesh_axestrailing_sizeragged	mesh_specs         rS   axis_groupsr    sw    	D4-	(7DJwz8<dCD) 8>>1JK-nn//)	i	++rR   c                   t        j                  t        j                  |             j	                  |       }t        j                  t        j
                  ||t        j                  t        |                  t        j                  t        j                  | |            df      }t        t        t        |j                              S )a}  Computes replica group ids for a collective performed over a subset of the mesh.

  Args:
    mesh_spec: A sequence of integers representing the mesh shape.
    mesh_axes: A sequence of integers between 0 and `len(mesh_spec)` (exclusive)
      indicating over which axes the collective is performed.
  Returns:
    A tuple of replica groups (i.e. tuples containing replica ids).
  r  )r   r%  re  rf  r  r  rd   taker   r   T)r  r  r  groupss       rS   r  r    s     
499Y'	(	0	0	;$::kk$	299S^#<=yyI./46& 
z%*	++rR   c                   |t         j                  u r|S t        |t         j                        rt	        |j
                        }|j                  |j                  d   g|z         }t        j                  | d|      }t        j                  t        j                  dt        j                              }t        |      g|gt        |      z  z   }	t!        j"                  |t        j$                  dg            }
t!        j&                  ||
|	      }t        j(                  t+        ||j,                  d               }t!        j.                  ||      }|dk7  rt	        t1        dt        |                  }|j3                  |d       t	        |      }|j3                  ||j                  d          t!        j4                  |t        j$                  |            }|S t7        |      )Nr  r   r   rQ   r   rY   )r   r   r   r$   r   r   r   rg  r(   full_like_avalrc  r   r   rh  rn  rd   r0   r  rs  dynamic_update_slicedense_int_elementsr  r@  cross_replica_sumr  r  	transposer   )ctxr   rN  r0  rV   ru  padded_avalpaddedrv  rw  broadcast_resultreplica_groupsr   permtransposed_dimss                  rS   _hlo_unshardr     s   	T   H$(()

D++X^^B%7$84$?+@K  a5FBHHRryy9:Dx()TFSY,>>D}}Q(<(<aS(AB%%f.>EF,,(HNN2./1N



7C1}%3t9%&d
kk(AT
oXx~~b'9:MM#t33D9:cJ
D/rR   c                |    t        j                  | j                  | j                  |fz   | j                  |fz         S rU   )r   r>  rd  r@  rg  )envr   r   s      rS   _extend_axis_envr  :  s6    					399w+> #		TG 3
5 5rR   r  c                  ~
t        j                  || j                  j                         | j                  j                  j
                  r|t        d      t        | j                  j                  ||      |j                  D cg c]  }|j                   }}fdt        |||      D        }t        j                  ||fg      5  | j                  j                  t        j                              }t        j                   ||| j"                  j%                  t'        j(                  d|            t        j*                         dg|| j,                  | j.                  d\  }}d d d        |j0                  D cg c]  }|j                   }}t        ||	      D cg c]  \  }}}t3        | |||       }}}}|S c c}w # 1 sw Y   bxY wc c}w c c}}}w )Nrg  c              3  J   K   | ]  \  }}}|t        |||      n|  y wrU   )rz  )r`   r   in_noder'  new_envs       rS   rb   z!_pmap_lowering.<locals>.<genexpr>M  s>      D 	gw  tWgw/ '(Ds    #)r  r#  rQ   )dim_var_valuesconst_lowering)r(   check_backend_matchesmodule_contextr  rN  r@  r   r  r;  r   r  r   r  r3  r   rD  jaxpr_subcomp
name_stackextendr!   rA   TokenSetr  r  rF  r  )r  r   r   r   r   r   r  r   r   r  r   r   in_nodesrp  r  in_nodes_shardedsub_ctxsharded_outsr   r  r   r0  shardr   r  s                           @rS   _pmap_loweringr  ?  s    Wc&8&8&B&BC 	  &&7+>
B
CCS//88)-/' )//0aff0(0D #&h'"BD
 ,< =>? N  ((#66w? ) AG((dnnVT:;N /N ))#:L:L	NOL!N  *112!qvv2)2'*9h'M
O 
O#dHe sD'8U
; 
O$ 
O	+# 1N N 3
Os   	GB$GG,G"Gc                    t        |t              sJ t        |j                        }|j	                         D ]%  \  }}||   | |   z  dk(  sJ ||xx   | |   z  cc<   ' |j                  t        |            S )Nr   r   r   r$   r   r   rl   r   r   )
axis_sizesr   r   r   r   r   s         rS   tile_aval_ndr  b  s    	D+	&&	&
tzz
%MMO %jdD;D))Q...	$KJt$$K% 
5<	((rR   c                    t        |t              sJ t        |j                        }|j	                         D ]  \  }}||xx   | |   z  cc<    |j                  t        |            S )Nr   r  )r  r  r   r   r   r   s         rS   untile_aval_ndr  j  sb    	D+	&&	&
tzz
%NN$ $jdD	$K:d##K$	5<	((rR   c           	     n    t        | j                  |t        | j                  j                  ||            S rU   )r  r   r  
local_meshmeshrR  r   s      rS   mesh_local_to_globalr  r  s/    	

D$T__%:%:D$G
I IrR   c           	     n    t        | j                  j                  |t        | j                  ||            S rU   )r  r  r   r  r  s      rS   mesh_global_to_localr  v  s-    	--t$TZZt<
> >rR   full_to_shardc                0    t        |j                  ||       S rU   )r  r   rV   rR  r  r   s       rS   _full_to_shard_abstract_evalr  }  s     
djj$	**rR   c           	        |j                   }t        |j                               }t        |j                        D ci c]  \  }}||
 }}}t        |t              }|j                  D cg c]  }||vr|
 }	}|	D 
cg c]  }
||
   	 c}
|D 
cg c]  }
||
   	 c}
z   }dg| j                  z  }|j                  t        j                  |	D 
cg c]  }
||
   	 c}
             |j                  t        j                  |D 
cg c]  }
||
   	 c}
             t        j                         }t        j                  j                  j                  |_        ||_        ||_        ||_        t        j                  j                  j&                  t        j                  j                  j(                  g|_        |S c c}}w c c}w c c}
w c c}
w c c}
w c c}
w )zoCreate an OpSharding proto that declares all mesh axes from `axes` as manual
  and all others as replicated.
  )keyrY   )r   r   r   rj   r^  r  r   rs  rk   re  rf  r   
OpShardingTypeOTHERrg   tile_assignment_dimensionsiota_reshape_dimsiota_transpose_perm
REPLICATEDMANUALlast_tile_dims)r   manual_axes_setr  named_mesh_shape
mesh_shaperu   r   
axis_ordermanual_axesreplicated_axesr}   tad_perm	tad_shapeprotos                 rS   manual_protor    s    ZZ$++-.*'0'ABGAtaB*BC0+&*oo 5dO3  5/ 5 '66z!}6&12z!}23(cDIIo)499?Ka.q1KLM499;Ga.q1GHI
--/%}}!!''%*%.%"&%&%--,,779K9K9R9RS%	,% C5 72KGs#   F>-G G	GGGc                  | j                   \  }| j                  \  }t        |t        |            j	                  |j
                        j                         }t        t        |j
                              t        |j                               z
  }t        j                  | ||||      }	t        |||      }
t        j                  | |	||
|      fS N)unspecified_dims)avals_in	avals_outr=   r:   _to_xla_hlo_shardingrs  to_protor  r  r   r(   wrap_with_sharding_opr  wrap_with_full_to_shard_op)r  rV   rR  r  r  aval_inaval_outsharding_protor  sxr  s              rS   _full_to_shard_loweringr    s     \\('mm)(D9$?@GLL)((*  w||,-DKKM0BB!!#q'>3CE"
wT
2%

)
)#r8U;KM 
O OrR   shard_to_fullc                0    t        |j                  ||       S rU   )r  r   r  s       rS   _shard_to_full_abstract_evalr    s     


D!	,,rR   c                  | j                   \  }| j                  \  }t        |||      }t        t	        |j
                              t        |j                               z
  }t        j                  | ||||      }	t        |t        |            j                  |j
                        j                         }
t        j                  | |	||
|      fS r  )r  r  r  r  r  rs  r   r(   r  r=   r:   r  r  wrap_with_shard_to_full_op)r  rV   rR  r  r  r  r  r  r  r  r  s              rS   _shard_to_full_loweringr    s     \\('mm)(
wT
2%w||,-DKKM0BB!!#q'53CE" D9$?@HMM*88:  
)
)#r8^*:< 
> >rR   c                6    | D ]  }t        |t              s y y)NTF)r   r8   )rn   r~   s     rS   check_if_any_autor    s#     a!T 
rR   c                 4   t        t        j                  j                  t              r>t        j                  t        j                  j                        j                         d   S t        j                  j                  xs t        j                         d   S r  )r   r   default_devicer   r   r   r!  r  rQ   rR   rS   get_default_devicer    sf    %%++S1>>&//556DDFqII  &&?"*:*:*<Q*??rR   c           	     $   d }|dn
t        |      }d }|rdnd}| D ]_  \  }}}t        |t              rt        |t              r}t        |j                  t
              rc|T|j                  j                  s>|j                  |j                  j                  k7  rt        d| d|j                         |j                  }d}|j                  }||||f}|sE|d   |k7  st        j                  t        j                  | t        j                  |||      g      ||k7  s	t        j                  t        j                  |t        j                  j                  d       t        j                  |||      g       ||r|}	n|t               f}	n|d   }	t!        j"                  |	d         }
|r>|<t%        |	      |j                  k7  r$t        d|j                   dt%        |	             |s||
|	t%        |	      fS |
d |j                  fS )	NrQ   TFzBAbstractMesh should be of the same size across all shardings. Got  and r   zAbstractMesh size: z, does not match the device assignment size: )r   r   r9   r=   r  r3   emptyr   r   r   r   DeviceAssignmentMismatchErrorDeviceAssignmentMismatchMismatchTypeCONTEXT_DEVICESr  r   r  rd   )rn   context_devicesfirst_sharding_infoabstract_meshany_concrete_shardingshs_typer?  arr_device_assignmentrd  r   s              rS    _get_and_check_device_assignmentr    sP    )1Bu_7M/-"1$u!* @b&+"&'	B	&:bgg|+L

#BGGMM



, ++8/rwwiI J 	Jggm" 33		$4fkJq!%::44--/BC--'>6? @ @
 3344--!6#6#6#F#FN--'>6? @ @-@8  _'"+-/+A.!!"3A"67' 9	
 2 22

m001 2$$'(9$:#;	=> > m3%s+<'===D-,,,,rR   c                
   dgt        | j                        z  }t        j                  | |      \  }}}t	        |      D ch c]
  \  }}|s	| }}}t	        |      D ch c]
  \  }}|s	| }}}|||fS c c}}w c c}}w NT)rd   rF  r	  dce_jaxpr_constsrj   )	r  r@  rA  used_constsrB  ru   bkept_const_idxr  s	            rS   prune_unused_inputsr    s     #emm,,,(*(;(;E<(P%)[+"+K"8>$!QAA>.> )+ 6<1!!<,<	NL	00 ?<s   
A9A9"
A?-A?c                  
 t        | t        j                        sJ | j                  }| j                  }| j
                  }|s|st        d |D              rt        t        t        |                  
nOt        |      \  }}
t        |      D 	cg c]  \  }}	||v s|	 }}}	t        
fdt        |      D              }~t        j                  ||      } | |
fS c c}	}w )Nc              3  x   K   | ]2  }t        |d       xr  t        j                  |j                          4 yw)r   N)hasattrr   is_constant_shaper   r`   r}   s     rS   rb   z_dce_jaxpr.<locals>.<genexpr>)  s9      
 !W

Ed&<&<QWW&E"E
E 
s   8:c              3  2   K   | ]  \  }}|v s|  y wrU   rQ   )r`   ru   rV   r  s      rS   rb   z_dce_jaxpr.<locals>.<genexpr>/  s     XAa<FW1X   )r   r   r  r  r  r  r   r  r  rd   r  rj   r   )r  keep_unusedr   auto_spmd_loweringr  r  r  r  ru   cr  s             @rS   
_dce_jaxprr  !  s    	L$"2"2	33	3


%&""('	 

 uS]+,L*=e*D'E><%f-EDAqn1DaEFEX>)BXXN!!%0,	~|	33 Fs   C  C c                  "    e Zd ZU ded<   ded<   y)MutationDatazlist[core.Ref]r	  zlist[int | None]r  Nr  rQ   rR   rS   r  r  5  s    
 rR   r  c           
     t   ddl m} t        |       \  } } ||       }t        j                  t        | j                              }t        | j                        D ci c]!  \  }}t        |t              r|t        |      # }}}|j                         D ci c]  \  }}||
 }	}}t        t        |j                  t!        t        |j                                          }
t#        t        |	j                  t!        t        |j                                          }||
t%        ||      fS c c}}w c c}}w )Nr   )discharge_state2)jax._src.state.discharger  _move_mutable_constsitcountrd   r  rj   r  r   rI   nextrl   r   r   r$  r  r   r  )r  r  r	  rA  r  ru   r}   	inout_mapr6  	outin_mapinout_aliasesr  s               rS   _discharge_refsr  ;  s     8&u--%u%)
((3u'
(%*3ENN*C .$!QQ, $u+~ .) . ) 121q!t2)2IMM5Y5G5G1H+IJK-Y]]E#i.A.A*B$CDE'	M<#@	@@.2s   &D.D4c           	     4   | j                   }| j                  D cg c]  }t        |t        j                         }}t        || j                        \  }}t        ||j                        \  }}g |j                  |}t        j                  |||j                  |j                        }	t        j                  |||j                  |j                  |	| j                   j                  j                               }t        j                  ||      |fS c c}w rU   )r  r  r   r   Refr@   	constvarsr;  r	  make_jaxpr_effectsrF  r  Jaxpr
debug_infor  r  )
r  r  r  hoistr  r	  r  mutvarsr;  r   s
             rS   r  r  J  s     

%,8,?,?
@q:a"
@%
@!%)<)<=.&&%eU__=)W$U\\$G$&!!)VU]]EJJO'
**Yuzzl00;;NNPR%			%	(&	00 As   !Dc                    ddl m}  || j                  | j                        \  }}| j                  j                  |_        t        j                  ||      S )Nr   )discharge_state)r  r$  r  r  r.  r   r  )r  r$  jaxpr_r  s       rS   _discharge_internal_refsr&  Y  sB    6"5;;=.&&{{..&			&&	))rR   c                  $    e Zd Z	 	 ddZd Zd Zy)SemanticallyEqualShardingsc           	        t        ||      D cg c]]  \  }}t        |t        t        f      s*t        |t              rt        |j
                  t              r|nt        ||j                        _ }}}|| _	        || _
        y c c}}w rU   )r  r   r9   r8   r=   r  r3   to_gspmd_shardingrs  _gspmd_shardingsrn   )rL  rn   r  r~   r}   gspmd_shardingss         rS   rK  z#SemanticallyEqualShardings.__init__c  s     	5)	+ Aq -t45!]+
166<0P 	
q!&&)	*+O +
 ,DDN+s   A"Bc                L    t        t        d | j                  D                    S )Nc              3  p   K   | ].  }t        |t              r|j                  |j                  fn| 0 y wrU   )r   r<   _hlo_sharding_hashmemory_kindr`   r~   s     rS   rb   z6SemanticallyEqualShardings.__hash__.<locals>.<genexpr>n  s:      O34:a' 
		q}}--.	/O   46)hashr   r+  r  s    rS   __hash__z#SemanticallyEqualShardings.__hash__m  s.     O8<8M8MO O P PrR   c                    t        |t              syt        d t        | j                  |j                        D              S )NFc              3     K   | ]q  \  }}t        |t              rUt        |t              rEt        j                  |j                  |j                        xr |j
                  |j
                  k(  n||k(   s y wrU   )r   r<   r   are_hlo_shardings_equal_hlo_shardingr0  )r`   r~   r   s      rS   rb   z4SemanticallyEqualShardings.__eq__.<locals>.<genexpr>u  si      
 Aq q-(Z=-I 
	-	-aooq	O 
,]]amm+!V	s   A7A9)r   r(  rm   r  r+  )rL  others     rS   __eq__z!SemanticallyEqualShardings.__eq__r  s?    e78 
 --u/E/EF  rR   N)rn   z,tuple[GSPMDSharding | UnspecifiedValue, ...]r  r  )rN   rO   rP   rK  r4  r:  rQ   rR   rS   r(  r(  a  s    2P
	rR   r(  c           
     >   | dkD  rt        j                  d| d       | t        j                  |      kD  r(t	        d| d|  dt        j                  |       d      t        j
                  |      dkD  r'| dkD  st        j                  |d      rt        d	      y y )
NrY   zThe function a{   includes a pmap. Using jit-of-pmap can lead to inefficient data movement, as the outer jit does not preserve sharded data representations and instead collects input and output arrays onto a single device. Consider removing the outer jit unless you know what you're doing. See https://github.com/jax-ml/jax/issues/2926. Or use jax.shard_map instead of pmap under jit compilation.zcompiling computation `z` that requires z replicas, but only z XLA devices are available.r%  zjit of multi-host pmap not implemented (and jit-of-pmap can cause extra data movement anyway, so maybe you don't want it after all).)	warningswarnr   r  r   r-  r   jaxpr_has_primitiver  )rd  r   r   r  s       rS   )_raise_warnings_or_errors_for_jit_of_pmapr?  ~  s    
QYMM
v D 	DE R__W%%

!$'7w ?())D	FG G g"ai8//zB
	MN N C #rR   c                r   | j                   }| j                  }|j                  }|j                  }t        j                  j
                  rt        j                  nt        j                  }t        j                  |      rt        j                  |d|||       t        |      }t        ||||       |dk(  rKt        t        ||      }t        t        ||      }dgt!        |      z  }t#        j$                  |	|
|      }|	}n4d }d }d }t#        j&                  |dd      }t#        j(                  |      }d}|	dkD  rdt*        j,                  j/                  | j*                        }t*        j0                  j3                  |      }t!        |      dkD  rt5        d|       t7        t*        j,                  j/                  | j*                              }d|z  |j8                  j;                  t!        |      |z
        z   } t=        j>                  d|t<        j@                  	      5  tC        jD                  || fi d
|d|d|d|d|d|d|d|d|d|d|d|d| d|j8                  jG                  t!        |            d|d|d|d|d|d|}!d d d        t=        jH                  t!        |      |jJ                        }"t7        t*        j,                  j3                  | j*                              }#!jL                  |!jN                  |!jP                  |#|||"|!jR                  fS # 1 sw Y   xY w)NzCCompiling %s with global shapes and types %s. Argument mapping: %s.rY   FrQ   r   zHThe following ordered effects are not supported for more than 1 device: r  r  r  r  r  r   r  r  r  r   r!  r"  r#  r  out_layoutsr$  r%  r&  r  all_default_mem_kindinput_output_aliasespropagated_out_mem_kindsr  )*r  r  rn   r   r:  r   r6  r;  r7  r4  r5  r<  r  r?  r   _to_logical_shardingrd   r   ShardingContextr>  rD  r   r  r@  shardable_ordered_effectsrA  r   r   r.  r/  r   r  rB  r(   rC  rE  r?  r"  rG  r)  r*  r,  )$r  rO  r   r  r  semantic_in_shardingssemantic_out_shardingsr  rA  num_devicesrd  r   rB  r  rD  r  r  r  r  r  r  r  rM  rd  in_mlir_shardingsout_mlir_shardingsr!  axis_ctxr  rN  unsupported_effectsr  r$  rP  r'  r(  s$                                       rS   _cached_lowering_to_hlorO    sz    $$)


%&00,(22-$*$7$7$=$=7==,&
JJ|'Hl4 %
 %+E7KO aZ0(LI19mLgH-O--k;L.;=H N O%%eR4H00:HN 1_!11;;L<P<PQ!;;II
!#	23	56 6 00::<;O;OPQ/n$u'8'8'G'GHXfHf'gg)  WH$G$GI 1 001 &1 (	1
 1 1 1 1 $1 (1 (1 ,1 1  1 1  &&88YH!1" #1$ &%1& 2'1( +)1* ":+1, 0-1O14 ))#h-9I9IJ*++L,@,@AC

 
 /";";

(
(*;_
_==
? ?;1 1s   $A4L--L6c                ,    t        j                  |       S rU   )r   
DeviceListrc  s    rS   _create_device_list_cachedrR    s     
(	))rR   c                T    | t        | t        j                        r| S t        |       S rU   )r   r   rQ  rR  rc  s    rS   r  r    s*     *->"N	#$5	66rR   c                   g }| j                   D ]D  }|j                  t        j                  u s |j	                  d |j
                  d   D               F t        j                  |       D ]  }|j	                  t        |              |S )Nc              3  T   K   | ]   }t        |t        j                        r| " y wrU   )r   r   MemorySpacer
  s     rS   rb   z+jaxpr_transfer_mem_kinds.<locals>.<genexpr>  s'      5qq$"2"23  5s   &(r   )	r  rW  r   device_put_pr  r   r   	subjaxprsjaxpr_transfer_mem_kinds)r  r   r  subjaxprs       rS   rY  rY    s~    
#ZZ 5c
}}---	jj 5CJJy1 5 55 ..' 3hJJ'123	*rR   c                    | D ]^  }t        |t        t        f      rt        |t        j                        rt        j
                  |      n|j                  }|X|dk7  s^ y y)Nr  FT)r   r9   r8   r   rV  mem_space_to_kindr0  )rn   ru   mem_kinds      rS   "are_all_shardings_default_mem_kindr^    sf     a!&-.-74;K;K-L&&q)]] 8 
rR   c                   i | j                   }fd}fd}t        ||j                  d gt        |j                        z         t        ||j                  d gt        |j                        z         |j
                  D ]_  }|j                  t        j                  u r|j                  d   g}nd gt        |j                        z  }t        ||j                  |       a t        t        ||j                              S )Nc                F    t        |       t        j                  u ry |    S rU   )rg   r   Literal)varr  s    rS   readz-get_out_layouts_via_propagation.<locals>.read%  s    CyDLL s8OrR   c                    || <   y rU   rQ   )rb  r  r  s     rS   writez.get_out_layouts_via_propagation.<locals>.write*  s    CHrR   rw   )r  r>   r;  rd   r  r  rW  r   sharding_constraint_pr   rF  r   )r  r  rc  re  r  out_eqn_layoutsr  s         @rS   get_out_layouts_via_propagationrh    s     	#


%
 5%,,U\\): :;
5%//D6C,@#@AZZ 2c }}222H-.oS[[!11oUCKK12 
xemm,	--rR   c                  &    e Zd ZU dZded<   ded<   y)AllArgsInfoz4Avals and debug_info for all arguments prior to DCE.r  r  r|  r   N)rN   rO   rP   r$  r  rQ   rR   rS   rj  rj  >  s    <&&rR   rj  c                    t        | t              r| S t        | j                  | j                  |      | j                        S Nr0  )r   r<   _internal_device_listr  r0  )r~   rs  s     rS   r*  r*  D  s:    =!H	q..0F0Ft0L#$==
2 2rR   c                (   t        d | j                  D              st        d | j                  D              rt        |       \  } }}g d |j                  D        g d |j                  D        g |dt        |j                        z  z   }t        t        ||            t        fd|j                  D              \  }}t        d       -J d x}}t        d | j                  D              rt        |       } | |||||fS )Nc              3  <   K   | ]  }t        |t                y wrU   )r   rJ   r`   r   s     rS   rb   z(_discharge_refs_jaxpr.<locals>.<genexpr>N  s     	Aq*Q	
"	A   c              3  <   K   | ]  }t        |t                y wrU   )r   rI   r  s     rS   rb   z(_discharge_refs_jaxpr.<locals>.<genexpr>O  s     	D*Q
$	Drr  c              3  p   K   | ].  }t        j                  |j                  |j                         0 y wrU   )r   finalize_arg_shardingrv   r   r`   r  s     rS   rb   z(_discharge_refs_jaxpr.<locals>.<genexpr>Q  s,      %Q@A""1::q{{;%Qr2  c              3  d   K   | ](  }t        |d       r|j                  j                  nd * yw)r  N)r  r  rw   rv  s     rS   rb   z(_discharge_refs_jaxpr.<locals>.<genexpr>S  s3      !@/0 5<Ax4H&*"+ !@s   .0r  c              3  L   K   | ]  }|t              n	|   |   f  y wrU   )r  )r`   ru   r  r  out_layouts_s     rS   rb   z(_discharge_refs_jaxpr.<locals>.<genexpr>W  s4      (  i\l1oz!}-MM(s   !$c              3  P   K   | ]  }t        |t        j                           y wrU   )r   r   InternalMutableArrayEffectrq  s     rS   rb   z(_discharge_refs_jaxpr.<locals>.<genexpr>]  s     
Xa:a889
X   $&)r   r   r  r  r	  rd   iterr  rE   r  r  r&  )	r  r  r  r   r  rA  r  r  ry  s	    ``     @rS   _discharge_refs_jaxprr~  L  s4   		AL,@,@	AA		Dl.C.C	DD'6|'D$L-R\ R %QEHZZ%Q RLA: A !@47JJ!@ AJ'~'(S_*DDNM;78L!' (( "M; d#+++MC

X<CWCW
XX-l;l
sL*
-
6 6rR   c	           	        t        t        j                  | j                              \  }	}
t	        |	      }|r3t        |
      |z   }t        j                  |	      }||z   }t        j                  |	|
|      }||z   }d|z  |z   }t        t        |            j                  |D ch c]  }||z   	 c}      }|d|z  |z   }|6t        |j                  |j                  D cg c]  }|d n||z    c}      }|j                  j                   d }nd|z  |j                  j                   z   }t#        t        |
      |j$                  z   |j                  j'                  |            }|	||||||||f	S c c}w c c}w )Nr  rU   )r	  r  r  )r$  )rE   r   r1  r  rd   r   r   const_args_shardingsconst_args_layoutsr  r  unionr  r	  r  r   r$  rj  r  _replace)r  global_in_avalsr  r  r   r  r  r  all_args_infor   rI  r  ca_shardings
ca_layoutskvi_idxr$  s                    rS   r0  r0  d  s    !'
L../!*o z?.?+o=O,,Z8L,.L((_*68Jj(J.?Nu^,-33'34n	46L .>m
 #- !=4en.DD -.c ))1i>)M,D,D,N,NNi_ 6 66  ))I)>@M o|Z
sM
; ;! 	5-s   'E8$E=
i   c                     t        j                  fdt        g      t        j                  t	                          }t        |j                  | j                        | j                  | j                        S )Nc                    |    S rU   rQ   )ru   rd  s    rS   r[   z,_abstract_to_concrete_mesh.<locals>.<lambda>  s    "3A"6 rR   otypes)
axis_types)
r   	vectorizeobjectr%  rd   r4   r  r  r^  r  )r  rd  np_devs    ` rS   _abstract_to_concrete_meshr    sf    )2<<6 &x))+37H3I)JL&	fnn]556&&=3K3K
M MrR   c           	        || S g }t        | ||      D ]W  \  }}}t        |t              r.t        |t        j                        r|j
                  j                  j                  r|j                  |       f|j
                  j                  j                  r|j                  |       |j
                  j                  j                  r<t        |j
                  j                  D cg c]  }|t        j                  n| c} n|j
                  j                  }	|j                  t        t        |j
                  j                  |      |	|             G|j                  |       Z t!        |      S c c}w rl  )r  r   r9   r   r$   rv   r  r  rk   _are_all_axes_auto_or_manual_any_axis_autor1   r  UNCONSTRAINEDr=   r  r   )
rn   r  rd  out_mem_kindsr   r~   r}   r]  spr  s
             rS   "_concretize_abstract_out_shardingsr    s*   
#Ium< naH!%&:a9I9I+J	
		

1::??77

1 ::??11 *+**// ;$& @Bz ; ;r Q  ; <78zz 	 	

=&qzz8IJh( 	) 
jjm 
s ;s   )E:
c                V   t               j                  r| S t               }|j                  s| j                  r| S || j                  k(  r| S | j                  |j                  k(  sJ t        | j                  j                  |j                        |j                  |j                        S rU   )r6   r  r5   r  r   r4   r   r  r  r^  r  )context_meshcur_meshs     rS   _get_context_meshr    s     (^^|))+++			hmm	++	+	l""**8+>+>?!!8#6#6
8 8rR   MeshComputationc               z  78 t        t        j                  j                  ||g            }t	        | j
                  | j                  j                        }t        | |||      \  } }7t        7fdt        |      D              }t        7fdt        |      D              }t        | |||||      \  } }}}}}}}| j                  }| j
                  }| j                  }|j                  rt        | ||||7|||	      \	  }}}}}7}}}ng }t        |       }t        d t        ||      D              }t!        |      t!        |      cxk(  rt!        |      k(  s'n J t!        |      t!        |      t!        |      f       t#        |	      }	|	j$                  rdn|	j&                  }t)        j*                  t-        j.                  |            }t)        j*                  |dt!        |             }t)        j*                  |t!        |      d       }t)        j*                  |      }t1        t        j                  d |D        d |D        d |D        d |D              |      \  }}}|D  cg c]  \  }} |	 }}} ||z  }~t-        j2                  |      }!||t5        d	      |!rt5        d
      |At!        |      dkD  r3t7        j8                  |      8t        8fdt;        ||      D              }|D ]  }"|"t<        j>                  us|"j@                  jB                  j$                  r7|"j@                  jB                  jD                  sX|[t!        |      |"j@                  jB                  jF                  k7  st5        dt!        |       d|"j@                  jB                  jF                   d|"jI                  dd       d       |xs tK        |d|jL                        f}#tO        |      }$tQ        |      }%tS        |xs2 |dkD  xs+ tU        d t        j                  |||      D              xs |%      }&tW        t        j                  ||||%            }'|'rdt!        |      z  }(nt        d | j                  D              }(tY        ||||(      }t;        ||      D ")cg c]7  \  }"})t[        |"t<        j\                        rt=        j^                  |"|)      n|"9 }}"})t;        ||      D ")cg c]7  \  }"})t[        |"t<        j\                        rt=        j^                  |"|)      n|"9 }}"})d}*|!r|$J t        j                  |||      D ]j  }+t[        |+t`              s|*?|*|+jB                  jb                  k7  r&t5        d|* d|+jB                  jb                   d      |+jB                  jb                  }*l te        ||      },te        ||      }-tg        jh                  || j                         t)        jj                  ||      }.tm        | |.|t!        |      t        |      |,|-||||!rt        |$      nd||'||(|#||*      \  }/}0}1}2}3}4}5}6to        |.|/|||#|
|$fi d|d|d|d|ddd|5d |d!|2d"|3d#|1d$|0d%7d&|d'|d(|d)|&d*|d+|d,|4d-|6d.|d/|d0|d1|	S c c} }w c c})}"w c c})}"w )2aI  Lowers a computation to XLA. It can take arbitrary shardings as input.

  The caller of this code can pass in a singleton UNSPECIFIED because the
  number of out_avals might not be known at that time and
  lower_sharding_computation calculates the number of out_avals so it can apply
  the singleton UNSPECIFIED to all out_avals.c              3  2   K   | ]  \  }}|v s|  y wrU   rQ   )r`   ru   r~   r  s      rS   rb   z-lower_sharding_computation.<locals>.<genexpr>  s     RTQ\@QqRr	  c              3  2   K   | ]  \  }}|v s|  y wrU   rQ   )r`   ru   r   r  s      rS   rb   z-lower_sharding_computation.<locals>.<genexpr>  s     N41aA<MQNr	  c              3  J   K   | ]  \  }}|t         j                  n|  y wrU   )r)   r8   )r`   r   ps      rS   rb   z-lower_sharding_computation.<locals>.<genexpr>  s+      O!Q &']fkk9 Os   !#Nc              3  V   K   | ]!  }|t         j                  j                  d f # y wrU   )r   r  ARG_SHARDINGr`   ru   s     rS   rb   z-lower_sharding_computation.<locals>.<genexpr>	  s#     
T1Av""//6
T   ')c              3  V   K   | ]!  }|t         j                  j                  d f # y wrU   )r   r  CONST_SHARDINGrv  s     rS   rb   z-lower_sharding_computation.<locals>.<genexpr>	  s#     
YQAv""1148
Yr  c              3  V   K   | ]!  }|t         j                  j                  d f # y wrU   )r   r  OUT_SHARDINGr`   r   s     rS   rb   z-lower_sharding_computation.<locals>.<genexpr>	  s#     
U1Av""//6
Ur  c              3  \   K   | ]$  \  }}|t         j                  j                  |f & y wrU   )r   r  SHARDING_INSIDE_COMPUTATION)r`   jsr?  s      rS   rb   z-lower_sharding_computation.<locals>.<genexpr>	  s1      Ar; ##??M As   *,zPassing lowering_platforms via jax.export or  jit(f).trace(*args).lower(lowering_platforms=...) is required when only AbstractMesh exists in a jitted computation.znAbstractMesh cannot be used when jaxpr contains primitives that require devices to be present during lowering.rY   c              3     K   | ]<  \  }}t        |t              r#|t        j                  ur|j                  d k(  rn| > yw)r   N)r   r9   r   r   rs  )r`   r~   r   rep_gss      rS   rb   z-lower_sharding_computation.<locals>.<genexpr>	  sN      B 1d a!12t222tyyA~ 		Bs   AAzLength of device assignment z& is not equal to the size of the mesh z	 of aval TzA. Please enter your `jit` into a mesh context via `jax.set_mesh`._raw_platformc              3  >   K   | ]  }t        |t                 y wrU   r   r9   r1  s     rS   rb   z-lower_sharding_computation.<locals>.<genexpr>7	  s      *QA/00 *   rU   c              3  Z   K   | ]#  }t        j                  |j                         % y wrU   )r   r\  memory_spacer  s     rS   rb   z-lower_sharding_computation.<locals>.<genexpr>D	  s&      %P34q~~.%Ps   )+zSmesh should be the same across the entire program. Got mesh shape for one sharding r  z for another)r  r  r  r  r  r  spmd_loweringr'  r  r(  r  r*  r)  r  r  r   rJ  r   r  rA  
pmap_nrepsr,  r  r  intermediate_shardingsr  )8r  r  chainfrom_iterablerj  r  r  r.  r  r   rj   r~  r  r0  rh  r?   rd   r  r  _flat_devices_tupler!   stable_uniquer   get_intermediate_shardingsr   jaxpr_has_prim_requiring_devicesr   r<   get_replicatedr  r   r   rv   r  are_all_axes_explicitr   	str_shortr  r"  r  rY  r   r   r^  r  r   r$   update_aval_with_shardingr=   r  r(  r   maybe_dump_jaxpr_to_filerA   rO  r  )9r  api_namer  r  r  r  rA  r   r
  r  compiler_options_kvsr  r  r  r  r  r  r  r  r  r  r   out_layouts_via_propdevices_from_contextunique_intermediate_shardingsunique_const_shardingsunique_in_shardingsunique_out_shardingsr   rd  rJ  r  r   prim_requires_devicesr}   r  device_listtransfer_mem_kind_in_jaxprr   rB  rD  r  r  rv   rH  rI  rO  rG  r)  r*  r(  r  rd  r'  r,  r  r  s9                                                          @@rS   lower_sharding_computationr    s   0 )hhlM:;= l33\5G5G5S5ST-/9K1C0E,,RY|%<RR,N9Z#8NN* 2G\:~}2/<\:=+ 

% ))/!++008O	j	}c=9J6Z,
N=#} J
 9F O"*;8L"MO O+ 
]	s;/	H37G3H	H C	-#k*C0@,AKC C	H #<0,"."4"4$&::  #'"4"4))%0#2--l;KC
O.LM**<J8H+IJ++M:,Lhh
T@S
T
YBX
Y
U@T
UA"?A	B -)'k 4Q"Q%"a2"Q"Q+.DD"CCEJ!?@ @ <= =
 "s+<'='A))*;<F B ",@B BL
  	*a	$$$QZZ__-B-B	

--%!**//"6"66(->)?(@ A&&'jjoo&:&:%; <kk$%& '))* *	* ! <g(8(89%<) $$56+7> $	q$	 *"((
3
';) * 
*$
 
$) <hh"$8,.HJK &-=)>>$ %P8D8N8N%P  P 5%'8 - #&o|"DFB #1d&6&67 33Ar:=>? F/ F
 $''7#GI!R $At'7'78 44Q;>?@ I I -"""HH02F:< 	4	Hm	,%X]]888))6 8----.l<= = !33	4 5O%5%' %%h0B0BCx2+ +B['
:o.4;2u[+]."	+$(69n&7*&  

! &
! (
!  
! "
! 
! 
! ,
! *
!  &!
!" $#
!$ %
!&  '
!( )
!* +
!, -
!. /
!0 1
!2 3
!4 5
!6 (7
!8 "9
!: ";
!< ;=
!>  ?
! !g #RFFIs   +Z+'<Z15<Z7c                    t        |t              ry t        |t              r|S t        | t        t        f      rt        |t
              sJ |S t        | t        j                        ry t        |       rU   )	r   r9   r8   r$   rI   	JShardingr   AbstractTokenr   )r   rv   s     rS   rE  rE  	  sb     *+$O$k23h	***O$**+
D/rR   c                  `    e Zd ZU ded<   ded<   	 	 	 	 	 	 	 	 	 	 ddZddZddd	 dd	Zdd
Zy)r  rZ  r[  zMeshExecutable | Noner\  c                    || _         || _        || _        || _        || _        || _        t        |      | _        || _        d | _	        y rU   )
_namer[  r   _donated_invars
_platforms_compiler_options_kvsr  _device_listr^  r\  )	rL  r   r0   r   r   r  r  rd  r^  s	            rS   rK  zMeshComputation.__init__	  sN     DJDI DO)DDO!5D+,=>D$DDrR   c                    | j                   S rU   r`  r  s    rS   ra  zMeshComputation.stablehlo	  rb  rR   Nrc  c                  |dnt        |j                               }| j                  |z   }t        |      }|| j                  }n| j                  | j                  |k7  rt        d|D cg c]  }|j                   c} d|d   j                  j                          d| j                  D cg c]  }|j                   c} d| j                  d   j                  j                                |}t        |t        d       t        j                  f      sJ | j                  |s|rEt        j                  | j                   | j"                  fi | j$                  ||d}|s|| _        |S | j                  S c c}w c c}w )NrQ   zdevice_assignment passed to `.compile` must match the device_assignment calculated from array shardings and out_shardings. Got device ids passed to compile z on platform r   zC and devices ids calculated from array shardings and out_shardings )r  r  )r   rl   r  r  r  r   r=  r"  upperr   rg   r   rQ  r\  UnloadedMeshExecutablerh  r  r[  r^  )	rL  rf  rd  t_compiler_optionsr  r  compilation_device_listr   ri  s	            rS   r  zMeshComputation.compile	  s    0 8" 0 6 6 89 558JJ%&78K $ 1 1



'


{
*  ++!+, -A''--/0 1#001!12 3!!!$--335689 	9 !,-T
BMM/JKKK#7;L)22
**dii/#'#4#4/3-/j "%! , 2s   ,E=
/Fc                    | j                   d   }t        j                  |      rt        d|j                   d      t        j                  || j                         j                               S )Nr   z3Lowered.cost_analysis not implemented on platform 'zE'. Use compile().cost_analysis() for post-compilation cost estimates.)	r^  r   using_pjrt_c_apir  r"  r,   hlo_module_cost_analysisr0   as_hlo_module)rL  r   s     rS   cost_analysiszMeshComputation.cost_analysis	  sl    	*G	7#  --. . (($((*2J2J2LMMrR   )r   r   r0   rZ  r   rj  r   Sequence[bool]r  r  r  tuple[tuple[str, Any], ...]rd  z,xc.DeviceList | tuple[xc.Device, ...] | Nonerk  rU   rl  MeshExecutable)rl  zdict[str, float])rN   rO   rP   r  rK  ra  r  r  rQ   rR   rS   r  r  	  s`    /$$*-:G &A #O	$   ! DNrR   c                b    g }| j                         }||}g }| j                         }||}||fS rU   )get_parameter_shardingsget_output_shardings)ri  in_op_shardingsparameter_shardings_from_xlaout_op_shardingsoutput_shardings_from_xlas        rS   get_op_sharding_from_executabler  	  sN    )+/!+!C!C!E!-2O*,(==?*0	*	**rR   c                   t        |       \  }}g }|D ]'  }|j                  t        j                  ||             ) g }|D ]'  }|j                  t        j                  ||             ) t	        |      t	        |      fS rU   )r  r  r   parse_flatten_op_shardingr   )ri  r  
input_op_soutput_op_sin_pspecr~   	out_pspecs          rS   get_pspec_from_executabler  
  s     <JG*k"$( GaOON<<QEFG $&) Ha^==aFGH	x%	*	**rR   c           	        t        |t        j                        sJ 	 | j                         d   }|dkD  r||d  }t	        |      |k(  sJ t	        |      |f       t	        |      dk(  r/|D cg c]#  }t
        j                  j                  ||      % c}S t        |       \  }}|sy |dkD  r||d  }|dk(  rt	        |      dk(  ry t	        |      dk(  rt	        |      |k  r||z  }t	        |      |cxk(  rt	        |      k(  sn J t	        |      |t	        |      f       t        ||      D cg c]  \  }}t        j                  |||       c}}S #  d g|z  }Y (xY wc c}w c c}}w )Nr   rY   rm  )
r   r   rQ  get_output_memory_kindsrd   r   r<   r  r  r?   )	r  r  num_out_avalsnum_ordered_effectsomkmkr   r  oss	            rS   !get_out_shardings_from_executabler  
  s    
K	//	/!

0
0
21
5CQ#$%c 
S]	"=SX}$==	" 	 ((77QS7T   8G!	1'(;(<= aC 01Q6 		aC(8$9M$I'-7		-	;3s8	; 6	
]CH>6 6	; !!137
9b" 
&
&{BB
G 
9 9C!&=
 C2
9s   E -(E&2"E+E#c                <   t        |t        j                        sJ t        |      dk(  rt	        j
                  |      g|z  S t        |       \  }}|sy|dkD  r||d }t        |      |k(  sJ t        |      |f       |D cg c]  }t	        ||       c}S c c}w )z!Returns input shardings from XLA.rY   Nr   )r   r   rQ  rd   r<   r  r  )r  r  num_in_avalsr  r  r   r   s          rS   _get_in_shardings_from_xlar  @
  s     
K	//	/((56EE6~F/1	1%&9&:;O	_		- *	/L0* *	- 4C	CR-R
(	CC	Cs   Bc                    t        | |      \  }}|D cg c]  }t        ||       c}|D cg c]  }t        ||       c}fS c c}w c c}w rU   )r  r=   )r  r  r  r  ru   r   s         rS   )_get_mesh_pspec_shardings_from_executabler  Z
  sO     2.$G(I+3
4a=q!
4+4
5a=q!
5
7 7
4
5s
   AA	c                   t        | t              sJ t        |t              sJ t        |j                  t              sJ |k|j
                  j                  j                  sK|j
                  j                  j                  s+t        |j
                  j                  | j                        }n|j                  }t        j                  | |      S rU   )r   r<   r=   r  r4   rv   r  _any_axis_manualr  r   r   !_gspmd_to_named_sharding_via_mesh)out_sout_aval	orig_in_sr  s       rS   _gspmd_to_named_shardingr  d
  s    	E=	))	)	I}	--	-	INND	))	)8#4#4#9#9#?#?



 
 
1
1% 8 8:D >>D		9	9%	FFrR   c                    t        | t              sJ t        |t              sJ t        | j                  d   | j                        S )Nr   rm  )r   r<   r;   r   r0  )r
  r  r  s      rS    _gspmd_to_single_device_shardingr  r
  sJ     
E=	))	)	I3	44	4	q!u/@/@
B BrR   c                F   g }t         t        |         }t        | |      D ]  \  }}t        |t        j
                        r|t        j                  ur|||j                  |j                  k(  rkt        |t              r[|j                  j                  |j                  j                  k(  r.|j                  ||j                        r|j                  |       	 |j                   ||||             |j                  |        |S #  |j                  |       Y xY wrU   )_orig_out_sharding_handlersrg   r?   r   r   r<   r   r   rs  r=   rv   r  r  is_equivalent_tork   )r  r  r  	orig_avalr   orig_handlerr   r  s           rS   $_get_out_sharding_from_orig_shardingr  |
  s    
#,T)_=,mY7 ka1n223+++ 
H$8mmy~~-M2$$	(D(DD  INN;

9	
**\!Xy9
: 
jjm#$ 
*		
**Q-s   DD c                   t        d |D              r|S t        | |      D ])  \  }}|	t        |      t        v st	        ||||      c S  |8|D ]3  }|t        |      t        v st	        |d gt        |      z  |d       c S  |D ]&  }	|	t        |	      t        v st	        |||	d       c S  |B|j                  s6|D 
cg c]*  }
t        |
t              rt        j                  |
|      n|
, c}
S |S c c}
w )Nc              3  R   K   | ]  }t        |t        j                          ! y wrU   )r   r   r<   r  s     rS   rb   z/maybe_recover_user_shardings.<locals>.<genexpr>
  s      PQZ>778	8Ps   %')rm   r?   rg   r  r  rd   r  r   r<   r   r	  )old_shardingsnew_shardings	old_avals	new_avalsr  r  oio_avalru   nsr  s              rS   maybe_recover_user_shardingsr  
  s:    	P-PP]I6 0jb&	~$r(&AA1
B0 00
 '# A	
47&AA3D6C$664A 	AA  .b	~$r(&AA1
B. ..
 l&8&8 #$ !]+ <<QM123 $ $ 
	$s   /C8c                v    t        | t              r%| j                  s| j                  |j                  k(  S | |k(  S rU   )r   r)   tilingmajor_to_minor)ulxls     rS   r   r   
  s2    FBII 1 1118OrR   c                   	 | j                         }| j                         }|dkD  r
||d  }||d  }g }t        ||      D ]Y  \  }}t	        j
                  |      }t        |t              rt        ||      st        d| d| d      |j                  |       [ g }	t        ||      D ]Y  \  }}t	        j
                  |      }t        |t              rt        ||      st        d| d| d      |	j                  |       [ t        d |D              sJ t        d |	D              sJ ||	fS #  dt        |      z  dt        |      z  fcY S xY w)	NrU   r   z&Unexpected XLA layout override: (XLA)  != z (User input layout)z (User output layout)c              3  <   K   | ]  }t        |t                y wrU   r   r)   r  s     rS   rb   z/_get_layouts_from_executable.<locals>.<genexpr>
  s     ;qZ6";rr  c              3  <   K   | ]  }t        |t                y wrU   r(  r  s     rS   rb   z/_get_layouts_from_executable.<locals>.<genexpr>
  s     <qZ6"<rr  )get_parameter_layoutsget_output_layoutsrd   r?   r)   r   r   r   AssertionErrorrk   rm   )
r  r  rA  r  in_layouts_xlaout_layouts_xlanew_in_layoutsrV   r   new_out_layoutss
             rS   _get_layouts_from_executabler1  
  s   A#99;N$779O 1#$7$89N%&9&:;O.~z2 da"A!V%=a%C21#T! =  !" "
 ! /4 da"A!V%=a%C21#T! =! "# #
 1 
;N;	;;	;	<O<	<<	<		((?AS_$gK0@&@@@s    D1 1Ec                p    t        j                  t        j                  |             j	                  |       S rU   )r   r%  re  rf  r  )r  s    rS   get_logical_mesh_idsr3  
  s%    	499Z(	)	1	1*	==rR   c                V   |	dkD  r|	d}}n|rd|j                   }}n|j                   d}}|	dkD  rd }n|j                  ||f      }|
j                  dd       }t        j                  ||||||
|t        j
                  |       |	      }|j                  }|rl|J t        |j                  j                               |_
        t        t        |j                  j                                     j                  d      |_        ||_        t        |      |_        t        |      |_        |S )NrY   fdo_profile)	r&  r  rd  r  use_auto_spmd_partitioningr  r5  r  r   r  )r   r  r  r   r  r  executable_build_optionsr   r   r   !auto_spmd_partitioning_mesh_shaper3  auto_spmd_partitioning_mesh_idsr  -allow_spmd_sharding_propagation_to_parameters)allow_spmd_sharding_propagation_to_output)computationr  r  r'  r  allow_prop_to_inputsallow_prop_to_outputsr   r  r  rf  r&  r  xla_device_assignmentr5  r  optss                    rS   create_compile_optionsrA  
  s)    !^#-q.L#$fkk.L#);;.L!^ !"NNL.+IJ $$]D9+00#-)!3,44[A
/ 
	1	1$-1$**2C2C2E-FD*T$**"3"3"567	 	( 3=//7;<P7Q$4378M3N$0	rR   c                x  
  t        j                  
fdt        g      t        j                  t	        
                  }t        |      }t        | |||||||	|||      }t        j                  d|t        j                        5  t        j                  |	| |||
|      }d d d        |S # 1 sw Y   S xY w)Nc                    |    S rU   rQ   )ru   das    rS   r[   z%_cached_compilation.<locals>.<lambda>  s    r!u rR   r  r  r  )r   r  r  r%  rd   r  rA  r   r  r  r   r  )r<  r   r  r  r'  r  r=  r>  r*  r   rD  r  r  r  devrf  r  r  s             `       rS   _cached_compilationrF    s     	7_fX6ryyR7IJ#./*4
4F17	:')/
   H899;  33c?N
MN 
 
s   	B//B9c                    t        | |t        |      |      }||S g }t        |||      D ]A  \  }}}	t        |t              r}|	t
        j                  urYt        j                  |	j                  t        j                        r+t        j                  |	j                  |	j                  |      }|j                  |       |j                  |	j                         }
|j                  |	j                         }t        j                  |	j                  t        j                        s8|
j#                         s(t%        j&                  |
|      st)        d| d| d      |j                  |       D t+        ||||      }|S )a_  Returns in_shardings extracted from XLA or checks and returns original
  shardings.

  If in_shardings exist on `jit` or on `jax.Array`, then this function will
  check that sharding against what XLA returns as in_shardings. If they don't
  match, an error is raised.

  If in_sharding is unspecified, then the sharding returned by XLA is returned.
  (Unexpected XLA sharding override: (XLA) r&   (User sharding))r  rd   r?   r   r9   r   r   r   r   r   r   r   logical_shardingr   rk   r  rs  	is_manualr   r7  r,  r  )r  r  r  r  r  in_shardings_xlanew_in_shardingsxla_sorigr   	xla_hlo_s
orig_hlo_ss               rS   !_maybe_get_and_check_in_shardingsrR  ,  s`    0k3#79LN#$4l$35 $eT4$()
d))
)


DJJ
8//

DJJNe$,,TYY7i,,TYY7j

FOO<!!#33IzJ6ugT$ H  	 d##$& 2$oH 
rR   c                   t        | |t        |      |      }||S g }t        |||      D ]  \  }}}	t        |t              r}|	t
        j                  urYt        j                  |	j                  t        j                        r+t        j                  |	j                  |	j                  |      }|j                  |       t        j                   |      r|	t
        j                  urYt        j                  |	j                  t        j                        r+t        j                  |	j                  |	j                  |      }	 |j                  t#        ||	|             4|j%                  |	j&                        }
|j%                  |	j&                        }t        j                  |	j                  t        j                        sQ|
j)                         sAt+        j,                  |
|      r|j.                  |j.                  k7  rt1        d| d| d      |j                  |        |S #  |j                  |       Y xY w)NrH  r&  rI  )r  rd   r?   r   r9   r   r   r   r   r   r   r   rJ  r   rk   r(   contains_unconstrainedr  r  rs  rK  r   r7  r0  r,  )r  r  r  r  r  out_shardings_xlanew_out_shardingsrN  rO  r   rP  rQ  s               rS   "_maybe_get_and_check_out_shardingsrW  W  s    8k3'7#8 #$5}$46 %eT4$()
d))
)


DJJ
8//

DJJNu%		$	$T	*
d))
)


DJJ
8//

DJJN(  !9%t!LM ,,TYY7i,,TYY7j

FOO<!!#33IzJ 0 006ugT$ H  	 t$5%6 
(  's   H**I c                    t        |      dk(  r:| D cg c].  }t        |t              rt        |d   |j                        n|0 c}S | S c c}w )NrY   r   rm  )rd   r   r<   r;   r0  )rn   rd  r   s      rS   finalize_shardingsrY    s\    	q <EG78z!]+ !!21!51==Q123 G G	Gs   3Ac                j    d|z  t        d | D              z   }d|z  t        d |D              z   }||fS )Nr  c              3  H   K   | ]  }t        |t        t        f        y wrU   )r   r9   r8   r  s     rS   rb   z+get_prop_to_input_output.<locals>.<genexpr>  s&      @E23j%t,-@Es    "c              3  v   K   | ]1  }t        |t        t        f      xs t        j                  |       3 y wrU   )r   r9   r8   r(   rT  r  s     rS   rb   z+get_prop_to_input_output.<locals>.<genexpr>  s:      A
 %t,-O1L1LQ1OOAs   79)r   )r  r  r  r=  r>  s        rS   get_prop_to_input_outputr]    sb    !$77% @E7C@E ;E E"%885 AA <  
4	44rR   c           
     L   t        | t              rt        | j                  t              ry| j                  j                  t        |      k7  r1t        d| j                  j                   d|  dt        |             | j                  t        | j                  |            S | S )NzThe size of abstract mesh z in z- must match the length of device assignment: )r  )	r   r=   r  r3   r   rd   r   r   r  )rv   rD  s     rS   maybe_concretize_meshr_    s    =)-}}SW$&x}}'9'9&:$xj I558WI?@ @ ?? :8==" M?NN	/rR   c                  ~   e Zd ZU ded<   ded<   ded<   ded<   d	ed
<   ded<   d	ed<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   d ed!<   d"ed#<   d$ Zd(d%Ze	 	 	 	 	 	 	 d)	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d*d'       Zy&)+r  r
   r  xc.DeviceListr  r  r   rw  input_avalsru  rv  output_avalsry  r   r   r   r   rz  r(  r  r{  r)  r*  r!  r  r"  r  r  Sequence[Layout | None]xla_in_layoutsdispatch_in_layoutsxla_out_layoutsAllArgsInfo | Noner  r#  r  c                   t        | j                  | j                        }t        | j                  | j
                  | j                        }t        | j                  | j                  | j                  ||| j                  | j                  | j                  t        | j                        | j                   | j"                  | j$                        }|S rU   )r  rv  rf  r  rc  ry  r   r  r  r   r   r(  r  r)  r   r*  r  r  r  )rL  r  r  r  s       rS   r  z(UnloadedMeshExecutable.build_unsafe_call  s     4 4d6N6NOK1400$..BK $TYYkT++T-A-A4>>T  !4#4#4dhh	K
 rR   c                6   t        | j                  | j                  | j                  | j                  | j
                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  |       S rU   )r  r  r  rb  rc  rv  ry  r  r  re  rf  rg  r  r  r  s    rS   r  zUnloadedMeshExecutable.load  s}    $--t/E/E**D,=,=..0E0E1143D3D--t/G/G..$:L:L   rR   Nc                   ~t        d      t        t        j                        sJ t	        fd|D              }t	        fd|D              }|!|j
                  rt        j                  |      }t        ||t        |
            \  }}d }|rDt        j                  j                  ||g      D ]   }t        |t              s|j                  } n t        j                   d       t#        || ||||||t	        |      ||||      } |rt|J t%        | |      \  }!}"t'        |!|      D #cg c]  \  }#}t        |t              r|#n| }}#}t'        |"|      D #$cg c]  \  }#}$t        |$t              r|#n|$ }}#}$nl|dk(  r5|J t)        | ||t        |
            }t+        | ||t        |
            }n2t-        | j/                         t        |      t        |            \  }}}t1        | ||t        |
            \  }%}&~~t'        |%||      D '()cg c]  \  }'}(})t3        |'|(|)      rd n|' }*}(}'})t5        ||||||      }t7        |      }t7        |      }t9        di d| dd|d	|d
|d|d|d|d| d|	d|
d|d|d|d|d|d|%d|*d|&d|d|j;                         S c c}}#w c c}$}#w c c})}(}'w )Nzwdevice_assignment cannot be `None` during compilation. Please pass a tuple of devices to `.compile(device_assignment=)`c              3  6   K   | ]  }t        |        y wrU   r_  )r`   ru   r  s     rS   rb   z2UnloadedMeshExecutable.from_hlo.<locals>.<genexpr>  s       0 /q+> 0   c              3  6   K   | ]  }t        |        y wrU   rm  )r`   r   r  s     rS   rb   z2UnloadedMeshExecutable.from_hlo.<locals>.<genexpr>  s       2 0;? 2rn  pxla_cached_compilationrY   r  r  r   rb  rv  rc  ry  r   r   r(  r  r)  r*  r  r  r  re  rf  rg  r  r  rQ   )r  r   r   rQ  r   r  r(   r  r]  rd   r  r  r  r8   r  r!   r   rF  r  r?   rR  rW  _get_metadata_jit_pmapr  r1  r   r  rY  r  r  )+r   r0   r  r  r  r  r  r'  r  r(  r  r*  r)  r  r   r  r   r  rA  r  rJ  r  r  r,  r  r  r  r  r=  r>  r  ru   r  rL  rU  rV   r   re  rg  r   r~   r}   rf  s+                  `                           rS   rh  zUnloadedMeshExecutable.from_hlo  s   < 	@A A k2==111 0".0 0L 2#02 2M #(8(F(F**3/c2JmS%93;// Dxx%%|]&CD !a$

 	OO-.(T4&(<u^4g{(-	9N ,U
$- )) #++;\"JL!Q &a.aA5 Ll L $,,=}#MO1a 'q$/qQ6 Om O 
q||8L+ " ;M;8H " ?U((*C,=s=?Q?S;mY 'C
K_1E'G#NOK !O Aq! "!Q*1 
 1m_6F.M &lK@L&}kBM! %%%% % $	%
 %% &% '% % % ,% (% % &% "% %  .!%" &#%$ 0%%& ('%( $)%* $+%* &*TV+,KLO.s   2J;"KKr  )rY   NNNNNN):r   r   r0   rZ  r  rw  r  rw  r  zSequence[JSharding | AUTO]r  z-Sequence[JSharding | AUTO | UnspecifiedValue]r  r   r'  r   r  r   r(  rz  r  rz  r*  r  r)  r
   r  r!  r   r  r  xc.DeviceList | Noner   r   r  MaybeLayoutrA  rs  r  r  rJ  r  r  r  r  r"  r,  r  r  rh  r  r#  r  zSequence[JSharding] | Noner  Mesh | Nonerl  r  )rN   rO   rP   r  r  r  r  rh  rQ   rR   rS   r  r    s   $$&&%%''/)&&$$))..**##--
  , "#*.FJ37=ADH+/7{,{, 5{, "7{, 8	{,
 N{, #{,  {, $({, #4{, !2{,  ){, {, &{, "{, 1{,  !{," '#{,$ (%{,& &A'{,(  ){,* +{,, (-{,. "D/{,0 11{,2 ;3{,4 (B5{,6 )7{,8 9{, {,rR   r  c                  h    e Zd ZU ded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   y)MeshExecutableFastpathDatazxc.LoadedExecutabler  r
   out_pytree_defru  r  r  rw  r  r  out_committedzIterable[bool]kept_var_bitvecrd  in_device_local_layoutszSequence[ArrayLike]r   Nr  rQ   rR   rS   rv  rv  H  s8    %%##$$""!!22!!rR   rv  )r  kw_onlyc                      e Zd ZU dZded<   dZded<   dZded<   dZded	<   dZd
ed<   dZ	ded<   dZ
d
ed<   dZded<   dZd
ed<   dZded<   dZd
ed<   dZded<   dZded<   ej$                  d        Zy)JitGlobalCppCacheKeysNztuple[int, ...] | Noner  tuple[str, ...] | Nonedonate_argnameszxc.Device | Noner  r  r   zPyTreeDef | Nonein_shardings_treedefztuple[Any, ...] | Nonein_shardings_leavesout_shardings_treedefout_shardings_leavesin_layouts_treedefin_layouts_leavesout_layouts_treedefout_layouts_leavesz"tuple[tuple[str, Any], ...] | Noner  c                   | j                   d uxs | j                  d uxs | j                  d uxs | j                  d uxs t	        d | j
                  D              xsf t	        d | j                  D              xsH t	        d | j                  D              xs* t	        d | j                  D              xs | j                  S )Nc              3  >   K   | ]  }t        |t                 y wrU   r  r  s     rS   rb   zEJitGlobalCppCacheKeys.contains_explicit_attributes.<locals>.<genexpr>j  s     VJq"233Vr  c              3  >   K   | ]  }t        |t                 y wrU   r  r  s     rS   rb   zEJitGlobalCppCacheKeys.contains_explicit_attributes.<locals>.<genexpr>k  s     WJq"233Wr  c              3  $   K   | ]  }|d u 
 y wrU   rQ   r  s     rS   rb   zEJitGlobalCppCacheKeys.contains_explicit_attributes.<locals>.<genexpr>l  s     >!>rc   c              3  $   K   | ]  }|d u 
 y wrU   rQ   r  s     rS   rb   zEJitGlobalCppCacheKeys.contains_explicit_attributes.<locals>.<genexpr>m  s     ?!?rc   )
r  r  r  r   r   r  r  r  r  r  r  s    rS   contains_explicit_attributesz2JitGlobalCppCacheKeys.contains_explicit_attributesd  s    t+ &  ,&KKt#& LL$& VT=U=UVV	&
 WT=V=VWW& >t'='=>>& ?t'>'>??& %%'rR   )rN   rO   rP   r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r	   r  rQ   rR   rS   r}  r}  T  s    +/.(/,0/)0!&
!':+/(/04-4,0)015.5)-&-.2+2*.'./3,3=A:A	' 	'rR   r}  c                n    t        j                  | |      }t         j                  j                  |d       S rU   )r   tree_unflattendispatch_registryflatten)out_treeout_flat
out_unflats      rS   reflatten_outputs_for_dispatchr  q  s/     ''(;*		$	$	,	,Z	>>rR   c                  J    e Zd Zg dZ	 	 d	 d	dZed
d       Zd Zd ZddZ	y)r  )r  r  r  r  r  _in_shardings_out_shardings_auto_spmd_lowering_kept_var_idx_xla_in_layouts_dispatch_in_layouts_xla_out_layouts_mut_all_args_infor  Nc                    || _         || _        || _        || _        d | _        || _        || _        || _        || _        |	| _	        |
| _
        || _        || _        || _        || _        y rU   )r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )rL  r  r  r  r  r  r  r  r  re  rf  rg  r  r  r  s                  rS   rK  zMeshExecutable.__init__  s{    
 )D.D DMDND%D'D1D%D)D 3D+DDI'D 3DrR   c                \    | j                   | j                         | _         | j                   S rU   r  r  s    rS   r  zMeshExecutable.unsafe_call  r  rR   c                    | j                   S rU   r  r  s    rS   r  z'MeshExecutable.xla_extension_executable  r  rR   c           
        t        |      D cg c]  \  }}|| j                  v s| }}}| j                  j| j                  j                  j                  Jt        | j                  j                  j                        D cg c]  \  }}|| j                  v r| }}}ndt        |      z  }| j                  Nt        | j                  j                  t        t        j                  |      | j                  j                         nEt        | j                  t        t        j                  |      t        j                  dd|d              | j                  s#t        || j                  | j                  |       nag || j                  j                   }|dt        | j                  j                         z  z  }t        || j                  | j                  |        | j"                  | S c c}}w c c}}w )Nr  r  z	<unknown>)rj   r  r  r   r$  rd   r  r  r   r   r   	DebugInfor  %check_array_xla_sharding_layout_matchr  r  r	  r  )rL  rq   ru   r}   args_after_dcer  arg_names_after_dces          rS   r  zMeshExecutable.call  s   $-dOODAqqD<N<N7NaONO'&&00<!$"5"5"@"@"J"JK&1$$$$ & & "C$77&t22;;"4#:#:DA#22==?
 t}}"4#:#:NK#~~.>.A4 IJ 99+
$,,d.B.B
 <;$))*:*:;nec$))*:*:&;;<+
$,,d.B.B
 4T""= P&s   G/G/G5c           
     H    t         j                  t              r, j                  j                  s j                  j                  ry  fd}t
        j                  j                   j                  j                  d |g g t               t        j                  t              S )Nc                    t        j                  j                  g| i |\  }}}j                  s;t	        j
                  |      \  }}t        d |D              xr j                   }nd}|r9|D cg c]  }|j                   }}|D cg c]  }|j                   }	}t        t        j                        t        |      z         D 
cg c]  }
|
j                  v  }}
t        j                  j                         D cg c]]  \  }}|t"        j$                  urDt'        j(                  |j*                  t&        j,                        rt/        j0                  ||      n|_ }}}t3        j4                  |j6                  ||	|j8                  j                  	      }nd }||dfS c c}w c c}w c c}
w c c}}w )Nc              3  P   K   | ]  }t        |t        j                           y wrU   )r   r   r   )r`   rV   s     rS   rb   zIMeshExecutable.create_cpp_call.<locals>.aot_cache_miss.<locals>.<genexpr>  s     JAJq",,7Jr|  F)r   Compiledr  is_highr  r  rm   r  r   
_committedr  rd   r   r  r  r  r  r   r   r   r   r   r   r   physical_shardingrv  r  r  r  )rq   ri  r   r  	args_flatout_tree_dispatchuse_fastpathr   r  rx  ru   ry  r~   r}   r  fastpath_datar   rL  s                   rS   aot_cache_missz6MeshExecutable.create_cpp_call.<locals>.aot_cache_miss  s    #)//"6"6v"O"O"OdHi^^&DOOX''##JJJ * $		M 	 	%-.QVV.	./78!88$)#f.?.?*@3y>*Q$RT   2 22 T T D..>	
 1 +++0A0A!''6??0[ ,,Q2
 
 3!2LM?%%v'8'8:
 =%''! /8T
s   9F5F:F?A"G)r   r  r  r  r  r   _xlar   r   r}  r   r  cc_shard_arg)rL  r   r  s   `` rS   create_cpp_callzMeshExecutable.create_cpp_call  s{    t''):;  66  33(@ 77<<t^R!<!<lL LrR   rU  )r  rh  r  )r   zstages.CompiledCallParams)
rN   rO   rP   r  rK  r  r  r  r  r  rQ   rR   rS   r  r  y  sE    ) 48#'	4 14.  #B(LrR   r  c                \    t        |g|gt        j                  j                  g| g      d   S r  )r   r   r  r  )rV   rv   rw   s      rS   r  r    s5    	XJ2+@+@+L+L*MC

 rR   c           
     n   t        |       t        |      k7  r#t        dt        |        dt        |             |j                  t        |             D cg c]  }d| d
 }}g }t        | ||      D ]n  \  }}}|j                  |j
                  f|j                  |j
                  fk7  s9|j                  d| d|j                          d|j                                 p |rId}dj                  |d |       }	t        |      |k\  rd	| d
t        |       }
nd}
t        d|
 d|	       y c c}w )NzComputation compiled for z inputs but called with '	Argument z compiled with z and called with    
z
The first z of ThezArgument types differ from the types for which this computation was compiled. Perhaps you are calling the compiled executable with a different enable_x64 mode than when it was AOT compiled? z mismatches are:
)	rd   r   r/  r?   r   r   rk   r  r  )	ref_avalsr  r+  r   r$  errorsref_avalarg_avalmax_num_errors
str_errorsnum_mismatch_strs              rS   r  r    sk   ^s9~%

#C	N#3 4y>*	,- - (8'F'Fs9~'VWt4&{W)W&"*9i"K *h$ 	'HNNHNN+KKmmdV?8+=+=+?*@ A$$&')** N6/>23J
6{n$%n%5T#f+G
	D 
.zl	<= =  Xs   D2c                    t         j                  j                  |       }|g|z  }|g|z  }d}|||t        t	        |             fS r  )r   r<   r  r  r   )r  num_in_shardingsnum_out_shardingsgsr  r  r   s          rS   rq  rq    sY     ##22=A"((,$**- )
y
eM2
3
5 5rR   c                ^    | D ](  }t        |t        t        f      rt        |dd      s( y y)N_device_backendFT)r   r9   r8   r  )rn   ru   s     rS   !check_device_backend_on_shardingsr  !  s9     a!&-.q#U+	
 
rR   c                   g }d}t        | |||      D ]K  \  }}}}	t        |t        j                        s$t        |t        t
        f      r;t        |g      }
|
ss|j                  rg|j                  j                  ||j                        sA|j                  d|	 d|j                  j                          d|j                   d| df       |
r|j                  s|j                  j                  ||j                  j                  |k7  s|j                  d|	 d|j                  j                          d|j                  j                   d| d	f       N |rt!        |d |       \  }}d
j#                  |      }t%        d |D              rd}nt%        d |D              rd}nd}t'        |      |k  rdt'        |       dn| dt'        |       }t)        d| d| d| d|       y )Nr  r  z with shape z:
  Passed sharding: z
  Required sharding: rv   z:
  Passed layout: z
  Required layout: rw   r  c              3  &   K   | ]	  }|d k(    yw)rv   NrQ   r`   r  s     rS   rb   z8check_array_xla_sharding_layout_match.<locals>.<genexpr>N  s     
0q1
?
0   rn   c              3  &   K   | ]	  }|d k(    yw)rw   NrQ   r  s     rS   rb   z8check_array_xla_sharding_layout_match.<locals>.<genexpr>P  s     0qQ(]0r  ro   zshardings and layoutszthe z mismatchesz mismatches out of z#Computation was compiled for input z that disagree with the z% of arguments passed to it. Here are z:
)r  r   r   r   r9   r8   r  r  rv   r  rs  rk   r   r  r  rw   rE   r  rm   rd   r   )rq   r  r  r$  r  
num_errorsrs   r   r$  r   db_xsfirst_errorserror_kindsr  kind_strr  s                   rS   r  r  *  s    &*t\:yI c2r4c5??+"'./-rd3EcnnLL))"chh7mmdV<(:(:(<'= >  #~ .""$' 	  cnn

%".

RmmdV<(:(:(<'= >!jj//0 1  "t% 	 '2  &vkz': ;L+<(J

0K
00h	0K0	0h(h+.v;+C$s6{m;',)#f+7  
-hZ7O* $%S	67 7 rR   c                b    |t        |       z
  }|dkD  r| d|z  z  } t        | ||      }t        | S )Nr   rU   )rd   rH   r1   )r  r   r  	too_shortnew_partitionss        rS   
batch_specr  \  s>    CIo)]GiDc3/.		''rR   )T)rn   ru  ro   zSequence[Any | None]rp   zSequence[xc.ArrayCopySemantics]rq   r{  rr   r   rl  zSequence[xc.ArrayImpl])TN)r   core.ShapedArrayrv   r  r   r{  r   zSequence[xc.Device]r   r   r   zbool | None)r   r  )r   core.AbstractValuerv   r  r|   ztuple[Index, ...] | Nonerl  z#Callable[[list[xc.ArrayImpl]], Any])r   r  r   r   rl  z'Callable[[Sequence[xc.ArrayImpl]], Any])r   lu.WrappedFunr   r  r   r  r   r  r   r  r   r  r   r   r   Sequence[int | None]r   r  r   r  r   r   rl  z tuple[Callable, list[ArrayLike]])r   r  )r   r  r   r  r   r  r   r  r   r  r   r  r   r   r   r  r   r  r   r  r   r   )r8  tuple[int | None, ...]rl  r  )
r   r   r?  r  r@  zlist[core.AxisName]rA  zlist[tuple[int | None, ...]]rl  z)tuple[Callable, dict[core.AxisName, int]])rs  r  r  zIterable[int]r  
int | Nonerl  r  )r   r  r   r  r  r
   r  r  r  r  rl  z$tuple[Any, dict[core.AxisName, int]])
rs  r  r  r  r  r  rq  r  rl  r  )r   r  r  r  r   r  r   r  r   r  r   r  r   r   r   r  r   r  r   r  r   r   )r{  zcore.Primitive)r  
core.Jaxprrl  r  )r  zcore.JaxprEqnrl  r  )r   zdict[str, Any]rl  r  )r  r  r   r  r   r  rl  r  )r  r  r   r  rl  z4tuple[core.Jaxpr, list[Any], ReplicaInfo, ShardInfo])r   r  r  r  r   r  r   r  r   r  r   r  r   r   r   r  r   r  r  r  )$r   r  r   r  r   r  r   r  r   r  r   r   r   r  r   r  r   r   r  r  r  r~  r  mlir.LoweringParametersr  core.ClosedJaxprr   r  r  r  r   r  r  r  rl  r  )r   r  r   r  r   r$   rl  r$   )r   zcore.AxisSizer   r  r   r  rl  r  )r   r  rl  r$   )r  rw  r  ru  rl  r  )r  rw  rn   ru  r   r   rl  r  )r   core.ParamDictrJ  r  rK  r  rl  r  )rN  sharding_impls.AxisEnvrl  ztuple[tuple[int, ...]])r  mlir.LoweringRuleContext)r  r  r   r  )r  r  r  r  )r   r7   )r  r7   )rR  r7   )r   r  r  &frozenset[sharding_impls.MeshAxisName]r  r4   )rR  r7   r  r4   r  r  )r  r  rR  r7   r  r4   r  r  )rn   z-Iterable[JSharding | AUTO | UnspecifiedValue]rl  r   )rl  z	xc.Device)rn   zIterable[ShardingInfo]r  r  rl  z3tuple[xc.Client, tuple[xc.Device, ...] | None, int])r  r  rl  z%tuple[core.Jaxpr, set[int], set[int]])r  r  rl  z;tuple[core.ClosedJaxpr, Sequence[int | None], MutationData])r  r  rl  z'tuple[core.ClosedJaxpr, list[core.Ref]])r  r  rl  r  )
rd  r  r   r  r   r   r  r  rl  None)r  r  r  r  r  zNone | tuple[None | int, ...]rD  ztuple[None | str, ...]r  ztuple[str, ...]r  r  r  zAbstractMesh | None)rd  ztuple[xc.Device, ...]rl  ra  )rd  z,tuple[xc.Device, ...] | xc.DeviceList | Nonerl  rr  )r  r  )r  r  rl  ztuple[None | Layout])r~   r  rs  r  rl  r<   )r  r  r  r!  r  rj  )r  r4   rl  r4   )r  r  r  r   r  r   r  Sequence[MaybeSharding]r  r  r  rs  rA  rs  r   r  r
  r   r  r4   r  r  r  r~  r  r  r  r#  rl  r  )r   r  rv   zMaybeSharding | AUTOrl  zJSharding | AUTO | None)rl  z7tuple[Sequence[xc.OpSharding], Sequence[xc.OpSharding]])r  r4   rl  z;tuple[tuple[PartitionSpec, ...], tuple[PartitionSpec, ...]])r  ra  r  r  r  r  rl  z-Sequence[sharding_impls.GSPMDSharding] | None)r  ra  r  r  r  r  rl  zSequence[GSPMDSharding] | None)r  r4   rl  z7tuple[Sequence[NamedSharding], Sequence[NamedSharding]])r
  r<   r  r=   rl  r=   )r
  r<   r  r;   rl  r;   rU  )r  rt  )r#  zLayout | AutoLayoutr$  r)   rl  r   )rl  z7tuple[Sequence[Layout | None], Sequence[Layout | None]])rD  ra  )r+  r|  )rl  r   )r  ru  r  zSequence[Layout]r$  r  rl  r  (7  r$  
__future__r   rh   r   collections.abcr   r   r   r2  r  r   r	   	itertoolsr  r6  re  r    r
   r   r   r   r<  numpyr   jax._srcr   r   r   r   r   r   r   r   r   r   r   lur   r   r   r   r   r   r   r!   r"   r   jax._src.abstract_arraysr#   jax._src.corer$   jax._src.interpretersr%   r&   r'   r	  r(   jax._src.layoutr)   r*   r+   jax._src.libr,   r-   r.   r   jax._src.lib.mlirr/   jax._src.lib.mlir.dialectsr0   jax._src.partition_specr1   jax._src.shardingr2   r  jax._src.meshr3   r4   r5   r6   jax._src.sharding_implsr7   r8   r9   r:   r;   r<   r=   r  jax._src.utilr>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   jax._src.state.typesrI   rJ   jax._src.typingrK   r   rM   r   r   r  
unsafe_zip	getLoggerrN   r4  r  slicer   Index	PyTreeDef
NoShardingChunked	UnstackedShardedAxis
ReplicatedAvalDimShardingMeshAxisNameMeshDimAssignmentShardingSpecrW   ro  r   rf   r  cacher   r   maMaskedArrayr   _tr   r   python_scalar_typesr   typed_scalar_typesr   r  r   r   r   r   r   r  ResultHandlerPxlaResultHandlerr   r   r   r   r  r  r   r9  rD  rE  Tracer  rr  ro  r  Tracerr  r   	dataclassr  r  r  r  r  r  r  r  r  r  transformation2r  r  r  r  rR  AvalMapHandlerPairrS  rX  Loweringr  rq  rg  
Executablern  r  r  r  r  r  r  MapPrimitive
xla_pmap_prG  r%  def_implr6  r8  rG  rM  rT  rY  r`  call_param_updaterscall_partial_eval_custom_rulepartial_eval_jaxpr_custom_rules	dce_rulescall_linearize_param_updaterscall_transpose_param_updatersmap_transposeprimitive_transposesrn  rz  r~  r  r  r  r  r  register_loweringr  r  r  r  	Primitivefull_to_shard_pdef_abstract_evalr  r  r  shard_to_full_pr  r  r  r  ShardingInfor  r  MaybeShardingr  r  r  r  r  r&  r(  r?  rO  rR  r  rY  r^  rh  rs  rj  r*  r~  r0  r  r  r  r  rE  r  r  r  r  r  r  r  r  r  r  r  r   r1  r3  rA  rF  rR  rW  rY  r]  r_  r  rv  r}  r  r  r  r  rq  r  r  r  rQ   rR   rS   <module>r,     s@   8 "  " 8 8  .     / /             & ! #   #     % 0 % $ * 4 & 6 6  1 )   * 1 3. .L L L) ) ) ) 8 %$  x
CCZ			8	$c5%c5j 13 6778	&&


 
 $$	((&&
	7J67**+z12 **  
 '"'!' 4' 	'
 ' ' 'b     T6 72W )< 255$$ %"  +B*R+ -< 8(( )/ 
$
$ 0B/R0 
%
% /B.R/>  4 488  HL15.!*.0=. 3.@D. $/.$O
 XZ T Y
7 %: [ !C
CC &C )	C2 S"''"7"778 KM H MC
C7;C,C. MO I N%"	%" %" 	%"
 %" %" "%" %" "%" 7%" #%" #'%" &%"N* z  2"2'2 2 "%	2
 -2 2 -2  B2  .2 .22h+ T66> 7$ ?-?,HI^Gtzz ^G@G'G,6G !9 !+  ; *EE E%=E3 30 4$.4!.4 "%4 ),	4
  44  4  44 'I4 '54 594 
4: d#! ! $!4
 *  25 . 6&?
NN"%N9<NN  )	)$1)9)B6	66 6 	6
 6 (6 6 "6 76 (64 
UL	ULUL UL 	UL
 (UL UL "UL #UL #'UL (UL /UL 1UL #UL UL  !UL" #UL$ 
%UL$ #2%UL ULp"- 3=) C12? ; 	M/	M4F	Mfoo 8 g2 g2 g2T#V&& #DH4 42F F	M3	M(	M-;	M?+?"? ? '?s sl Tz*
?? 
  M "-.@ 0<<)7<IL<<95N &J  z "B,,B8:  " ": . *Z %?  z "/P     ,/P     ,&-b.>.>
&K  
 #?&A,,$45 48)@   z> 2))I>
 !$..1""+ #+
;CG8 			1O)OO 2O !$..1""- #- 			1>)O> 2>>CG 	)%t
+,
	#t)@7-%7-/7- 97-r i!11211*1 4 4&:  AA@A A 1"1,1 1 * * :NN"N*-N6@NEIN2 ^?,/^? ,I^? 7M^? (7^? 2I^? ,?^? ^?B T6*"/* 7*7C7	7  
 .)=. .6 uVZ567*  T62 7260#;"#;"*#; #;L T6M 7M08 
b!"b!b! b! *	b!
 +b! b! b! #b! b! b! 6b! /b! 1b! 0b!  !b! b!J
(< BNfoo BNJ+J+++@+.9.9 .9 	.9
 3.9bD!.D>ADD &D477<7 ! GG/<GANG .F M *BB/CB	B 5U 0 16 >B/:<!',0%)<%)P>*Z  .(V&R5 h, h, h,V	" 	" dD1' ' 2'8?sLV&& sLj
=/==@507%07 !07 	07
 
07d(rR   