
    ukis                    4   U d dl mZ d dlmZmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZm Z  d dlm!Z!m"Z"m#Z#m$Z$ d dl%m&Z' d dl(m)Z)m*Z*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 e0e:cZ:Z;e1e<cZ<Z=e	Z>e	Z?e	Z@e	ZAe	ZBeg e"f   ZCeeeCe>e@ge?f   ZDeeeAe?e@ge>f   ZEeeAge-f   ZFd\dZGi ZHdeId<   	 	 	 	 	 	 	 	 d]dZJi ZKdeId<   d^dZLi ZMdeId<   	 	 	 	 	 	 d_dZNi ZOdeId <    eP       aQd!eId"<   d`d#ZRdad$ZSej                  	 	 	 	 dbd%       ZU eVd      ZWdZX G d& d'e"      ZY ej                  d()       G d* d+             Z[d, Z\ G d- de!      Z]	 	 dcd.Z^ej                  d/        Z`ej                  ddd0       Za	 	 	 	 	 	 	 	 ded1Zbej                  d2        Zcd3 Zd	 	 	 	 	 	 dfd4Zee8	 	 	 	 	 	 dgd5       Zfd6 Zgd7 Zhd8 Zie8	 	 	 	 	 	 dhd9       Zjej                  d:        Zkej                  d;        Zlej                  d<        Zmd= Zn G d> d?      Zo eo       Zpej                  d@        Zq	 	 	 	 	 	 	 	 didAZrej                  dB        ZsdjdCZtedDeue	e
evdeue
evdf   dDf   f   f   f   Zwi ZxdEeIdF<    G dG dH      Zy ey       Zz G dI dJ      Z{ e{       Z|i Z}dEeIdK<   dL Z~dM ZdN ZdO ZdP ZdQ ZdR ZdS ZdT ZdjdUZdjdVZ G dW dXe      ZdkdYZdZ Zeexe <   d[ e}e <    e~ej                         y)l    )annotations)CallableSequenceN)partial)AnyUnion)config)core)typeof)source_info_util)linear_util)PartitionSpec)NamedSharding)mesh)ZeroSymbolicZeroadd_jaxvalsadd_jaxvals_p)TraceTracerTraceTagAxisName)partial_eval)tree_unflattentree_flatten	PyTreeDef)Array)unzip2safe_mapsafe_zip
split_listcanonicalize_axismoveaxisas_hashable_functioncurrymemoizeweakref_lru_cachetuple_insertc                `   t         j                  t        |            }|r |t        t        | |      |||      S t        |t              s|P|xr( t        |t        t        j                  |                  }| t        | ||t        j                               S |S J dt        |              )NzUnexpected type in ELT? )to_elt_handlersgettyper   to_elt
isinstanceintr"   lennpshapeBatchTracerr   current)traceget_idxxspechandlers        Y/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/interpreters/batching.pyr-   r-   9   s    Q('765'2GQEE$=%dC,<=D q$(8(@(@(BC (%&(
 7,T!WI665    zdict[type, ToEltHandler]r*   
BatchTracec                (    t         j                  t        |            }|r fd} |||||      S  j                  |      \  }}		 t	         j
                  j                  ||	||      S # t        $ r t        |j                  |      d w xY w)Nc                $    t        | ||      S N)from_elt)	axis_sizeeltaxisi	mesh_axisr5   s      r:   _contzfrom_elt.<locals>._contM   s    eY	1c4@@r;   )	from_elt_handlersr+   r,   to_batch_info	matchaxis	axis_datanameSpecMatchError	batch_dim)
r5   rA   rE   rD   r7   r8   r9   rF   valbdims
   ` ``      r:   r@   r@   I   s    !!$q'*'A5)Q--!!!$)#t9U__))9i4& &	 9
AKK
.D89s   #A/ /"Bzdict[type, FromEltHandler]rG   c                    ddl m} t        j                  t	        |             }|r ||       S |j                  dt        |             S )Nr   laxint32)jaxrR   make_iota_handlersr+   r,   iotar/   )rA   rR   r9   s      r:   	make_iotarW   X   s>     ""4	?3'988GS^,,r;   zdict[type, MakeIotaHandler]rU   c                    ||ft         | <   t        j                  |       |t        | <   |t        | <   |r
|t
        |<   y y r?   )
vmappables
spec_typesaddr*   rG   rU   )	data_type	spec_typeaxis_size_typer-   r@   rW   s         r:   register_vmappabler_   c   sC     %n5*Y..%/)!)IY">2Yr;   zdict[type, tuple[type, type]]rY   z	set[type]rZ   c                    t         j                  |       \  }}t        | = t        | = |t        v rt        |= t               t         j                         D ch c]  \  }}|	 c}}z  ay c c}}w r?   )rY   popr*   rG   rU   setvaluesrZ   )r\   _r^   r]   s       r:   unregister_vmappablere   n   sb     nnY/!^i 	"))>* 
eZ->->-@A\YyAA As   A)c                $    t        |       t        v S r?   )r,   rY   )r7   s    r:   is_vmappablerg   y   s    	aJ	r;   c                ~    t        ||      \  }} | |i |}t        |t              \  }}|j                  |       |S )N)is_leaf)r   r   rg   store)frj   in_tree	args_flatpy_args	py_kwargsansout_trees           r:   flatten_fun_for_vmaprr   |   sE     &gy9'9	7 i #sL9-#x++h	*r;   c                  P    e Zd Zg dZ	 d
	 ddZd Zed        Zd Zd Z	d Z
d	 Zy)r3   )rN   rM   source_infoNc                4   t         j                  j                  rbt        |      t        t
        fv sJ t        |      t
        u r8t        j                  |      }d|cxk  rt        |j                        k  sJ  J || _
        || _        || _        || _        y Nr   )r	   enable_checksvaluer,   	NotMappedr/   r
   get_avalr0   r2   _tracerN   rM   rt   )selfr5   rN   rM   rt   avals         r:   __init__zBatchTracer.__init__   s    !!)_C 0000	iC	}}S!I/DJJ/////DKDHDN"Dr;   c                N    d| j                    dt        | j                         dS )NzVmapTracer(aval=z
, batched=))r}   r   rN   r|   s    r:   _short_reprzBatchTracer._short_repr   s%    dii[
6$((3C2DAFFr;   c                $   t        j                  | j                        }| j                  j                  j
                  `t        j                  j                  rF|j                  |j                  t        | j                  j                  j
                        z
        }| j                  t        u r|S t        | j                        t        u r8t        j                   |j"                  | j                     | j                  |      S t%        d      )Nvmaz'batch dim should be int or `not_mapped`)r
   rz   rN   r{   rJ   	spmd_namer	   
_check_vmarx   updater   	frozensetrM   
not_mappedr,   r/   mapped_avalr2   	Exception)r|   r}   s     r:   r}   zBatchTracer.aval   s    =="D{{&&2				 	 {{9T[[%:%:%D%DEE  G~~#k	dnn		$djj8$..$OO?@@r;   c                h    | j                   t        u rt        j                  | j                        S | S r?   )rM   r   r
   
full_lowerrN   r   s    r:   r   zBatchTracer.full_lower   s&    ~~#__TXX&&kr;   c                x    | j                   ydt        |        dt        j                  | j                          S )N z!
This BatchTracer with object id z was created on line:
  )rt   idr   	summarizer   s    r:   _origin_msgzBatchTracer._origin_msg   sD    0D
 ;#--d.>.>?@B Cr;   c                :    d| j                   fd| j                  fgS )NrN   rM   )rN   rM   r   s    r:   	_contentszBatchTracer._contents   s    DHHT^^<==r;   c                    | j                   t        | j                         t        u rt        j                  | j
                        S | S r?   )rM   r,   r/   r
   get_referentrN   r   s    r:   r   zBatchTracer.get_referent   s6    ~~dnn!5!<txx((kr;   r?   )rM   zNotMapped | intrt   z"source_info_util.SourceInfo | None)__name__
__module____qualname__	__slots__r~   r   propertyr}   r   r   r   r    r;   r:   r3   r3      sL    1) BF
#>
#G A AC>r;   r3   T)frozenc                  P    e Zd ZU ded<   ded<   ded<   ded<   ed        Zd ZeZy	)
AxisDatar   rK   sizer   ztuple[Any, ...] | None_emac                   | j                   t        | j                   t              sJ | j                   y t        j                         j
                  r| j                   S j                  | j                   d      t        fd| j                   D              sJ t        j                  j                  k7  ry | j                   S )Nr   c              3  B   K   | ]  }j                   |   k(    y wr?   )_name_to_type).0ecur_mesh	ema0_types     r:   	<genexpr>z.AxisData.explicit_mesh_axis.<locals>.<genexpr>   s!     I!x%%a(I5Is   )
r   r.   tuplemesh_libget_abstract_meshemptyr   allAxisTypeExplicit)r|   r   r   s    @@r:   explicit_mesh_axiszAxisData.explicit_mesh_axis   s    99
499e <<<yy))+H~~YY&&tyy|4IItyyIIIIH%%...99r;   c           	     p    d| j                    d| j                   d| j                   d| j                   d	S )NzAxisData(name=z, size=z, spmd_name=z, explicit_mesh_axis=r   )rK   r   r   r   r   s    r:   __repr__zAxisData.__repr__   sC    TYYKwtyyk :..) *##'#:#:";1> ?r;   N)r   r   r   __annotations__r   r   r   __str__r   r;   r:   r   r      s7    	*	*/ 	 ?
 'r;   r   c                    | j                   }t        t        |j                  ||       }t	        |j
                  |      S r?   )r   Pr(   r8   r   r   )rJ   orig_shardingrC   rN   new_specs        r:   get_sharding_for_vmapr      s;    $$# ]//s;<(	}))8	44r;   c                  D     e Zd Z fdZd Zd Zd ZddZd Zd Z	 xZ
S )	r<   c                ~    t         |           || _        t        |t              sJ || _        || _        d| _        y )NF)superr~   parent_tracer.   r   rJ   tagrequires_low)r|   r   r   rJ   	__class__s       r:   r~   zBatchTrace.__init__   s<    	G$Di***DNDHDr;   c                    t        |t              r:|j                  j                  | j                  u r|j                  |j
                  fS |t        fS r?   )r.   r3   r{   r   rN   rM   r   )r|   rN   s     r:   rH   zBatchTrace.to_batch_info   s<    #{#

$(((BWWcmm##*_r;   c                \    t        t         j                  |            \  }}t        d |D              }|t        v r|rE|t
        v r=t         fdt        |   |      D              s|j                   j                  ||      S t        j                   j                        5  t	        |    j                  ||fi |\  }}d d d        n.|r|j                   j                  ||      S t        d| d      t        j                         }	|j                  r^t        j                   j                        5  t!              D 
cg c]  \  }
}|t"        urt%         |
||	      n|
 c}}
cd d d        S t"        urt%         ||	      S S # 1 sw Y   xY wc c}}
w # 1 sw Y   y xY w)Nc              3  ,   K   | ]  }|t         u   y wr?   r   )r   rO   s     r:   r   z/BatchTrace.process_primitive.<locals>.<genexpr>   s     A$*,As   c              3  P   K   | ]  }j                   j                  |k(    y wr?   )rJ   rK   )r   	axis_namer|   s     r:   r   z/BatchTrace.process_primitive.<locals>.<genexpr>   s*      F# nn))Y6 Fs   #&zBatching rule for 'z' not implemented)r   maprH   r   fancy_primitive_batchersskippable_batchersanybind_with_tracer   r
   set_current_tracerJ   NotImplementedErrorr   r4   multiple_resultszipr   r3   )r|   ptracersparamsvals_indims_inargs_not_mappedval_outdim_outsrcr7   ds   `           r:   process_primitivezBatchTrace.process_primitive   s   c$"4"4g>?GWAAAO$$
%% F'9!'<V'DF F  !2!2GVDD##D$5$56 	:5a8nngw:28:
'7	: 	: 
t00'6BB"5aS8I JKK

"
"
$C!!$"3"34 313Aq 120CD!Q,J 33 3
 
* $#6 9079	: 	:33 3s*   $F9F"	"F+F"FF""F+c           
     4   |j                   sJ t        ||j                  d|j                              }t	        t        | j                  |            \  }}t        || j                  | j                  t        |            \  }}t        j                  | j                        5   |j                  |g|i |}	d d d        t        j                          }
t#        	 |             D cg c]  \  }}t%        | |||
       c}}S # 1 sw Y   MxY wc c}}w )NrK   )rK   )r   dictr+   r   r   r   rH   batch_subtracer   rJ   r   r
   r   r   bindr   r4   r   r3   )r|   call_primitiverk   r   r   valsdimsf_dims_outvals_outr   vr   s                r:   process_callzBatchTrace.process_call  s    ****&vzz&!**=>FD..89JD$!!TXXt~~uT{KLB			 1 1	2 :$$$R9$9&9h:

"
"
$C588:5NOTQKaC(OO: : Ps   %D,DDc           
        t        t        | j                  |            \  }}d t        fdt	        ||d         D              }t        fdt	        ||d         D              }t        || j                  | j                  |      \  }|d   t              fd       }	t        |||	      }
t        j                  | j                        5   |j                  |g|i |
}d d d        t	                              D cg c]  \  }} ||      r
||k  r|d	z   n| }}}t        j                         }t	        |      D cg c]  \  }}t!        | |||       c}}S # 1 sw Y   xY wc c}}w c c}}w )
Nc                    | d uxr |t         uS r?   r   )in_out_axisr   s     r:   both_mappedz+BatchTrace.process_map.<locals>.both_mapped,  s    $<*)<<r;   c              3  N   K   | ]  \  }} ||      r
||k  r|d z   n|  yw   Nr   r   r   in_axisr   s      r:   r   z)BatchTrace.process_map.<locals>.<genexpr>.  s5      6
!W !!,ggk7J6   "%in_axesc              3  N   K   | ]  \  }} ||      r
||k  r|d z
  n|  ywr   r   r   s      r:   r   z)BatchTrace.process_map.<locals>.<genexpr>1  s5      6
!W 7A&7Q;a!eA=6r   out_axes_thunk)closurec                 T    t         fdt                              D              S )Nc              3  N   K   | ]  \  }} ||      r
||k  r|d z   n|  ywr   r   )r   out_axisr   r   s      r:   r   zEBatchTrace.process_map.<locals>.new_out_axes_thunk.<locals>.<genexpr>;  s8      J#! $/x#;H8a<RZZ Jr   )r   r   )r   r   r   s   r:   new_out_axes_thunkz2BatchTrace.process_map.<locals>.new_out_axes_thunk9  s-     J'*>+;XZ'HJ J Jr;   )r   r   r   )r   r   rH   r   r   r   r   rJ   r$   r   r
   r   r   r   r   r4   r3   )r|   map_primitiverk   r   r   r   r   new_in_axesnew_dimsr   
new_paramsr   r   r   	dims_out_r   r   r   r   r   s                    @@@r:   process_mapzBatchTrace.process_map  s   D..89JD$= 6D&"346 6K  6D&"346 6H !DHHdnnhGKAx,-N .1J 2J fkBTUJ			 1 1	2 <###A;;
;h< &)^5E%FH!a &h2x1}Q!K HI H

"
"
$C5895MNTQKaC(NN< <H Os   E1E=F1E:c          
        t        t        | j                  |            \  }}t        || j                  | j
                  |      \  }}t        || j                  | j
                  |      \  }}	|j                  | j                  ||g|t        |            }
t        j                  ||	      \  }}t        j                         }t        |
|      D cg c]  \  }}t        | |||       c}}S c c}}w )N)symbolic_zeros)r   r   rH   r   r   rJ   batch_custom_jvp_subtracer   r   r   lumerge_linear_auxr   r4   r   r3   )r|   primfunjvpr   r  in_valsin_dims	out_dims1	out_dims2out_valsfstout_dimsr   r   r   s                   r:   process_custom_jvp_callz"BatchTrace.process_custom_jvp_callE  s    c$"4"4g>?GW#C4>>7KNC.sDHHdnngVNC##D$5$5S7K77K$($GIH''	9=MC

"
"
$C5885LMTQKaC(MMMs   C1c          
        t        t        | j                  |            \  }D 	
cg c]  }	|	t        fD ]  }
|
  }}	}
t	        || j
                  | j                        \  }}t	        || j
                  | j                  |      \  }fd}t        || j
                  | j                  |      }|j                  | j                  |||ft        |      z   t        |            }t        j                  |      \  }}|s:        \  }}}|j                  t        d |D              z
  }t!        ||g      \  }}t#        j$                         }t'        ||      D 
cg c]  \  }}
t)        | ||
|       c}
}S c c}
}	w c c}
}w )Nc                             \  } } }t                      }|D cg c]  }|t        |      n|    }}g ||S c c}w r?   )iternext)rd   
input_fwdspruned_dimsrk   	full_dimsr
  r  	out_treess        r:   bwd_in_dimsz7BatchTrace.process_custom_vjp_call.<locals>.bwd_in_dimsW  sX    "aJ%kKUVa	4$wqzAViV'y';'' Ws   A)r  r  c              3  $   K   | ]  }|d u 
 y wr?   r   )r   rk   s     r:   r   z5BatchTrace.process_custom_vjp_call.<locals>.<genexpr>d  s     )LA!4-)L   )r   r   rH   r   r   r   rJ   batch_custom_vjp_bwdr   r   r   r   r  r  
num_leavessumr!   r   r4   r   r3   )r|   r  r  fwdbwdr   r  r  r	  in_dimr   fwd_in_dimsr  r  r  r  r  rd   res_treer  num_resr   r   r
  r  s         `                @@r:   process_custom_vjp_callz"BatchTrace.process_custom_vjp_callO  sh   c$"4"4g>?GW#*IVZ4HIq1I1IKI#C4>>7KNC#C4>>;ONC( sDHHdnnk7
SC##D$5$5%(#sOeGn$D$(9^$\^H ''	9=MC )a:##c)L)L&LLgx'3ka

"
"
$C5885LMTQKaC(MM+ J* Ns   F'F	)rk   lu.WrappedFun)r   r   r   r~   rH   r   r   r   r  r%  __classcell__)r   s   @r:   r<   r<      s+    94	P$OLNNr;   c                6    t        | ||      }t        |||      S r?   )_batch_inner_batch_outer)r  rJ   r
  out_dim_destsrk   s        r:   batchr,  k  s!     3	=1!	aG	,,r;   c                    t               }t        j                  d      5   | ||g| \  }}d d d        t        j                        5  ~d d d        S # 1 sw Y   *xY w# 1 sw Y   S xY w)Nvmap)r   r   transform_name_stackr
   ensure_no_leaks)rk   rJ   r
  r	  r   outsr5   s          r:   r*  r*  q  se    
#,,V4 ,C+7+KD%,E"--	+, ,-	+s   AA#A #A-c                   t        |      r |       n|}t        j                         5 }t        ||      t	        fd      }t        j
                  |      5  t        t        t        |      ||      }d d d        t        j
                        5  t        j                  j                  j                  fg      5  t        j                  j                        5  t        j                  j                        5   |  }	t        |      r |       n|}t        t        t         j                  j                        t#        t%        |	            |	|      }
d d d        d d d        d d d        d d d        d d d        
fS # 1 sw Y   xY w# 1 sw Y   :xY w# 1 sw Y   >xY w# 1 sw Y   BxY w# 1 sw Y   FxY w# 1 sw Y   
fS xY w)Nc                 j    t        t         j                        dt        j                               S rv   )r3   rW   r   r   r4   )rJ   r5   s   r:   <lambda>z_batch_inner.<locals>.<lambda>~  s(    +eYy~~-F&6&>&>&@B r;   )callabler
   take_current_tracer<   r&   r   r   r   r-   extend_axis_env_ndrK   r   add_spmd_axis_namesr   add_explicit_mesh_axis_namesr   r@   ranger0   )rk   rJ   r+  r   r
  r	  r   idx
in_tracersr1  r  r5   s    `         @r:   r)  r)  y  s   !'*GI'  <L|S)4E
 B CC				- Fwvuc2GWEjF 
 
 
' <

!
!INNINN#C"D
E<

"
"9#6#6
7< 
+
+I,H,H
I< 
^d)1-)@mommWXuinni>Z>Z[3t9%t]<h< < < <<  
5F F< < < < < < < <<  
5s   2GF:G-F? F3$ F'	AF"F'	*F32F?:GF	GF$ F'	'F0,F33F<8F??G	GGc                    t         dd       ddt        j                  fd       }t        |d d       } |t	        | |            S )Nc                v    || S t        | j                        }|||   |z  g|||dz    | j                  |      S )Nr   listr2   reshape)argrC   	tile_sizer2   s       r:   	tile_axiszvtile.<locals>.tile_axis  sF    |jOE#U4[I%=>E$tAv;;ur;   c                    || S t        | j                        }||   ||dz      z  g|||dz    | j                  |      S )Nr      r?  )outrC   r2   s      r:   untile_axiszvtile.<locals>.untile_axis  sL    |jOE+d1f56E$tAv;;ur;   c                    d t        |      D        }xs t        |d       }|J d        | t         |      |       }t        	|      S )Nc              3  F   K   | ]  \  }}|	|j                   |     y wr?   )r2   )r   r7   rD   s      r:   r   z.vtile.<locals>._map_to_tile.<locals>.<genexpr>  s      WDAqQWWQZWs   
!!zNo mapped arguments?)rC  )r    r  r   )
rk   rm   sizes
tile_size_outputs_flatin_axes_flatout_axes_flatrD  rC  rH  s
        r:   _map_to_tilezvtile.<locals>._map_to_tile  s^    WHY$EWE/d5$/J!9#99!c)j99lSTL{L-88r;   )rC   
int | None)r%   r  transformation2r   r,  )	f_flatrN  rO  rC  r   rP  rJ   rD  rH  s	    ```   @@r:   vtilerT    s^    
 	 	 9 9 y)T48)	eFI|]K	LLr;   c                   t        j                         5 }t        |||      }t        j                  |      5  t	        |      r |       n|}t        ||      D 	cg c])  \  }}	|	 t        |||	t        j                               n|+ }
}}	 | |
 }d d d        t        t        |j                              \  }}d d d        |j                         S c c}	}w # 1 sw Y   LxY w# 1 sw Y   .xY wr?   )r
   r6  r<   r   r5  r   r3   r   r4   r   r   rH   rj   )rk   rj   r   rJ   r
  r	  r   r5   r7   dimr<  r1  r  r  s                 r:   r   r     s      @L|S)4E				& %g.	Gg<?<QS28!Ss  q#/?/G/G/IJ,-. Sj S
^d	
  E$7$7 >?Hh@ ++h	/S @ @s4   #C)$C.C

C*C)CC&	"C))C2c                    t        |      D ]*  \  }}| t        |      k(  st        j                  |      c S  t        |      D ]*  \  }}| t        |      k(  st        j                  |      c S  J d       )NzCould not find segment lengths)	enumerater   peInDBIdxOutDBIdx)keyr	  r  ix	candidates        r:   _locate_valuer_    su     ) mb)
bmZZ^ !* mb)
bm[[_ 100r;   c                .    t        | |t        |            S r?   )_batch_jaxpr2r   )closed_jaxprrJ   r   s      r:   batch_jaxpr2rc    s    
 
|Yg	??r;   c                   t        j                  t        j                  |       | j                  j
                        }t        ||      \  }}t        |||      }g }t        | j                  |      D ]  \  }}|t        u r|j                  |        t        j                  |j                  |||j                        }|j                  Lt         j"                  j$                  r2|j'                  |j(                  t+        |j                        z        }|j                  |        t-        j.                  ||      \  }}	}
t        j0                  ||
       |       fS )N
debug_infor   )r  	wrap_initr
   jaxpr_as_funjaxprrf  _batch_jaxpr_inner_batch_jaxpr_outer
unsafe_zipin_avalsr   appendunmapped_avalr   r   r   r	   r   rx   r   r   r   rY  trace_to_jaxpr_dynamicClosedJaxpr)rb  rJ   r   rk   out_axes	avals_in2r}   b	jaxpr_outrd   constss              r:   ra  ra    s,    	ll4$$\2*00;;=!"1i0+!XIw/!)L117; 	gdAJt
..!T9#?#?Ad				(""Ii6I6I,J!JK$t	 221i@)Q			)V	,hj	88r;   c                j    t        |t              rt        |      n|}t        | |t        |      |      S r?   )r.   r@  r   _batch_jaxpr)rb  rJ   
in_batchedinstantiateinsts        r:   batch_jaxprr|    s-    )+t<{	+$	lIuZ/@$	GGr;   c                T   t        |t              s*t        |t        t        f      rt	        d |D              sJ t        |t              r|gt        | j                        z  }|D cg c]  }|rdnt         }}|D cg c]  }|rdnt         }}t        | |||      S c c}w c c}w )Nc              3  <   K   | ]  }t        |t                y wr?   )r.   bool)r   rt  s     r:   r   z_batch_jaxpr.<locals>.<genexpr>  s     7ajD!7s   r   )
r.   r  r@  r   r   r0   	out_avalsr   zero_if_mappedbatch_jaxpr_axes)rb  rJ   ry  rz  rt  r   r{  out_axes_dests           r:   rx  rx    s    
[$
'
[4-
0
7;7
79 8 T"-#l&<&<"==K-78!Q#8'8=HIT1.0I-I	,	7M	JJ 9Is   (B ?B%c                B    t        | |t        |      t        |            S r?   )_batch_jaxpr_axesr   )rb  rJ   r   r  s       r:   r  r    s    	<E'NE-DX	YYr;   c           
     
   t        j                  t        j                  |       | j                  j
                        }t        ||      \  }}t        ||||      \  }}t        |||      }t        | j                  |      D cg c];  \  }}|t        ur,t        j                  |j                  |||j                        n|= }	}}t        j                   ||	      \  }
}}t        j"                  |
|       |       fS c c}}w )Nre  )r  rg  r
   rh  ri  rf  rj  _match_axes_jaxprrk  rl  rm  r   ro  r   r   rY  rp  rq  )rb  rJ   r   r  rk   rr  out_batchedr}   rt  avals_inru  rd   rv  s                r:   r  r    s    	ll4$$\2*00;;=!"1i0+!X$Q	=(K.![Iw/! (2,2G2G'QS $D! *$   D!*!=!=?  S( S 221h?)Q			)V	,km	;;Ss   A C?c           
        t        j                         5 }t        |||      }t        ||      D 	cg c]  \  }}	|	t	        |||	      n| }
}}	t        j
                  |      5  t        j                  |j                  |j                  fg      5  t        j                  |j                        5  t        j                  |j                        5   | |
 }d d d        d d d        d d d        d d d        t        t        |j                              \  }}d d d        |j!                         S c c}	}w # 1 sw Y   dxY w# 1 sw Y   hxY w# 1 sw Y   lxY w# 1 sw Y   pxY w# 1 sw Y   RxY wr?   )r
   r6  r<   r   r3   r   r7  rK   r   r8  r   r9  r   r   r   rH   rj   )rk   rj   rJ   r   r   r	  r   r5   rN   rV  r<  r1  r  rr  s                 r:   rj  rj    s^     @L|S)4E"%gw"79c3 36/+eS#.sJ 9J 9 
 
 
' 

!
!INNINN#C"D
E

"
"9#6#6
7 
+
+I,H,H
I 
^d	   
  E$7$7 >?Hh@ ++h	/9       @ @s|   E+D5E+%-E E2 E	D;E	 E(E0*E+5E+;E E	EEEEE(	$E++E4c                    | ||g| } |       }t        ||      D 	
cg c]  \  }	}
|
t        u r|	t        u rd ndn|
 }}	}
t        |      t        |      k7  r|\  }|gt        |      z  }t	        t        t        |j                  |j                  |j                        |||      }|D 
cg c]  }
|
d u }}
|j                  |       |S c c}
}	w c c}
w rv   )rl  r  r   r0   r   r   rI   rK   r   r   rj   )rk   rj   rJ   r  rr  r5   r   r	  r  r   dstout_axis_destr  s                r:   r  r    s     ug(((Z( $.h#FHsC ^+  :-D1145 H- H 	3x=("NM"Oc(m3MINNINN"557=(4( -::SD:+:++k	/H ;s    C.Cc           
         t        |      r |       n|}t        ||      D cg c]6  \  }}t        |t              rt	        |t        j                  |            n|8 }}}t               } | ||g| S c c}}w r?   )r5  rl  r.   r/   r"   r1   ndimr   )rk   rJ   r
  r	  r7   axr   s          r:   rk  rk  +  s}    !'*GI'",Wg">@B 3=R2Er2771:. @' @
#	
3	"'	""@s   ;A6c                <    | |k(  r| S | t         u r|S |t         u r| S | S r?   r   )r7   ys     r:   _merge_bdimsr  3  s+    !VHJHJHHr;   c                      e Zd Zy)ZeroIfMappedN)r   r   r   r   r;   r:   r  r  =  s    r;   r  c                   |j                   }|j                  }t        j                         5 }t	        |||      }	t        ||dz        D 
cg c]Q  \  }
}||
nGt        |
      t        u r*t        t        j                  |||
j                              nt        |	|
|      S }}
}t        j                  |	      5   | | }d d d        d d d        t        t        	j                              \  }}t        |t!        |      dz  g      \  }}t        |t!        |      dz  g      \  }}t        t"        ||      }t        t%        t&        |	j(                  j*                  ||      |||      }t        t%        t,        |	j(                  j*                  ||      |||      }|j/                  |       ||z   S c c}}
w # 1 sw Y   xY w# 1 sw Y   xY w)NrF  )r   r   r
   r6  r<   r   r,   r   r   r}   r3   r   r   r   rH   r!   r0   r  r   rI   rJ   rK   _matchaxis_symzerosrj   )rk   rj   r   rJ   r
  r	  r   rE   r   r5   rN   rV  r<  out_tracersr  r  out_primalsout_tangentsout_primal_bdsout_tangent_bdss                       r:   r  r  B  s   	$**)  EL|S)4E #&gw{";= c3 ##Y,. t//c388DE4?sC4PQ =J = 
			& E67nkEE c%"5"5{CD(H(CMQ4F3GH+|$.x#h-1:L9M$N!./~?(WY(<(<dIN#X=+W0%//2F2FiX$h>,++h	|	##=E EE Es6    GAF6#G;F<G6G<G	GGc                     j                   j                  j                   fd}t        j                  | j
                        S )Nc                    t              r        n}t        | |      D cg c]B  \  }}t        |      t        u r*t        t	        j
                  	||j                              n|D } }}t        | |      D cg c]  \  }}t        |      t        u rd n| }}}t        
|      \  }}t        |	|      } |j                  |  S c c}}w c c}}w r?   )
r5  r   r,   r   r
   r   r}   r   _match_axes_and_sumcall_wrapped)argsin_dims_r7   rV  r   bwd_out_dims_thunkrJ   r   rA   r   r
  rE   r+  r   s          r:   new_bwdz%batch_custom_vjp_bwd.<locals>.new_bwd`  s    $W-wy7H dH-/3 Aw,& )))S!&&AB,-. /D /  h/1Aq Q</Q6 1H 1)#sIxHD.tY	9-}>D4d##/1s   AC>Cre  )r   rK   r   r  rg  rf  )	r   r   rJ   r
  r+  r  r   rA   rE   s	   ````` @@@r:   r  r  Y  sC     nn)nn)**)
$ 
$ 
g#..	99r;   c           	     V     | | }t        t        t        |||d       |       ||      S )NT	sum_match)r   r   r  )rk   rA   r   rE   r  r+  r	  r  s           r:   r  r  m  s7     [(	W()Y	#%}h
8 8r;   c           	     L   t        |t        t        f      r||k(  r|S t        |      t        |      cxk(  r	t        k(  rKn nHt        j                  |||j                        } t        |      t        j                  ||||            S |t        u r9|t        ur1 t        |      t        j                  |||j                  |            S |t        u r2|r0 t        |      t        j                  |||j                              S t        | |||f      t        | ||||||      S )Nr  )r.   r   r   r,   r/   r
   r   r}   ro  r   
ValueErrorrI   )r   szrE   r   r  r7   r  r}   s           r:   r  r  v  s     D,'(
czh	cd3i	&3	&b#qvv.dT!WT''CyABB	
	s*4T!WT''CCDD	
	yT!WT%%b#qvv677	1c3/00YIsCiPPr;   .zdict[core.Primitive, Callable]r   c                      e Zd Zd Zd Zy)AxisPrimitiveBatchersProxyc                "    fd}|t         |<   y )Nc                F     | j                   | j                  d ||fi |S r?   )r   rK   )rJ   r   r   r   batchers       r:   wrappedz7AxisPrimitiveBatchersProxy.__setitem__.<locals>.wrapped  s#    Y^^Y^^T4PPPr;   r   r|   r  r  r  s     ` r:   __setitem__z&AxisPrimitiveBatchersProxy.__setitem__  s    Q%,T"r;   c                    t         |   S r?   r  r|   r  s     r:   __getitem__z&AxisPrimitiveBatchersProxy.__getitem__      #D))r;   N)r   r   r   r  r  r   r;   r:   r  r    s    -
*r;   r  c                      e Zd Zd Zd Zd Zy)PrimitiveBatchersProxyc                &    fd}|t         <   y )Nc                    ~ t        d |D              r3 j                  |i |}j                  r|d gt        |      z  fS |d fS  ||fi |S )Nc              3  $   K   | ]  }|d u  
 y wr?   r   r   r   s     r:   r   zFPrimitiveBatchersProxy.__setitem__.<locals>.wrapped.<locals>.<genexpr>  s     %1Q$Y%r  )r   r   r   r0   )rJ   r   r   r   or  r  s        r:   r  z3PrimitiveBatchersProxy.__setitem__.<locals>.wrapped  sa    
	%%	%DIIt&v&'+'<'<D6CF?#K1d)KT4*6**r;   r  r  s    `` r:   r  z"PrimitiveBatchersProxy.__setitem__  s    + &-T"r;   c                    t         |= y r?   r  r  s     r:   __delitem__z"PrimitiveBatchersProxy.__delitem__  s
     &r;   c                    t         |   S r?   r  r  s     r:   r  z"PrimitiveBatchersProxy.__getitem__  r  r;   N)r   r   r   r  r  r  r   r;   r:   r  r    s    -'*r;   r  r   c                2    t        t        |       t        | <   y r?   )r   vectorized_batcherr   r  s    r:   defvectorizedr    s    #*+=t#D4 r;   c                    | j                   rJ t        d D              r | j                  |i |d fS t        fddd  D              sJ         | j                  |i |d   fS )Nc              3  $   K   | ]  }|d u  
 y wr?   r   r  s     r:   r   z%vectorized_batcher.<locals>.<genexpr>       'qd'r  c              3  .   K   | ]  }d    |k(    yw)r   Nr   )r   bd
batch_dimss     r:   r   z%vectorized_batcher.<locals>.<genexpr>  s     :RZ]b :s   r   r   )r   r   r   )r  rJ   batched_argsr  r   s      ` r:   r  r    sy    """	"'J''499l-f-t33	::ab>:	:FJF	:	L	+F	+Z]	::r;   c                2    t        t        |       t        | <   y r?   )r   broadcast_batcherr   r  s    r:   defbroadcastingr    s    #*+<d#C4 r;   c           	       
 t        |      dkD  sJ t        d |D              r3 | j                  |i |}| j                  r|d gt        |      z  fS |d fS t	        d t        ||      D              \  
t        
fdt        ||      D              r3 | j                  |i |}| j                  r|
ft        |      z  fS |
fS t        ||      D cg c])  \  }}t        j                  |      rt        ||d      n|+ }}}t        d |D              }	t        ||      D cg c]  \  }}t        |	||       }}} | j                  |i |}| j                  r|dt        |      z  fS |dfS c c}}w c c}}w )Nr   c              3  $   K   | ]  }|d u  
 y wr?   r   r  s     r:   r   z$broadcast_batcher.<locals>.<genexpr>  s     !qd!r  c              3  N   K   | ]  \  }}|t         ur|j                  |f  y wr?   )r   r2   )r   r7   r   s      r:   r   z$broadcast_batcher.<locals>.<genexpr>  s+      ,TQ
* WWaL ,s   #%c              3     K   | ]C  \  }}t        j                  |      r)t        j                  |j                        xr |k(   E y wr?   )r1   r  r
   definitely_equal_shaper2   )r   r7   r   rV  r2   s      r:   r   z$broadcast_batcher.<locals>.<genexpr>  sD      	4Q
 
	$	$UAGG	4	Ac	A 	4s   A	Ac              3  F   K   | ]  }t        j                  |        y wr?   )r1   r  )r   r7   s     r:   r   z$broadcast_batcher.<locals>.<genexpr>  s     (arwwqz(s   !r   r   )r0   r   r   r   r  r   r1   r  bdim_at_frontmax_handle_scalar_broadcasting)r  rJ   r  r   r   r  rG  r7   r   r  rV  r2   s             @@r:   r  r    s   	TQ!D!!		4"6"A#'#8#8AvAGq$iG ,St_ , ,*% 	4t_	4 4 $))T
$V
$C'+'<'<C##c("#L3*L D$)1 ')ggajM!Q"a7 )D )(4((D@CD$P1'a3PDP
$))T
$V
$C%)%:%:CC!HaH) Qs   .E6*E<c           
         ddl m} |t        u s| t        j                  |      k(  r|S |j                  |t        t        t        j                  |      |                   S )Nr   rQ   )rT   rR   r   r1   r  expand_dimsr   r:  )ndr7   r   rR   s       r:   r  r    sG     *_bggaj(H??1eE"''!*b$9:;;r;   c                4    t        t        | |      t        | <   y r?   )r   reducer_batcherr   )r  idents     r:   
defreducerr    s    #*?D%#H4 r;   c           
       
 t        d |D              r | j                  |d|i|d fS 
fd}|\  
|\  }t        |t              rt	        t        j                  t        j                  ||      |t        j                  |d                  } |||      }	d|v rt        |
j                        } | j                  
fd|i||	fS J )Nc              3  $   K   | ]  }|d u  
 y wr?   r   r  s     r:   r   z"reducer_batcher.<locals>.<genexpr>  r  r  axesc           	         t        t        t        j                  t        j                  j
                        |             j                  |            S r?   )r/   r@  r1   deletearanger  index)r  rC   operands     r:   r   z!reducer_batcher.<locals>.out_axis  s6    tBIIbii5t<=CCDIJJr;   r   input_shape)r  )r   r   r.   r/   r   r1   wherelessr[   r   r2   )r  r  rJ   r  r  r  r   r   rO   bdim_outr  s             @r:   r  r    s    'J''499l888$>>K('
%$c"''$-tRVVD!_EFDd#HF6f499W24262H<<5r;   c           	     >   t        ||      D ch c]  \  }}|t        us|j                  |    c}}\  }t        ||      D cg c]  \  }}t        |||       }}} | j                  |i |}| j
                  r|dt        |      z  fS |dfS c c}}w c c}}w )zdA batching rule for primitives that support matching leading batch
  dimensions in all arguments.
  r  r   )r   r   r2   r  r   r   r0   )r  r  r   r   r7   r  r   rG  s           r:   expand_dims_batcherr    s     %(dO
L51br7K1772;
L%$25dD/	BB-2t
$	B$	B		4"6"##'#8#8#tc#h	FsAhF M	Bs   BBBc           	     J   ddl m} t        t        j                  |             }|j                  ||       t        t        j                  t        j                  t        |            |            }t        j                  |       }|j                  j                  j                  rd }t        t!        |j                  j"                  ||       }|j                  j%                  |      }	t'        j(                  |	j                        5  |j+                  | |||	      } t,        j.                  j0                  rSt        j2                         j4                  }
t        |
      dkD  rt6        |
rt        j8                  | t        |
            } | cd d d        S # 1 sw Y   y xY w)Nr   rQ   )r8   )out_shardingr   )rT   rR   r@  r1   r2   insertr   r  r  r0   r
   rz   shardingr   r   r   r(   r8   r   r   use_abstract_meshbroadcast_in_dimr	   r   rx   get_axis_envspmd_axis_namesr   pvary)r7   r  rC   rE   rR   r2   broadcast_dimsx_avalr   r  
spmd_namess              r:   	broadcastr    s3   
rxx{
%,,tR299SZ#8$?@.==&__IV__114CD(__###2( !!(--0 	Q~HMA$$&66j	Z1	!!	JJq%
+,	 	 	s   BFF"c           	     `    t        | j                  | j                  | j                  ||||      S r?   )rI   rK   r   r   )rJ   r   r  r7   r  s        r:   
matchaxis2r    s,    	9>>9>>93O3OQ	
+ +r;   c           
     >   	 t        j                  |      }||k(  r|S t        |      t        |      cxk(  r	t        k(  rn nt        |||      S |t        u r6|t        ur.t        ||t        |t        j                  |      dz         |      S |t        u r|r|j                  |      S t        | t         j                        s#| t         j                  urt        d| d|       t!        d d d       # t        $ r!}t        d|dt        |       d      |d }~ww xY w)NzOutput from batched function z with type z is not a valid JAX typer   z"vmap has mapped output (axis_name=z) but out_axes is )r
   rz   	TypeErrorr,   r/   r#   r   r  r"   r1   r  r  r.   _TempAxisNameno_axis_namer  rL   )	r   r  rE   r   r  r7   r  rd   r   s	            r:   rI   rI     s   AaA 	CZHCyDI$$AsC  
jS
2Q-c2771:>BINN
jY55:y$"4"45***<)=OPSuUVV4t,,! 
 A
3A5Awi79 :?@AAs   C2 2	D;DDc                      e Zd Zd Zy)rL   c                .    || _         || _        || _        y r?   )leaf_idxr   r  )r|   r  r   r  s       r:   r~   zSpecMatchError.__init__4  s    DMDHDHr;   N)r   r   r   r~   r   r;   r:   rL   rL   3  s    r;   rL   c                J    |t         u rt        | |d|      S t        | |d      S )Nr   rE   )r   r  r#   )r7   rO   r   rE   s       r:   r  r  9  s*    	ZQa955AtQr;   c                R   |\  }}|\  }}| j                   }||k(  rt        ||      |fS |t        u r*t        ||j                  |   ||      }t        ||      |fS |t        u r*t        ||j                  |   ||      }t        ||      |fS t        |||      }t        ||      |fS )Nr  )r   r   r   r  r2   r#   )rJ   r  r  bdxbdyr7   r  rE   s           r:   add_batchedr  @  s    (#s	$!Q**)CZq!c!!
j!QWWS\3)<Aq!c!!
j!QWWS\3)<Aq!c!!CAq!c!!r;   c                     y)Nr   r   )rd   s    r:   r4  r4  Q  s    r;   )
r5   r   r6   GetIdxr7   	Vmappabler8   MapSpecreturnElt)r5   r<   rA   AxisSizerE   MeshAxisrD   r/   r7   r  r8   r  r  r  )rA   r  r  r   )r\   r,   r]   r,   r^   r,   r-   r   r@   r   rW   zCallable | None)r\   r,   r  None)r7   r   r  r  )rk   r   rj   zlu.Storerl   r   )r  r&  r  r&  )rk   r   )
rS  r&  rN  tuple[int | None, ...]rO  r  rC  rQ  r   r   )rb  core.ClosedJaxprr   ztuple[int | NotMapped, ...]r  z/tuple[core.ClosedJaxpr, tuple[int | NotMapped]])rb  r  r   ztuple[int | NotMapped]r  z4tuple[core.ClosedJaxpr, tuple[int | NotMapped, ...]])rb  r  rJ   r   r   Sequence[int]r  r  )r   r&  r   zcore.TraceTagrJ   r   r
  z"Callable[[], Sequence[int | None]]r+  zSequence[int | None]r  r&  )Fr?   )
__future__r   collections.abcr   r   dataclasses	functoolsr   typingr   r   numpyr1   jax._srcr	   r
   jax._src.corer   r   r   r  jax._src.partition_specr   r   jax._src.sharding_implsr   r   r   jax._src.ad_utilr   r   r   r   r   r   r   r   jax._src.interpretersr   rY  jax._src.tree_utilr   r   r   jax._src.typingr   jax._src.utilr   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r   
unsafe_mapr   rl  r  r  r  r  r  r  ToEltHandlerFromEltHandlerMakeIotaHandlerr-   r*   r   r@   rG   rW   rU   r_   rY   rb   rZ   re   rg   transformation_with_aux2rr   r,   ry   r   r3   	dataclassr   r   r<   r,  rR  r*  r)  rT  r   r_  rc  ra  r|  rx  r  r  rj  r  rk  r  r  r  r  r  r  r  r   r/   BatchingRuler   r  axis_primitive_batchersr  primitive_batchersr   r  r  r  r  r  r  r  r  r  r  rI   r   rL   r  r  ref_pr   r;   r:   <module>r1     s   # .         % & 6 1 % K K ; ; 4 H H !L L L L
 CZCZ
 		
	"f*	69g>CD8XsG<iGHH:u,-7 -/) .999#*9/89 13 - 2- 35 / 4?'?3;?"1? -/
) .
I 	  (3<  J	
3& 3j d#  $>5}N }NB-%2-    *M.M/M  M 	MB 
 
1@"@ )@ 8	@ 9"9 %9 <	9 9.HKZ <!)<,<=J< <     " # #   $ $,:$,:"D: )=: BO:( 8 8Q* 	#uS$eCI&6&; <<=
=>? <> 8 =* * 56 * *  ,-  79 3 8E;DI.<I"G0+-*Y  "  +6  '$0 = ! djj r;   