
    ukiM                       U d Z ddlmZ ddlmZmZ ddlZddlmZ ddl	Z	ddl
Z
ddlmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ddl!m"Z" ddl!m#Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? ddl@ZAe;eBcZBZCe<eDcZDZEej                  ZFdd	 	 	 	 	 	 	 dSdZG	 dT	 	 	 dUd ZHe?	 	 	 	 dVd!       ZIej                   G d" d#             ZK G d$ d%e      ZLi ZMd&eNd'<   dWd(ZO G d) d*e      ZPi ZQd+eNd,<   dWd-ZR	 	 	 	 	 	 	 	 dXd.ZSdYd/ZT	 	 	 	 dZd0ZU	 	 	 	 d[d1ZV	 	 	 	 d\d2ZW	 	 	 	 	 	 	 	 d]d3ZXd^d4ZY eOe)      	 	 	 	 d_d5       ZZdYd6Z[d7 Z\d8 Z]d9 Z^ eOe*      	 	 	 	 d_d:       Z_d; Z` eOe(      	 	 	 	 d_d<       Zad= Zbe?d`d>       Zc eOej                        	 	 	 	 dad?       Ze ej                  d@      Zgdeg_h        dA Zieieg_j        dB Zkekeg_l        dC Zm	 	 	 	 dbdDZnegj                  en        ej                  eg ej                  en             	 	 	 	 dcdEZregj                  er       	 	 	 	 	 	 dddFZtetej                  eg<    eOeg      	 	 	 	 	 	 	 	 	 	 dedG       Zv	 	 	 	 	 	 	 	 	 	 dfdHZwe?	 	 	 	 	 	 dgdI       Zx edJ      ZydhdKZzdidLZ{ eOej                        dM        Z} eOej                        dN        Z eej                   dOP      dQ        Zej                  dR        Zy)jz(Module for discharging state primitives.    )annotations)CallableSequenceN)partial)AnyProtocolTypeVar)ad_util)api_util)config)core)literals)linear_util)pjit)sharding_impls)source_info_util)	tree_util)custom_derivatives)ad)mlir)partial_eval)lax)slicing)indexing)addupdate_pget_pswap_ppinunpin)AbstractRefRefBitcaster	RefEffectRefReshaperget_ref_aval_from_valueuninitialized)bitcasthoist_consts_to_refs)Array)foreachsafe_mapsafe_zip
split_listunzip2weakref_lru_cacheTshould_dischargec                  t        |t              r|gt        | j                        z  }t	        | j                  |      D cg c]C  \  }}t        |j
                  t              r|r|j
                  j                  n|j
                  E }}}t        j                  t        t        | ||      | j                  j                               }t        j                  ||      \  }}}	||	fS c c}}w )a  Converts a stateful jaxpr into a pure one.

  Discharging replaces ``Ref`` inputs with regular values, threads updates
  through the computation, and returns updated ``Ref``s as additional outputs.

  Args:
    jaxpr: A stateful jaxpr with ``Ref`` inputs.
    consts: Constants for the jaxpr.
    should_discharge: Whether to discharge each ``Ref`` input. If a single bool,
      applies to all inputs.

  Returns:
    A tuple of ``(new_jaxpr, new_consts)`` where ``new_jaxpr`` is a jaxpr with
    no ``Read``/``Write``/``Accum`` effects. Discharged ``Ref`` inputs become
    regular value inputs, and their updated values are appended to the outputs.
  
debug_info)
isinstanceboolleninvarszipavalr    
inner_avallu	wrap_initr   _eval_jaxpr_discharge_stater3   with_unknown_namespetrace_to_jaxpr_dynamic)
jaxprconstsr0   vdin_avals
eval_jaxpr	new_jaxpr_
new_constss
             S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/state/discharge.pydischarge_staterK   <   s    ,  $'()C,== '*%,,8H&IK"a AFFK0Q ff66 K( K ||G$?$4f>','7'7'J'J'LN* !77
HM)Q	J	Ks   AC+c                ~    t        |t              r|ft        | j                        z  }t	        | t        |            S N)r4   r5   r6   rE   _discharge_state2tuple)rA   r0   s     rJ   discharge_state2rP   ^   s9      $'(*S-@@	5%(8"9	::    c                x    t        | j                  | j                  |      \  }}t        j                  ||      S )Nr/   )rK   rA   rB   r   ClosedJaxpr)rA   r0   jaxpr_rB   s       rJ   rN   rN   e   s5     #5;;4DF.&&			&&	))rQ   c                  (    e Zd ZU ded<   ddZddZy)Environmentzdict[core.Var, Any]envc                    t        |      t        j                  u r|j                  S t	        |t        j
                        sJ | j                  |   S rM   )typer   Literalvalr4   VarrW   )selfrC   s     rJ   readzEnvironment.readq   s=    Aw$,,UUla"""88A;rQ   c                "    || j                   |<   y rM   )rW   )r]   rC   r[   s      rJ   writezEnvironment.writew   s    DHHQKrQ   N)rC   z	core.Atomreturnr   )rC   zcore.Varr[   r   ra   None)__name__
__module____qualname____annotations__r^   r`    rQ   rJ   rV   rV   m   s    rQ   rV   c                  (    e Zd Z	 	 	 	 	 	 	 	 	 	 ddZy)DischargeRulec                     y)am  Discharge rule for a primitive.

    See :func:`discharge_state` for an explanation of what discharge means.

    Args:
      in_avals: Input abstract values.
      out_avals: Output abstract values.
      *args: Input values.
      **params: Primitive parameters.

    Returns:
      A tuple of ``(new_invals, new_outvals)`` where:

      * ``new_invals`` contains updated values for discharged ``Ref`` inputs,
        or ``None`` if the input is not a ``Ref`` or was not updated.
      * ``new_outvals`` is the primitive's output. A sequence if the primitive
        has multiple results, otherwise a single value.
    Nrg   )r]   rE   	out_avalsargsparamss        rJ   __call__zDischargeRule.__call__}   s    rQ   N)
rE   Sequence[core.AbstractValue]rk   ro   rl   r   rm   r   ra   0tuple[Sequence[Any | None], Any | Sequence[Any]])rc   rd   re   rn   rg   rQ   rJ   ri   ri   {   s7    , . 	
  8rQ   ri   z#dict[core.Primitive, DischargeRule]_discharge_rulesc                     d fd}|S )Nc                    | t         <   y rM   )rq   fprims    rJ   registerz)register_discharge_rule.<locals>.register   s    TrQ   )ru   ri   rg   rv   rw   s   ` rJ   register_discharge_rulery      s     
/rQ   c                  0    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 ddZy)PartialDischargeRuleaJ  Discharge rule that supports selective discharging of ``Ref`` inputs.

  Generalizes :class:`DischargeRule` by accepting a ``should_discharge``
  argument that specifies which ``Ref`` inputs to discharge. The returned
  ``new_invals`` must contain a non-``None`` value if and only if the
  corresponding ``Ref`` was discharged.
  c                     y rM   rg   )r]   r0   rE   rk   rl   rm   s         rJ   rn   zPartialDischargeRule.__call__   s     rQ   N)r0   Sequence[bool]rE   ro   rk   ro   rl   r   rm   r   ra   rp   )rc   rd   re   __doc__rn   rg   rQ   rJ   r{   r{      sF    & - .	
   8rQ   r{   z*dict[core.Primitive, PartialDischargeRule]_partial_discharge_rulesc                     d fd}|S )Nc                    | t         <   y rM   )r   rt   s    rJ   rw   z1register_partial_discharge_rule.<locals>.register   s    %&T"rQ   )ru   r{   rg   rx   s   ` rJ   register_partial_discharge_ruler      s    '	/rQ   c                
   t        i       }t        |j                  | j                  |       t        |j                  | j                  |       t        | j                  |      D ch c]6  \  }}|r/t        |j                  t              rt        |j                        8 }}}| j                  D ]  }t        j                         |j                  j                  z   }	|j                  j                  }
t        j                   |
|	      5  |j"                  j$                  5  |j                  D cg c]  }t        |j                        |v  }}|j&                  t(        j*                  u ry|j                  |j,                  c\  }\  }|j/                  |      }t0        j2                  j4                  rt7        |      }|j9                  t        |j                               n\|j&                  t(        j:                  u ry|j                  |j,                  c\  }\  }|j/                  |      }t0        j2                  j4                  rt=        |      }|j?                  t        |j                               ntA        |      st(        jB                  |jD                  v r8|j&                  tF        v rtI        tF        |j&                     |      }n>|j&                  tJ        v rtJ        |j&                     }ntM        d|j&                         tO        |j.                  |j                        }|j                  D cg c]  }|j                   }}|j,                  D cg c]  }|j                   }} |||g|i |jP                  \  }}t        |j                  ||      D ]/  \  }}}|
|stS        d|d|d      |j                  ||       1 ng|j&                  jU                  |jP                        \  }} |j&                  jV                  g |tO        |j.                  |j                        i |}d d d        d d d        |j&                  jX                  r#t        |j                  |j,                         |j                  |j,                  d           tO        |j.                  | j,                        }tO        |j.                  | j                  D cg c]  }t        |j                        |v s| c}      }||z   S c c}}w c c}w c c}w c c}w # 1 sw Y   xY w# 1 sw Y   xY wc c}w )N)
name_stackz3No state discharge rule implemented for primitive: z:Did not ask for inval to be discharged but it was. (invar=z, new_inval=)r   )-rV   r)   r`   	constvarsr7   r8   r4   r9   r    ideqnsr   current_name_stacksource_infor   	tracebackuser_contextctxmanager	primitiver   ref_poutvarsr^   r   refs_to_pinsvaluer   addfreeze_pr   removeanyinternal_mutable_array_effecteffectsr   r   rq   NotImplementedErrormaprm   
ValueErrorget_bind_paramsbindmultiple_results)rA   r0   rB   rl   rW   rC   rD   refs_to_dischargeeqnr   r   invaroutvaransruleinvalsrE   rk   
new_invalsshould	new_invalsubfunsbind_paramsout_valsref_valss                            rJ   r=   r=      sD    	B#	#))U__f- 
#))U\\4(.1%,,@P.Q Ada
166; ? !&&z A A ZZ 1%c!4469S9SSJ))I		&	&j
* */+.77??*/CF::Na"QVV*(99NN	$**	$JJ&hhuo$$C#bo.==DMM)JJ&hhuo$$c
#  EJJ0 D$F$F#++$U==44 '(@(OQa b$]]..!#--0$#CCMM?SU USXXszz*$'JJ/qAFF//%([[1QVV1	1i8"(8,/JJ8
C(+CJJ8H*(U 	($E69"O Q!\$  IIeY'	(  #}}<<SZZH cmm   /' /C#**,E /"-/S*/ */V }}%%ciic*	iiA$c1%j 5==)(	hhELLLqBqvvJ:K,KLN(	H	wA O, 013*/ */ */ */h Msh   ';TT;$T/3T 
G0T/T%
T/$T*
7<T/4BT/T;-U

U
 T//T84T;;U	c                    t        | j                  | j                        D ][  \  }}t        |t        j
                        s y|j                  s|j                  r y|j                  dk7  s|j                  |k7  s[ y y)z5Returns whether the indexer selects the entire shape.Fr   T)
r8   shapeindicesr4   r   Sliceis_dynamic_startis_dynamic_sizestartsize)indexersidxs      rJ   _is_trivial_indexerr     sf    GMM7??3 fac8>>*
s22
yyA~Q 
rQ   c                X   g }| j                   D ]  }t        |t        j                        s y |j                  }|j                  |j
                  dz
  |j                  z  z   dz   }|j                  }t        |t              rt        |t              s y |j                  |||f        |S )N   )	r   r4   r   r   r   r   strideintappend)r   rl   ir   endr   s         rJ   _maybe_convert_to_slicer     s     
$?? &aa(GGE
''QVVaZ188+
+a
/CXXF eS!C)=KKV$%& 
+rQ   c                   t        d | j                  D              sy | j                  D ].  }t        |t        j                        s|j
                  dkD  s. y  d t        fd| j                  D              }t        d | j                  D              }t        d t        | j                        D              }|||fS )Nc              3     K   | ]6  }t        |t        j                        xs t        j                  |        8 y wrM   )r4   r   r   npr   .0r   s     rJ   	<genexpr>z2_maybe_convert_to_dynamic_slice.<locals>.<genexpr>0  s5      ' 8>>*="((1+o= 's   <>r   c                T    t        j                  | t        j                  d            S )Nint32)r   convert_element_typer   dtype)xs    rJ   <lambda>z1_maybe_convert_to_dynamic_slice.<locals>.<lambda>9  s    333Arxx7HI rQ   c              3     K   | ]8  }t        |t        j                        r |j                        n |       : y wrM   )r4   r   r   r   )r   r   _convert_i32s     rJ   r   z2_maybe_convert_to_dynamic_slice.<locals>.<genexpr>:  s;         *!X^^<l177?s   >Ac              3  l   K   | ],  }t        |t        j                        r|j                  nd  . ywr   N)r4   r   r   r   r   s     rJ   r   z2_maybe_convert_to_dynamic_slice.<locals>.<genexpr>>  s+      9:
1hnn-aff14s   24c              3  Z   K   | ]#  \  }}t        |t        j                        s| % y wrM   )r4   r   r   )r   r   r   s      rJ   r   z2_maybe_convert_to_dynamic_slice.<locals>.<genexpr>A  s*      
!SX^^, s   )+)allr   r4   r   r   r   rO   	enumerate)r   r   startssizessqueeze_dimsr   s        @rJ   _maybe_convert_to_dynamic_slicer   (  s     
 'oo' 
' ?? a!X^^$A J, $+OO &  >Eoo %  goo. ,
 
	$$rQ   c           	     8   t        j                  |       \  }}}t        t        j                  t        j
                  t        j                  |      d         dk(              }|ry g }g }t        |      D ]  \  }}|r|n|j                  |        g ||}|S )Nr   r   )	r   unpack_ndindexerr5   r   r   diffwherer   r   )	r   is_int_indexingrH   int_indexers_contiguousint_indexer_idxsnon_int_indexer_idxsr   is_int_indextranspose_orders	            rJ   _maybe_transpose_before_gatherr   X  s     #33G</1a ffRWWRXXo.q12a78  "$&"?3 Koa%+?GGJK>&>)=>/	rQ   c                    | j                  |      }t        fd|D              }t        j                  |j                  j
                        }||fS )Nc              3  <   K   | ]  }j                   |     y wrM   )r   )r   r   r   s     rJ   r   z3_perform_transpose_before_gather.<locals>.<genexpr>q  s     H1GOOA.Hs   )r   r   int_indexer_shape)	transposerO   r   	NDIndexerr   r   )
target_arrr   r   new_target_arrreordered_indicesnew_indexers    `    rJ    _perform_transpose_before_gatherr   k  sV    
 ''8.HHH""MM11+
 
	$$rQ   c                   | j                         }t        j                  |       \  }}t              rXt	        | j
                        | j                  t        d D              dz
  d d d   j                  d      z
  }|z
  dz
  fd}g }t        | j
                        D ]7  \  }}t        |t        j                        r ||      }t        j                  t        j                  ||         |j                   z  |j"                  z   }	t	        |      |z
  dz
  }
|	j%                  |	j&                  d|
z  z         }	|j)                  |	       t        |t        j*                  t,        t.        j0                  f      r<dz
  z
  }
|j%                  |j&                  d|
z  z         }	|j)                  |	       t3        dt5        |      j6                          t9        |      S )Nc              3  &   K   | ]	  }|sd   ywr   rg   )r   ps     rJ   r   z,_convert_to_gather_arrays.<locals>.<genexpr>  s     9qq9s   r   Tc                v    t              s| S | z
  z
  k  r| S | z
  k  rt        | z
  t              z   S rM   )r   r   r6   )r   r   r   n_idxersn_int_indexersn_slice_index_dims_after_ints    rJ   get_idx_in_shape_after_indexingzB_convert_to_gather_arrays.<locals>.get_idx_in_shape_after_indexing  sR    h822^CCh8222~$5 666rQ   )r   zInvalid type of idxer: )get_indexer_shaper   r   r   r6   r   r   sumindexr   r4   r   r   iotar   r   r   r   reshaper   r   ndarrayr(   r   TypedNdArrayr   rY   rc   rO   )r   total_shaperH   last_int_index_idxr   arrsr   idxeridx_in_shape_after_indexingarrr   r   r   r   r   r   s              @@@@@rJ   _convert_to_gather_arraysr  z  s   ))++"33G</1a7??#H119O99N!A"(=(C(CD(II#+.@#@1#D 7 7 
$GOO, Iha%($CA$F!
((288[)DE
FLLKK 

  ;;a?dKK		D4K/0c
kk#	EBJJx/D/DE	F\..dMM%++t34c
kk#0e1E1E0FGHH!I$ 
trQ   c               F    ~ ~t        |||      }dt        |      dz   z  |fS )NrM   r   )_get_discharger6   )rE   rk   r   treer   ys         rJ   _get_discharge_ruler	    s/    
 	QT"!	CHqL	!1	$$rQ   c                T   t        |      r| S t        |      x}rt        j                  | gt	        |  } | S t        |      x}r5|\  }}}t        j                  | ||      }t        j                  ||      } | S t        |      }|t        | ||      \  } }t        |      }	| |	   } | S rM   )r   r   lax_slicingslicer8   r   dynamic_slicer   squeezer   r   r  )
r   r   maybe_slicemaybe_dynamic_slicer   r   r   r  r   arrayss
             rJ   _index_arrayr    s    !H ,G44[4!0c;/0A 
( >gFFF"5FE<!!!VU3AA|$A 
( 5W=O"3AwPja&w/F	&	A	
(rQ   c                (   |g }| }|D ]  }||xt         j                  d x\    t        ||      }+ xt        d x\    t	        ||j
                        }P t        d x\   |j                  |j                        }x 	 t        d|        |S )Nrg   Unsupported transform: )
r   r   r  r!   r&   r   r#   r   r   r   )r   
transformsresult	transforms       rJ   transform_arrayr    s    J& Ii
8fi0  <>1 =	0 !$;I;"GHHI 
-rQ   c                   |g }| }| g}|D ]5  }|xt         j                  d x\    |}t        |      r|j                  |d          >t	        |      x}r2|\  }}	}
t        j                  t        j                  |||	      |
      }n-t        |      }|t        |||      \  }}t        |      }||   }|j                  |        xt        d x)\    |j                  t        ||j                                t        d x.\   |j                  |j!                  |j"                               ( 	 t%        d|        |}t'        t)        |d d |            D ]  \  }}t+        |t         j                        r|}t        |      r.t	        |      x}r2|\  }}}
t        j,                  |t        j.                  ||
      |      }mt        |      }|t        |||      \  }}t        |      }|j0                  |   j3                  |      }|t5        j6                  |      }|j9                  |      }t%        d|        ||fS )Nrg   r   r  )r   r   r   r   r   r   r  r  r  r   r   r  r!   r&   r   r#   r   r   r   reversedr8   r4   dynamic_update_sliceexpand_dimsatsetr   argsortr   )r   r  r[   new_valintermediatesr  r   r  r   r   r   r   r  new_xintermediaterH   transpose_order_inverseds                    rJ   transform_swap_arrayr%    sx   J '#-   Ii
8w'


}R0
1
 :'BB;B(3
%&%KK''?' ;7C/(?/ GW -W5&FO' 	W%1  2 <>WWioo>? =W__Y__=> !$;I;"GHHA IH % "*#mCR.@**M!N GlI)X//0g	W	%7@	@	@"-<00#//%>
 9A&"BG_#
, +73'++E2&%'ZZ%@
"//":;%"9) EFF-G0 
%rQ   c                F    t        j                  ||      }t        | |      S rM   )r   tree_unflattenr  )r   r   r  r  s       rJ   r  r  &  s!    ''c2*	J	''rQ   c               R    ~ ~t        ||||      \  }}|d fdt        |      z  z   |fS NrM   )_swap_discharger6   )rE   rk   r   r[   r  r   zx_news           rJ   _swap_discharge_ruler-  *  s:    
 	QS$/(!U
3s8+	+Q	..rQ   c                H    t        j                  ||      }t        | ||      S rM   )r   r'  r%  )r   r[   r   r  r  s        rJ   r*  r*  3  s#    ''c2*	aS	11rQ   c               L    ~ ~t        ||||      }|d fdt        |      z  z   g fS r)  )_addupdate_discharger6   )rE   rk   r   r[   r  r   r   s          rJ   _addupdate_discharge_ruler1  7  s6    
 	QS$/#
twS)	)2	--rQ   c                0   t        j                  ||      }|s| |z   S t        |      dkD  rt        d      |d   }t	        |      r| |z   S t        |      x}rO|\  }}}	t        j                  | ||      }
t        j                  ||	      }t        j                  | |
|z   |      }|S t        |      }|t        | ||      \  } }t        |      }| j                  |   j                  |      } |&t!        j"                  |      }| j%                  |      } | S )Nr   z!Only single indexer is supported.r   )r   r'  r6   r   r   r   r  r  r   r  r  r   r   r  r  r   r   r  r   )r   r[   r   r  r  r   r  r   r   r   x_oldr  r   r  r$  s                  rJ   r0  r0  @  s   ''c2*	s7N_q
A
BBqM'!s7N
 4G<<[<"-FE<%%a7E
//#|
,C((ECK@AH27;/ 1!WoNJAw$W-&dd6ls! !zz/:	,-A	
(rQ   c                   | j                   | j                  }}t        |j                        }t	        ||      \  }}t        j                  ||      }t        j                  t        j                  |      |j                        }|||fS Nr2   )rA   rB   r6   r   rK   r   rS   r;   r<   jaxpr_as_funr3   )closed_jaxprrA   rB   num_outsdischarged_jaxprdischarged_constsdischarged_closed_jaxprfuns           rJ   _cached_closed_jaxpr_discharger=  `  s    $$l&9&9%((7v(F%% ,,-=?PQ
T&&'>? 0 ; ;	=#	 (C	//rQ   c                  t        |      \  }}}t        j                  j                  |g|d|i}t	        ||g      \  }}	t        |	      t        fd| D              }
t               }t        |      |u sJ |
|fS )N
call_jaxprc              3  X   K   | ]!  }t        |t              rt              nd  # y wrM   r4   r    nextr   r9   ref_vals_iters     rJ   r   z._closed_call_discharge_rule.<locals>.<genexpr>s  /      5#' -7t[,IT-( 5   '*)	r=  r   closed_call_pr   r,   iterrO   objectrB  )rE   rH   r?  rl   r;  r8  r<  out_and_ref_valsr   r   r   sentinelrD  s               @rJ   _closed_call_discharge_rulerL  j  s     ,J*+U(8S'',,S Q4 Q8OQ!"2XJ?(Hx.- 5+35 5*X(	mX	&(	22	2	X	rQ   	run_statec                    | j                   S rM   )is_high)rA   rH   __s      rJ   _run_state_is_highrQ  ~  s    	rQ   c                   | j                   rJ t        j                  | d      }t        t        j                  |      }t        d t        |||      D              \  }}t        j                  |      }t        j                  g |j                  ||j                  |d|}t        |t        j                  |       g      \  }}	t        j                  | ||       t        j                   ||	      S )Nrg   c              3     K   | ]?  \  }}}|j                   r|j                  |      n|j                  |      D ]  }||f 
 A y wrM   )has_qdd
read_loval	lower_val)r   ar   is_initlo_vals        rJ   r   z&_run_state_to_lojax.<locals>.<genexpr>  sR       H)Aq'()		Q\\!_q{{1~ H
 w H Hs   AA)rA   is_initialized)r   r   rS   r   typeofr-   r8   r?   lower_jaxprrun_state_pr   rB   rA   r,   num_himuts_outapply_himutraise_lo_outs)
rA   rZ  rl   rm   r7  	arg_avalslo_jaxprall_outsout_mutlo_outss
             rJ   _run_state_to_lojaxrf    s    __	!!%,,$++t$)  H-0D.-Q H H$ ^^L)( Gx G GX^^-;G?EG(2+<+<U+C*DE'7..g&			)W	--rQ   c                N   t        | d      sJ t        | d      sJ t        j                  |       }t        j                  |t        j                        r t        j
                  |      j                  }nt        j                  }t        j                  | j                  ||      S )Nr   r   )hasattrr   r   
issubdtypeintegeriinfominmathnanr   fullr   )r   r   r   s      rJ   _default_initializationrp    st    	G			G		
((1+%]]5"**%HHUOEHHE	!''5%	((rQ   c                    ~t        | d      \  }}t        |      t        fdt        ||j                        D              }t        j                  ||g| S )Nrg   c              3  h   K   | ])  \  }}|rt              nt        |j                         + y wrM   )rB  rp  r9   )r   rX  varargs_its      rJ   r   z"_run_state_impl.<locals>.<genexpr>  s2      
'3 d7m$;CHH$EEs   /2)rK   rH  rO   r8   r7   r   rF   )rA   which_linearrZ  rl   r9  rB   rt  s         @rJ   _run_state_implrv    sa     ,UB7FJ'	 n.>.E.EF 
$ 
)6	9D	99rQ   c                r   ~t        |      t        |      k(  sJ i }d}t        |      D ]  \  }}|s	 |||<   |dz  } t               }t        |      D 	ch c]  \  }}	t	        |	t
              s| }
}}	| j                  D ]u  }t	        |t              s|j                  |       %|j                  |vr4||j                     }||
v sH|j                  |j                  ||j                                  w t        | j                        t        |      k(  sJ t        |      st        ||fS c c}	}w )Nr   r   )input_index)r   r6   r   r  r4   r    r   r"   r   rx  replacer7   r   r   )rA   ru  rZ  avalsinner_to_outer_aval_mappingouter_ref_indexr   rX  nonlocal_effectsr9   is_refeffouter_indexs                rJ   _run_state_abstract_evalr    sI    	^	E
	**	*
 !#/n- ja
%4"qO	
 U&u-O'!TD+1NAO&O]] cc9%3
99  .coo>Kf
++"=coo"N+
O  
U\\	c.1	11	1	^	
	 	  ) Ps   D30D3c          	        t        |      st        d      |D cg c]  }t        |t        j                          }}t        |d      \  }}t        t        |            D ]S  }	t        j                  t        j                  ||      ||      \  }	}
|
|k(  r n(t        t        j                  ||
      }U t        d      ~~~
t!        ||      D cg c]  \  }}|rt        j"                  |      n|  }}}|D cg c]   }t%        |      t        j                  us|" }}t        j                  t'        j(                  |      |g       \  }}	|j*                  |j,                  }}t/        |      }g dt        |      z  |dt        |      z  }t1        j2                  g || |||dt        |j4                        z  d}t7        |t        |      t        |       g      \  }}}~t9        |      }t!        ||      D cg c]1  \  }}|rt;        |      nt        j                  j=                  |      3 }}}||fS c c}w c c}}w c c}w c c}}w )Nz,Uninitialized Refs are not supported in jvp.rg   )instantiatezInvalid fixpointFTrA   ru  rZ  )r   r   r4   r
   ZerorK   ranger6   r   	jvp_jaxprr   rS   r   operatoror_	Exceptionr8   instantiate_zerosrY   r?   close_jaxprrA   rB   r'   r]  r   r7   r,   rH  rB  from_primal_value)primalstangentsrA   ru  rZ  tnonzero_tangentsr9  body_constsrH   out_nonzero_tangentsinstclosed_jvp_jaxpr
jvp_jaxpr_
jvp_constsr  jvp_which_linearout
out_constsout_primalsout_tangentsout_tangents_iterr   nzs                           rJ   _run_state_jvpr    sy    
^	
L
MM?GH!*Q55HH"1%"<K%&' (a ll);7&68A //8<<)9;OP( &
''%9 +;<>!T *.b""1%14 >( >!AAT!WGLL%@aA(AR^^E%:%5r;A+113C3J3Jj*":.)[x#j/1[L[7SQY]CZ[ 	I* 	Iw 	I 	I&6(/#i6F6F2G(G	I# +5S3z?;>w<;I +J'*k<<(";0@ACa .0$()W\\5S5STU5VV C, C	l	""= I>ACs   "I#I? I I
6Ic                   t        |      st        d      ~t        j                  ||||d}g }t	        | |      D ](  \  }}	|j                  t        |t              r|	nd        * ||fS )Nz2Uninitialized Refs are not supported in discharge.r  )r   r   r]  r   r8   r   r4   r    )
rE   rk   rA   ru  rZ  rl   r   r   r9   out_vals
             rJ   _run_state_discharge_ruler    s     
^	
<  t5|-;=(*8X. JmdGD+!>gDIJ	X	rQ   c                0    t        | |t        |      |      S rM   )_initial_style_jaxprrO   )r<  in_treerE   dbgs       rJ   initial_style_jaxprr    s     
c7E(OS	AArQ   c                    t        j                  t        j                  | |      t	        j
                  |f            \  }}t        j                  ||      \  }}}|| |       fS r5  )r   flatten_fun_nokwargsr;   r<   r   treedef_tupler?   r@   )	r<  r  rE   debugfun_out_tree_thunkrA   rH   rB   s	            rJ   r  r    sb    
 "66ll35)wj)+$ ..tX>%F	(	((rQ   Tc                      fd}|S )Nc                   t        j                  d| fi       }t        j                  |       \  }}t	        t        t        |            \  }}t        |||      \  }}}t        |      }	dt        |      t        |      z   z  }
t        d |D              }t        d |D              }dt        |      z  |z   }t        j                  g |||	|
|d}t        |t        |      g      \  }}|j                  |      S )NrM  r  c              3  ,   K   | ]  }|t         u  y wrM   r%   r   rs     rJ   r   z-run_state.<locals>.wrapped.<locals>.<genexpr>'  s     I1 6Is   c              3  2   K   | ]  }|t         us|  y wrM   r  r  s     rJ   r   z-run_state.<locals>.wrapped.<locals>.<genexpr>(  s     DAQm-CaDs   r  r  )r   r3   r   tree_flattenr-   r   r$   r  r'   r6   rO   r]  r   r,   	unflatten)rl   r  	flat_argsr  	ref_avalsref_argsrT   rB   rH   rA   ru  refs_is_initialized	init_argsrZ  out_const_flatout_flatru   s                   rJ   wrappedzrun_state.<locals>.wrapped  s   


k1tgr
:C"//5Iw %<i!HIIx+Aw	3GFFA (Es6{S]:;LIIIDDDIs6{*-@@N %% Ev E	 E3?5CEN ^c&k];KAxX&&rQ   rg   ru   r  s   ` rJ   rM  rM    s    '" 
.rQ   c                      fd}|S )Nc                   t        j                  d| fi       }t        j                  |       \  }}t	        t        t        |            \  }}t        |||      \  }}}t        |      }	t        |	d      \  }
}t        ||      D cg c]  \  }}|t        u rt        |      n| }}}t        j                  |
|g|| }t        |t!        |      g      \  }}|j#                  |      S c c}}w )NrM  rg   )r   r3   r   r  r-   r   r$   r  r'   rK   r8   r%   rp  r   rF   r,   r6   r  )rl   r  r  r  r  r  rT   rB   rH   rA   r9  r:  r  r9   r  r  ru   s                   rJ   r  z$run_state_reference.<locals>.wrapped3  s	   


k1tgr
:C"//5Iw %<i!HIIx+Aw	3GFFA (E*9%*D''
 8Y/At *+m);%BH 
 __%57H 9&,9/79N^c&k];KAxX&&s   
C/rg   r  s   ` rJ   run_state_referencer  2  s    '$ 
.rQ   c          	        t        d |j                  D              s_t        d |j                  D              sCt        |      }	t	        j
                  j                  ||	||||d|}
d gt        |       z  }||
fS t        d g ||D              st        t        d |D              rt        d |D              st        t        |      }t        j                  ft        |j                        z  }t        j                  ft        |j                        z  }dt        |j                        z  }dt        |j                        z  }t	        j
                  j                  ||||||d|}t        |t        |j                        g      \  }}t        |      t        fd| D              }t!               }t#        |      |u sJ ||fS )	Nc              3  <   K   | ]  }t        |t                y wrM   )r4   r"   )r   es     rJ   r   z-_pjit_state_discharge_rule.<locals>.<genexpr>K  s     >1jI&>   c              3  <   K   | ]  }t        |t                y wrM   )r4   r    )r   rW  s     rJ   r   z-_pjit_state_discharge_rule.<locals>.<genexpr>L  s     DA{+Dr  )rA   in_shardingsout_shardings
in_layoutsout_layoutsc              3  P   K   | ]  }t        |t        j                           y wrM   )r4   r   UnspecifiedValue)r   r   s     rJ   r   z-_pjit_state_discharge_rule.<locals>.<genexpr>Z  s     eZ>::;es   $&c              3  $   K   | ]  }|d u  
 y wrM   rg   r   ls     rJ   r   z-_pjit_state_discharge_rule.<locals>.<genexpr>]  s     ,Aa4i,   c              3  $   K   | ]  }|d u  
 y wrM   rg   r  s     rJ   r   z-_pjit_state_discharge_rule.<locals>.<genexpr>^  s     -Aa4i-r  rM   c              3  X   K   | ]!  }t        |t              rt              nd  # y wrM   rA  rC  s     rJ   r   z-_pjit_state_discharge_rule.<locals>.<genexpr>l  rE  rF  )r   r   rE   rP   r   jit_pr   r6   r   r   r   UNSPECIFIEDrk   r,   rH  rO   rI  rB  )rE   rk   rA   r  r  r  r  rl   rm   rT   r  r   r9  new_in_shardingsnew_out_shardingsnew_in_layoutsnew_out_layoutsrJ  r   r   rK  rD  s                        @rJ   _pjit_state_discharge_ruler  G  s    >>
>DU^^DDe$F
**//	!# C #h-'Js?	eEd|EdVcEde	e

,,
,
--
-
%e,$002S9I9R9R5SS%113c:J:T:T6UUS!1!:!:;;.c"2"<"<==/ZZ__#2B%.!- &,- ""2S5I4JK(Hx.- 5+35 5*X(	mX	&(	22	2	X	rQ   c               J   ~~~~~t        |j                  |j                        \  }	}
t        |	|
|      }t	        |t        |j                        g      \  }}t        |      }| D cg c]  }t        |t              rt        |      nd ! }}t        |d       J ||fS c c}w rM   )rK   rA   rB   _eval_jaxpr_ad_errorr,   r6   rk   rH  r4   r    rB  )rE   rk   r?  fwd_jaxpr_thunkbwd	out_treessymbolic_zeros
num_constsrl   	dis_jaxpr
dis_constsoutsr   r   	ref_vals_r9   r   s                    rJ   custom_vjp_call_discharger  s  s    
 sI~z)**:*:J<M<MN)Z	iT	:$!$Z-A-A)B(CD(H8n)$& $.dK#@YdJ &* &	i		&&	&	X	&s   ($B )r   )nondiff_argnumsc                0    t        j                  | |g| S rM   )r   rF   )r  rB   rl   s      rJ   r  r    s    	F	2T	22rQ   c                     t        d      )Nz)should be unreachable, AD after discharge)r  )rH   s    rJ   _eval_jaxpr_ad_error_jvpr    s    =>>rQ   )rA   
core.JaxprrB   Sequence[Any]r0   bool | Sequence[bool]ra   z tuple[core.Jaxpr, Sequence[Any]]r  )rA   core.ClosedJaxprr0   r  ra   r  )rA   r  r0   tuple[bool, ...]ra   r  )rv   zcore.Primitive)rA   r  r0   r}   rB   r  rl   r   )r   indexing.NDIndexer)r   r  ra   z!list[tuple[int, int, int]] | None)r   r  ra   zOtuple[tuple[Array | int, ...], tuple[Array | int, ...], tuple[int, ...]] | None)r   r  ra   ztuple[int, ...] | None)r   r(   r   r  r   ztuple[int, ...]ra   z tuple[Array, indexing.NDIndexer])r   r  ra   ztuple[Array, ...])rE   ro   rk   ro   )r7  r  )rE   ro   r?  r  )rl   r   rA   r  ru  r  rZ  r  )rz  zcore.AbstractValuerA   r  ru  r  rZ  r  )
r  r  r  r  rA   r  ru  r  rZ  r  )rE   ro   rk   ro   rl   r   rA   r  ru  r}   rZ  r  )
r<  r   r  	PyTreeDefrE   ro   r  core.DebugInfora   z'tuple[core.Jaxpr, list[Any], PyTreeDef])r<  r   r  zapi_util.PyTreeDefrE   ro   r  r  )ru   Callable[..., None]ra   zCallable[[T], T])ru   r  )r~   
__future__r   collections.abcr   r   dataclasses	functoolsr   rm  r  typingr   r   r	   jax._srcr
   r   r   r   r   r   r;   r   r   r   r   r   jax._src.interpretersr   r   r   r?   jax._src.laxr   r   r  jax._src.stater   jax._src.state.primitivesr   r   r   r   r   jax._src.state.typesr    r!   r"   r#   r$   r%   jax._src.state.utilsr&   r'   jax._src.typingr(   jax._src.utilr)   r*   r+   r,   r-   r.   numpyr   r   
unsafe_mapr8   
unsafe_zipr  rK   rP   rN   	dataclassrV   ri   rq   rf   ry   r{   r   r   r=   r   r   r   r   r   r  r	  r  r  r%  r  r-  r*  r1  r0  r=  rG  rL  	Primitiver]  r   rQ  rO  rf  to_lojaxrp  rv  def_implregister_lowering	lower_funr  def_effectful_abstract_evalr  primitive_jvpsr  r  r  r  rM  r  r  r  custom_vjp_call_pr  
custom_jvpr  defjvpr  rg   rQ   rJ   <module>r     s   / " .     ) )      &  # %  ' $ & 4  / # L L  ? !. .  CZCZ	 /3	 ,	
 &F @D;'<;*; *(8*+* * 
 
 
H : 9; 5 :8 ( HJ D IEE)7EANEEN	&,%%%`&%%% %% &	%)X %*%+%  %.$JZ(  /*/+/ !/2 %.*.+. &.@ 0 0 ++,*  -  dnn[)# ( . + ):"2:$4:   _ %   {NDNN?$C D%!+;%!-=%!L  ' '(@ A##$##4D###3##H "0  + %)E%(1; -; /?	 &"B	B%B1MB	B /B )"4)#?) !/) ) CL(* $( %(V +==> ? 			&	&=3 >3? ?rQ   