
    uki                       d dl mZ d dlmZmZ d dlmZ d dlZd dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z" d dl m#Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5 d dl6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>  ej~                  e@        ej~                  e@       e;ZAe<ZB ej                  eD      ZEdSdZFdSdZGdSd ZHeHZIdSd!ZJd" ZK ej                  d#      ZMd$ ZNd% ZOd& ZPd' ZQd( ZR e
j                  eFeGeHeHeJeJeKeNeOePeReQ)      ZT ee,d*+      d,dd- e5       d.	 	 	 	 	 	 	 dTd/       ZUd,dd- e5       d.	 	 	 	 	 	 	 dTd0ZVd1 ZW G d2 d3      ZX	 	 	 	 dUd4ZY	 	 	 	 dUd5ZZ e>eZ      Z[e>	 	 	 	 	 	 	 	 dVd6       Z\	 	 dWd7Z]	 	 	 	 dXd9Z^d: Z_ ej                  d;      Z`d,e`_a        d< Zbebe`_c        e`j                  d=        Zee`j                  d>        Zgd? Zhehej                  e`<   	 	 dYd@Zjejej                  e`<   e>dZdA       ZldB Zm eej                  d8em      ej                  e`<   dC Zpepej                  e`<   	 	 	 	 d[dDZre>	 	 	 	 d\dE       ZsdF Ztetej                  e`<   	 	 d]dGZvevej                  e`<   dSdHZx	 	 	 	 	 	 d^dIZyd_dJZz	 	 	 	 	 	 	 	 d`dKZ{ ej                  e`e{       dL Z}eMj                  dM        eMj                  dN        dO Zeej                  eM<    ej                  eMdP        dQ Zeej                  eM<    e(j                  e`      dR        Zy)a    )annotations)CallableSequence)partialN)Any)ad_util)api)config)core)deprecations)dtypes)linear_util)effects)source_info_util)traceback_util)api_util)ad)batching)mlir)partial_eval)lax)convolution)hlo)	discharge)AbstractRef)api_boundary)	PyTreeDeftree_flattentree_unflattentree_structurebroadcast_prefixtree_map)DeprecatedArg)unzip2wraps
split_listpartition_listsafe_mapsafe_zipmerge_listsweakref_lru_cachec                      y)zThe default strategy, as if ``jax.checkpoint`` were not being used at all.

  This is the effective policy without any use of jax.remat.T ___s     Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/ad_checkpoint.pyeverything_saveabler2   ?   s     
    c                      y)zRematerialize everything, as if a custom policy were not being used at all.

  This is the effective policy when using jax.remat without explicit policy.Fr-   r.   s     r1   nothing_saveabler5   E   s     
r3   c                h    | t         j                  t        j                  hv xs | j                  dk(  S )Nscaled_matmul_wrapper)lax_internaldot_general_plax_convolutionconv_general_dilated_pname)primr/   r0   s      r1   dots_saveabler>   K   s9     <--!88: : /
)).
.0r3   c                    | t         j                  u r|d   \  \  }}\  }}|s|sy| j                  dk(  r|d   }|j                  d   dk(  S y)z,This is a useful heuristic for transformers.dimension_numbersTr7   r      F)r8   r9   r<   shape)r=   argsparamsr/   lhs_brhs_blhss          r1    dots_with_no_batch_dims_saveablerH   T   s^    	\'''#$78FQNUE 
YY))
q'C99Q<1	r3   c                      fd}|S )zSame as ``dots_with_no_batch_dims_saveable``, but offload to CPU memory
  instead of recomputing.

  This is a useful heuristic for transformers.c                    | t         j                  u r)|d   \  \  }}\  }}|s|st        j                        S t        j                  S )Nr@   srcdst)r8   r9   peOffloadable	Recompute)r=   r/   rD   rE   rF   offload_dstoffload_srcs        r1   policyz.offload_dot_with_no_batch_dims.<locals>.policyi   sI    |)))%&9:fq!nue5~~+;??<<r3   r-   )rR   rQ   rS   s   `` r1   offload_dot_with_no_batch_dimsrT   d   s    
 
-r3   r<   c                 (     t                 fd}|S )z@Save any values (not just named ones) excluding the names given.c                $    | t         u r|d   vS y)Nr<   Tname_pr=   r/   rD   names_not_to_saves      r1   rS   z0save_anything_except_these_names.<locals>.policyw   s    v~F^#444r3   	frozensetrZ   rS   s   ` r1    save_anything_except_these_namesr^   t   s     12 
-r3   c                 (     t                 fd}|S )z[Save only named values, i.e. any outputs of `checkpoint_name`, excluding
  the names given.c                $    | t         u r|d   vS yNr<   FrW   rY   s      r1   rS   z(save_any_names_but_these.<locals>.policy   s    v~F^#444r3   r[   r]   s   ` r1   save_any_names_but_theserb   }   s       12 
-r3   c                 (     t                 fd}|S )z7Save only named values, and only among the names given.c                $    | t         u r|d   v S yra   rW   )r=   r/   rD   names_which_can_be_saveds      r1   rS   z%save_only_these_names.<locals>.policy   s    v~F^777r3   )set)re   rS   s   ` r1   save_only_these_namesrg      s     !9: 
-r3   c                     t                t               j                        }|rt        d  d d|        fd}|S )zVSame as ``save_only_these_names``, but offload to CPU memory instead of
  recomputing.zThe names should be exclusive and should not intersect in `names_which_can_be_saved` and `names_which_can_be_offloaded`. Got names_which_can_be_saved=z, names_which_can_be_offloaded=z and the intersection=c                    | t         u r|d   v rt        j                  S | t         u r|d   v rt        j                        S t        j                  S )Nr<   rK   )rX   rN   SaveablerO   rP   )r=   r/   rD   names_which_can_be_offloadedre   rQ   rR   s      r1   rS   z1save_and_offload_only_these_names.<locals>.policy   sN    v~&.,DD[[v~&.,HH^^==<<r3   )rf   intersection
ValueError)re   rk   rR   rQ   rl   rS   s   ````  r1   !save_and_offload_only_these_namesrn      sp    
 !!9:!$%A!B)667ST,
	%%=$> ?))E(F G%		() ) 
-r3   c                      fd}|S )zjLogical OR of the given policies.

  A residual is saveable iff it is saveable according to either policy.c                     | g|i |} | g|i |}t        |t              rt        |t              st        d| d| d      |xs |S )Nz;The return value of the policies should be a boolean. Got: z and zY. Please write a custom policy function directly, rather than using this helper function.)
isinstanceboolrm   )r=   rC   rD   out1out2policy_1policy_2s        r1   rS   z'save_from_both_policies.<locals>.policy   sq    D*4*6*DD*4*6*DtT"z$'=V5 556 6 <4r3   r-   )ru   rv   rS   s   `` r1   save_from_both_policiesrw      s     
-r3   )r2   r5   r>   checkpoint_dotsrH   "checkpoint_dots_with_no_batch_dimsrT   r^   rb   rg   rw   rn   zjax.checkpoint)repro_api_nameTr-   prevent_cserS   static_argnumsconcretec               p    t        |t              s'd}t        j                  d|d       |rt	        |      t        t
              rft        t              rt              t        t        t        f      st        dt                    t               t         fd              }|S )aP  Make ``fun`` recompute internal linearization points when differentiated.

  The :func:`jax.checkpoint` decorator, aliased to :func:`jax.remat`, provides a
  way to trade off computation time and memory cost in the context of automatic
  differentiation, especially with reverse-mode autodiff like :func:`jax.grad`
  and :func:`jax.vjp` but also with :func:`jax.linearize`.

  When differentiating a function in reverse-mode, by default all the
  linearization points (e.g. inputs to elementwise nonlinear primitive
  operations) are stored when evaluating the forward pass so that they can be
  reused on the backward pass. This evaluation strategy can lead to a high
  memory cost, or even to poor performance on hardware accelerators where memory
  access is much more expensive than FLOPs.

  An alternative evaluation strategy is for some of the linearization points to
  be recomputed (i.e. rematerialized) rather than stored. This approach can
  reduce memory usage at the cost of increased computation.

  This function decorator produces a new version of ``fun`` which follows
  the rematerialization strategy rather than the default store-everything
  strategy. That is, it returns a new version of ``fun`` which, when
  differentiated, doesn't store any of its intermediate linearization points.
  Instead, these linearization points are recomputed from the function's saved
  inputs.

  See the examples below.

  Args:
    fun: Function for which the autodiff evaluation strategy is to be changed
      from the default of storing all intermediate linearization points to
      recomputing them. Its arguments and return value should be arrays,
      scalars, or (nested) standard Python containers (tuple/list/dict) thereof.
    prevent_cse: Optional, boolean keyword-only argument indicating whether to
      prevent common subexpression elimination (CSE) optimizations in the HLO
      generated from differentiation. This CSE prevention has costs because it
      can foil other optimizations, and because it can incur high overheads on
      some backends, especially GPU. The default is True because otherwise,
      under a :func:`~jax.jit` or :func:`~jax.pmap`, CSE can defeat the purpose
      of this decorator.
      But in some settings, like when used inside a :func:`~jax.lax.scan`, this
      CSE prevention mechanism is unnecessary, in which case ``prevent_cse`` can
      be set to False.
    static_argnums: Optional, int or sequence of ints, a keyword-only argument
      indicating which argument values on which to specialize for tracing and
      caching purposes. Specifying arguments as static can avoid
      ConcretizationTypeErrors when tracing, but at the cost of more retracing
      overheads. See the example below.
    policy: Optional, callable keyword-only argument. It should be one of the
      attributes of ``jax.checkpoint_policies``. The callable takes as input a
      type-level specification of a first-order primitive application and
      returns a boolean indicating whether the corresponding output value(s) can
      be saved as residuals (or instead must be recomputed in the (co)tangent
      computation if needed).
    concrete: Optional boolean; deprecated. Will raise a DeprecationWarning if
      used, and passing True will result in a NotImplementedError.

  Returns:
    A function (callable) with the same input/output behavior as ``fun`` but
    which, when differentiated using e.g. :func:`jax.grad`, :func:`jax.vjp`, or
    :func:`jax.linearize`, recomputes rather than stores intermediate
    linearization points, thus potentially saving memory at the cost of extra
    computation.

  Here is a simple example:

  >>> import jax
  >>> import jax.numpy as jnp

  >>> @jax.checkpoint
  ... def g(x):
  ...   y = jnp.sin(x)
  ...   z = jnp.sin(y)
  ...   return z
  ...
  >>> jax.value_and_grad(g)(2.0)
  (Array(0.78907233, dtype=float32, weak_type=True), Array(-0.2556391, dtype=float32, weak_type=True))

  Here, the same value is produced whether or not the :func:`jax.checkpoint`
  decorator is present. When the decorator is not present, the values
  ``jnp.cos(2.0)`` and ``jnp.cos(jnp.sin(2.0))`` are computed on the forward
  pass and are stored for use in the backward pass, because they are needed
  on the backward pass and depend only on the primal inputs. When using
  :func:`jax.checkpoint`, the forward pass will compute only the primal outputs
  and only the primal inputs (``2.0``) will be stored for the backward pass.
  At that time, the value ``jnp.sin(2.0)`` is recomputed, along with the values
  ``jnp.cos(2.0)`` and ``jnp.cos(jnp.sin(2.0))``.

  While :func:`jax.checkpoint` controls what values are stored from the
  forward-pass to be used on the backward pass, the total amount of memory
  required to evaluate a function or its VJP depends on many additional internal
  details of that function. Those details include which numerical primitives are
  used, how they're composed, where jit and control flow primitives like scan
  are used, and other factors.

  The :func:`jax.checkpoint` decorator can be applied recursively to express
  sophisticated autodiff rematerialization strategies. For example:

  >>> def recursive_checkpoint(funs):
  ...   if len(funs) == 1:
  ...     return funs[0]
  ...   elif len(funs) == 2:
  ...     f1, f2 = funs
  ...     return lambda x: f1(f2(x))
  ...   else:
  ...     f1 = recursive_checkpoint(funs[:len(funs)//2])
  ...     f2 = recursive_checkpoint(funs[len(funs)//2:])
  ...     return lambda x: f1(jax.checkpoint(f2)(x))
  ...

  If ``fun`` involves Python control flow that depends on argument values,
  it may be necessary to use the ``static_argnums`` parameter. For example,
  consider a boolean flag argument::

    from functools import partial

    @partial(jax.checkpoint, static_argnums=(1,))
    def foo(x, is_training):
      if is_training:
        ...
      else:
        ...

  Here, the use of ``static_argnums`` allows the ``if`` statement's condition
  to depends on the value of ``is_training``. The cost to using
  ``static_argnums`` is that it introduces re-tracing overheads across calls:
  in the example, ``foo`` is re-traced every time it is called with a new value
  of ``is_training``. In some situations, ``jax.ensure_compile_time_eval``
  is needed as well::

    @partial(jax.checkpoint, static_argnums=(1,))
    def foo(x, y):
      with jax.ensure_compile_time_eval():
        y_pos = y > 0
      if y_pos:
        ...
      else:
        ...

  As an alternative to using ``static_argnums`` (and
  ``jax.ensure_compile_time_eval``), it may be easier to compute some values
  outside the :func:`jax.checkpoint`-decorated function and then close over them.
  zThe `concrete` option to `jax.checkpoint` has been deprecated. In its place please use `static_argnums`; for details refer to https://docs.jax.dev/en/latest/jep/11830-new-remat-checkpoint.html.zjax-checkpoint-concrete   )
stacklevelzDprevent_cse must be a bool or tuple of bools, got type(prevent_cse)=c                    t        j                  d| |      }t        |       \  }} t        | |f      \  }}|D cg c]  }t	        j
                  |       }}t        ||t        |      |      \  }}	}
t        t              r?|rt        |       |fn
t        |       }dt        |	      z  t        t        |            z   }n}t        j                  g |	|||dd}t        |
|      S c c}w )Nzcheckpoint / remat)r}   )FFjaxprr|   differentiatedrS   )r   
debug_info_remat_static_argnumsr   r   shaped_abstractify_trace_to_jaxprtuplerq   lenr!   remat_pbindr   )rC   kwargsdebugfun_	args_flatin_treexin_avalsr   constsout_treecse_argscseout_flatfunrS   r|   r}   s                 r1   	fun_rematzcheckpoint.<locals>.fun_rematp  s    cf^5E 'sNDAJD$%tVn5Iw4=>q''*>H>-dGU8_eTE68+u%*0%+v&eDkhs6{"U+;K+R%SScc|| 	#(c%H (H-- ?s    C3)rq   r#   r   warnNotImplementedErrorintlistr   rr   	TypeErrortyper%   r   )r   r|   rS   r}   r~   concrete_msgr   s   ````   r1   
checkpointr      s    f 
Hm	,	O 
 /!L--$#_NT"$K	K%	/
 *K(*, - - 	:.  ." 
r3   c               "    t        | ||||      S )z Alias of :func:`jax.checkpoint`.r{   )r   )r   r|   rS   r}   r~   s        r1   rematr     s    
 
C[#1H
F Fr3   c                   t        |      t        u r|f}n1t        |      t        u rt        d |D              st	        d|       t        fd|D              st        d| dt                     |s| fS t              }t        fd|D              }g g }}t              D ]5  \  }}||v r|j                  t        |             %|j                  |       7 t        | |t        |      |      }	|	|fS )Nc              3  >   K   | ]  }t        |      t        u   y wN)r   r   ).0ds     r1   	<genexpr>z(_remat_static_argnums.<locals>.<genexpr>  s     71Q37s   zvthe `static_argnums` argument to `jax.checkpoint` / `jax.remat` must be an int, tuple of ints or, bool, but got value c              3  d   K   | ]'  }t               |cxk  xr t              k  nc  ) y wr   r   r   r   rC   s     r1   r   z(_remat_static_argnums.<locals>.<genexpr>  s(     AQc$iZ1(s4y((As   -0zthe `static_argnums` argument to `jax.checkpoint` / `jax.remat` can only take integer values greater than or equal to `-len(args)` and less than `len(args)`, but got z, while `len(args)` = c              3  :   K   | ]  }|t              z    y wr   r   r   s     r1   r   z(_remat_static_argnums.<locals>.<genexpr>  s     Da#d)mDs   )r   r   r   allr   rm   r   r\   	enumerateappendWrapHashably_dyn_args_fun)
r   r}   rC   nargsstatic_argnums_dyn_argsstatic_argsir   new_funs
     `       r1   r   r     s   	.S $&N E)777
 !!/ 02 3 3 
A.A	A
 Q '''=c$i[J K K
 
9
d)%D^DD/bK(o daO[//Q@
//!
 #k0BEJ'	(	r3   c                  >    e Zd ZU ded<   ded<   ded<   d Zd Zd	 Zy
)r   r   valr   hashrr   hashablec                ~    || _         	 t        |      | _        d| _        y #  t        |      | _        d| _        Y y xY w)NTF)r   r   r   id)selfr   s     r1   __init__zWrapHashably.__init__  s9    DHs)didmS'didms   ! <c                    | j                   S r   )r   )r   s    r1   __hash__zWrapHashably.__hash__  s    99r3   c                    t        |t              rI| j                  r%|j                  r| j                  |j                  k(  S | j                  |j                  u S y)NF)rq   r   r   r   )r   others     r1   __eq__zWrapHashably.__eq__  sB    %&	5>>xx599$$xx599$$r3   N)__name__
__module____qualname____annotations__r   r   r   r-   r3   r1   r   r     s     
().r3   r   c                ^    t        d |D              rt        | |||      S t        | |||      S )Nc              3  d   K   | ](  }t        |j                  t        j                         * y wr   )rq   r   r   Tracer)r   r   s     r1   r   z _dyn_args_fun.<locals>.<genexpr>  s     =AAEE4;;	'=s   .0)any_dyn_args_fun_uncached_dyn_args_fun_cached)r   r}   r   r   s       r1   r   r     s2    ===!#~{EJJ	c>;	FFr3   c                      fd}|S )Nc                     t        	      t        |       }}t              D cg c]&  }|v rt        |      j                  n
t        |      ( }} |i |S c c}w r   )iterrangenextr   )
r   r   static_args_	dyn_args_r   	full_argsr   r   r}   r   s
         r1   r   z'_dyn_args_fun_uncached.<locals>.new_fun  si    ";/h)L/4U|=*+ ,-+>l#''9o& =I =	$V$$=s   +Ar-   )r   r}   r   r   r   s   ```` r1   r   r     s    %
 
.r3   c                D   t        j                  t        j                  | |      |      \  }}	 t	        j
                  ||      \  }}}t	        j                  |      | |       fS # t        j                  $ r%}	|	j                  \  }
d|
v r|
dz  }
|
f|	_         d }	~	ww xY w)Nr   zfor checkpointz

Consider using the `static_argnums` parameter for `jax.remat` or `jax.checkpoint`. See the `jax.checkpoint` docstring and its example involving `static_argnums`:
https://docs.jax.dev/en/latest/_autosummary/jax.checkpoint.html
)
r   flatten_funlu	wrap_initrN   trace_to_jaxpr_dynamicr   ConcretizationTypeErrorrC   convert_constvars_jaxpr)r   r   r   r   flat_funr   r   r/   r   emsgs              r1   r   r     s      ++BLL,OQXY(H
008DE1f 
	#	#E	*FHJ	>> 
	%	% 

66DC3	  c taf	

s   A' 'B: BBc                J    t        ||f      \  } fdt        j                  d ||      } t        j                  fdd      | }t        |t              sJ |\  }}|j                  }|d   }	t        |	      j                  }
|j                  |j                  t        |j                        |
z
  d  |j                  d             }t        |j                        t        |      k(  sJ t        ||j                   xs d	t        |j                        z        S )
Nc                 2    t        |       \  } } | i |S r   )r   )rC   r   fr   s     r1   f_zsaved_residuals.<locals>.f_  s#    !'40LD&dfr3   saved_residualsc                 0    t        j                  g|  S r   )r	   	linearize)rC   r   s    r1   <lambda>z!saved_residuals.<locals>.<lambda>  s    S]]2%=%= r3   T)return_shaperA   )result_paths)outvarsr   )unknown)r   r   r   r	   
make_jaxprrq   r   r   r    
num_leavesreplacer   r   _replaceinvars_saved_residuals	arg_names)r   rC   r   	in_leavesr   outjaxpr_
out_shape_r   	out_shapenum_resr   r   s   `          @@r1   r   r     s   #T6N3)W ""#4avF*	*=$(	*+4	6#	C		&*
,,%m)9%00'
--mmC.89:$$$$7  9% 
U\\	c)n	,,	,	%!5!5!YELLHY9Y	ZZr3   r   c           	     :   | j                   D cg c]  }t        |t        j                        s|! }}| j                   D ch c]  }t        |t        j                        r|! }}g }|D ]  }|j	                  |j
                  df       ! | j                  D ]$  }||v s|j	                  |j
                  df       & t        | j                        D ]7  \  }}||v s||   r	d||    }nd}|j	                  |j
                  |f       9 | j                  D 	ci c])  }	|	j                  t        u s|	j                  D ]  }||	 + }
}	}| j                  D ]  }t        j                  |j                        }|j                   D ]  }||v s|j                  t        u s||
v r8|
|   x}r1|j	                  |j
                  d|j                  d    d| f       V|j                  j                   dk(  r1|j	                  |j
                  d	|j                  d    d| f       |j	                  |j
                  d
|j                  j                    d| f        
 t#        |      t#        | j                         k(  sJ |S c c}w c c}w c c}}	w )Nzfrom a literalzfrom a constantzfrom the argument z(from the argument at flattened index {i}znamed 'r<   z' from jitzoutput of jitted function 'z
output of z from )r   rq   r   Literalr   aval	constvarsr   r   eqns	primitiverX   r   	summarizesource_inforD   r<   r   )r   r   r   res_litsres_varsresultsvr   rL   r   
named_varseqns               r1   r   r   	  s   JAjDLL.IaJ(JJAjDLL.IaJ(J' /aNNAFF,-./ ?? 2aH}nnaff/012 % $daH}	1"9Q<.18nnaffc]#$ $jj #AKK6,A# 1 # #* # ZZ Qc

$
$S__
5C[[ 	Q	
h==F"a:o*Q-;O3;O
..!&&GCJJv,>+?wse"LM
N]]5(
..!&&7

68J7K L""%() * ..!&&Js}}/A/A.B&"NO
P	QQ 
WU]]+	++	+	.K KJ&#s"   JJJ#J J8Jc                r    t        | g|i |D ]%  \  }}t        |j                  d       d|        ' y )NT)short_dtypes )r   print	str_short)r   rC   r   r  rL   s        r1   print_saved_residualsr  2  sC    "16t6v6 8idC	T^^^./q
678r3   remat2c                    t        |t              st        |      t        |      k(  sJ t        j                  j
                  t        g|| |||dS )Nr   )rq   rr   r   r   	Primitiver   r   r   r|   r   rS   rC   s        r1   _remat_bindr  <  sT    	K	&#k*:c$i*GG	G			W 
Kt 
K5k,:6
K Kr3   c                6    ~~~t        j                  | dg| S Nr-   r   
eval_jaxprr  s        r1   
remat_implr  B  s    >6		*T	**r3   c                    ~~~~| j                   D cg c]  }|j                   c}t        j                  |       fS c c}w r   )r   r  r   eqn_effects)r   r|   r   rS   rC   r
  s         r1   remat_abstract_evalr  G  s7    
K--	(Q!&&	($*:*:5*A	AA	(s   >c                   |j                   rJ |D cg c]  }t        |      t        j                  u }}t	        j
                  t        j                  |      |d      \  }}	|D cg c]   }t        |      t        j                  us|" }
}t        j                  |j                        }t        |t              r|dt        |
      z  z  }t        j                  g |j                  | |
||||d}t!        |t        |j"                        g      \  }}t%        |      }t'        ||	      D cg c]1  \  }}|rt)        |      nt        j                  j+                  |      3 }}}||fS c c}w c c}w c c}}w )NFTr   )r  r   r   Zeror   	jvp_jaxprrN   close_jaxprr   r   rq   r   r   r   r   r   r&   r   r   zipr   from_primal_value)primalstangentsr   r|   r   rS   tin_nonzeros
jaxpr_jvp_out_nznonzero_tangents	jaxpr_jvpoutsout_primalsout_tangents_pnzout_tangentss                     r1   	remat_jvpr5  L  sm   __	6>?a,?+?||BNN5$9;N*f!)IAT!WGLL-HaII(()9)9:)U#7S!1222K	 
M
M"
M%5
M=FnV
M$  *$U]]1C0DE+}}%-";79a *,$}%1O1OPQ1RR 9, 9	l	"" @I9s   "E2( E7	E756E<c          
        |j                   rJ t        j                  j                  |j                        }|rt	        d|       |d   xs t
        }|D cg c]  }|j                           }}t        j                  ||dgt        |      z  dd|      \  }	}
}}}t        ||      \  }}t        j                  |
|      \  }}t        ||g      \  }}dgt        |      t        |      z
  z  |z   }t        j                  |	|      \  }	}t        |      }t        |	|      }t        d |D              \  }}t        ||      \  }}t!        j"                  |dg| }t        |t        |      |z
  g      \  }}t%        | j&                  |      }t        ||      \  }}|t%        | j(                  |      z   }|j*                  D cg c]@  }t        j,                  | t        j.                  j1                  |j2                        d       B }}t5        |t6              r)t        ||      \  }} dt        |      z  t7        |       z   }t9        ||d|      }!t        j:                  | ||t<        |!t!        j>                  |      tA        jB                               }"tD        jF                  jH                  rtJ        jL                  ntJ        jN                  }#tP        jS                  |#      rE	 t        ||      \  }}$t        |$|jT                  |d        \  }%}|	j*                  t        |	j*                        |z
  d  }&tW        |	jY                  |&	      d
t        |	jT                        z        }'tQ        jZ                  |#ddt]        |%      z  z   dt        |%      z  z   dt]        |%      z  t]        |'      z  z   dt]        |'      z  z   dt        |'      z  z   g|%D (cg c]  }(|(j2                  j_                          c}(|'D )*+cg c]  \  })}*|)j_                         |*fD ]  }+|+   c}+}*})  |D ]	  }|"|_0         tc        |||      S c c}w c c}w c c}(w c c}+}*})w #  Y 5xY w)Nz?Effects not supported in partial-eval of `checkpoint`/`remat`: rS   TFc              3  j   K   | ]+  }|j                   j                         s|j                    - y wr   )pvalis_known)r   r)  s     r1   r   z%remat_partial_eval.<locals>.<genexpr>|  s!     FAAFFOO4EFs   33r-   r!  )r   r   r|   r   ) zremat-decorated function zsaving inputs with shapes:
z  %s
zand zsaving these intermediates:
z  %s from %s
)2r  r   remat_allowed_effectsfilter_not_inr   r5   r9  rN   partial_eval_jaxpr_customr   r'   	dce_jaxprr&   sum_insert_reduce_precisionr$   r   r  mapnew_instantiated_constinstantiate_constr   JaxprTracer
PartialValr   r  rq   r   dictnew_eqn_reciper   r  r   currentr
   log_checkpoint_residualsvalueloggingWARNINGDEBUGloggerisEnabledForr   r   r   logrr   r  reciper*   ),tracer   r|   tracersrD   disallowed_effectsrS   r)  in_unknownsjaxpr_knownjaxpr_stagedout_unknownsout_instr   r/   out_inst_unknownjaxpr_unknownin_used_stagedused_resout_used_knownin_used_knownjaxpr_known_
in_consts_	in_consts
out_consts
out_knowns	residualsres_tracerstracers_stagedin_jaxpr_tracersr   out_jaxpr_tracersprevent_cse_
new_paramsrR  	log_level
staged_unk
res_invarsres_outvarsbody_resr
  aselts,                                               r1   remat_partial_evalru  ^  sP   __	44BB5==Q
I
	 ! ! (//&+23aQZZ\!3+3""
tfs;'77vO =+|\8W
 'x>!"$,,|=M"N-'	B(N 6S.\1BBChN.!||KH+}M'
 *+w?, FFF-!Zz:,!Y|R<)<*$Z#j/'2I1JK*i E00)<+$^W=!^ 3u'>'>#OO - 5 57 ~~eR]]-B-B166-JDQ 7 7U#$^[AOA|C,,u\/BBKF- +-*U$46G')9)9-)H-5579&
 "(!@!@!F!FgooGMM)#$^[Ama$Z1E1Egh1OPmj!''K,?,?(@7(J(KLk!+"5"5k"5"J"'#k.@.@*A"AChjj+.j1AAB3z?*+ j))DN:; 0$x.@	A
 !3x=01M /99!&&""$9M %-KK&1a8JK#K#KM /aah/ 
\:/@	AAC 4<76 :K
s8   Q	AQ-C/Q" !Q
=	Q" #Q)Q" Q" "Q&c           
     >   | j                   t        | j                         |z
  d  }| j                  D ch c]0  }|j                  D ]  }t	        |t
        j                        s|! 2 }}}| j                  d d  | j                  d d  | j                  d d  }}}|D ],  t	        j                  t
        j                        r8t        j                  j                  j                  t        j                        sa|vrft	        t
        j                        sJ t        j                  j                        }	t        j                  j                  j                        }
t!        |
j"                  |
j$                        }|v s|v ra|v r|n|}t        j&                  |	ggt(        j*                  |t-                     }|	||j/                        <   |j1                  d|       Tfdt3        |      D        \  \  }}|j4                  t(        j*                  k(  r|j6                  |k(  r|j9                  |j                   D cg c]  }|k7  r|n|	 c}      }t        j&                  |	ggt(        j*                  |t-               |j:                  |j<                        }|||<   |j1                  |dz   |       / | j9                  |||      }t>        j@                  jB                  xr t        jD                  |       |S c c}}w c c}w )N)exponent_bitsmantissa_bitsr   c              3  J   K   | ]  \  }}|j                   v s||f  y wr   r:  )r   r   r   r
  s      r1   r   z+_insert_reduce_precision.<locals>.<genexpr>  s#     NDAqqAII~!QNs   #	#r:  rA   )r   r  r  )#r   r   r  r   rq   r   Varr  r  ShapedArrayr   
issubdtypedtypenpinexactfinforG  nexpnmantnew_jaxpr_eqnr8   reduce_precision_prf   indexinsertr   r  rD   r   r  ctxr
   enable_checksrK  check_jaxpr)r   r   r  r   r   	used_varsr   r  r  newvarr  rD   lstnew_eqneqn_idxr  v_replace_eqn	new_jaxprr
  s                      @r1   rA  rA    s   ]]3u}}-789(**RQahhR*Q:QqRqR)R!LLOU__Q-?AT)& &aqvvt//0affllBJJ7	a"""XXaffFLL&E

%++FFI~fiIVc""(QC88&#%Ig c#))A,
kk!WNIdONonw
--<::
:
**
KK36;;)@-/ 02QwF)B )@K Ak""(QC88&#%
//377$g "d7m
kk'!)W%7&8 mm6YTmJ)<!1!1)!<	A S.)@s   .L%LL
c                    | ^}}}}}|d   }||d   k(  sJ t        |t              rJt        ||      \  }}t        ||      \  }}t        |t        |            }t        |t        |            }|t        |d      fS )Nr|   )r|   T)r   )rq   r   r'   rG  )	rC   unks_ininst_inr/   params_knownparams_stagedr|   prevent_cse_knownprevent_cse_stageds	            r1   (remat_partial_eval_custom_params_updaterr    s    6:3'7Qm]++	m4	44	4U#)';?q*7K@A%8I2JKLE:L4MNM	tM$?	??r3   c                  |D cg c]=  }t        |t        j                        rt        j                  |j                        n|? }}|j
                  rJ |D cg c]  }t        j                  |       }}| D cg c]  }t        |      t        j                  u  }	}t        t        j                  |      ||	      \  }
}|
j                  |
j                  }}t        j                  |      }t!        || f      \  }}t        |t"              r5t%        ||      \  }}t#        |      dt'        |	      t)        |	      z
  z  z   }t+        j,                  g ||||d|}t/        |      t/        |      }}|D ]C  }t        |t        j                        st1        |      r*|j3                  t1        |             E y c c}w c c}w c c}w )Nr!  r   r|   )rq   r   ValAccumUndefinedPrimalr  r  is_undefined_primalr   r   r"  transpose_jaxprrN   r$  r   r   r   r   r   r'   r   r@  r   r   r   r   accum)out_ctsr   r|   rC   rD   r   args_	in_linearct	out_zerostransposed_jaxpr_in_zerostransposed_jaxprr   	flat_argsr/   rk  	in_cts_nz
in_cts_nz_	in_zeros_s                       r1   remat_transposer    s     *4Ar{{)C2aff%J %  __	278Qr%%a(8)829:BtBx7<<':): /nnUY	!3X.446G6N6NF//0@Aug./,)QU#$Y<OL!%3y>C	N3R(SSKll >F >Y >6F'2>6<>)y/4>i*  a!R[[!$y/ggd: # 9:s   AGG="Gc                    t        |      t        u r|ft        | j                        z  }t        |      t        u r|ft        | j                        z  }t        | t        |      t        |            S r   )r   rr   r   r   	out_avals_transpose_jaxprr   )r   r  r  s      r1   r  r    s_     
)_s5>>22I	)_s5??33I	%y!153C	DDr3   c                    t         j                        D cg c]
  \  }}|r	| c}}t         j                        D cg c]
  \  }}|r	| c}}z   }d  fd} j                  j                  j                         }t        j                  ||      }	t        j                  |	|      \  }
}}t        j                  |
|      }|j                  fS c c}}w c c}}w )Nc                      y r   r-   r-   r3   r1   r   z"_transpose_jaxpr.<locals>.<lambda>  s    r3   c            	        t        | t              t              z
  g      \  }}t        |      }t	        j
                        D cg c]N  \  }}|rt        j                  j                  |      n't        j                  j                  t        |            P }}}t        |d       J t        j                  d      5  t        j                  d      \  }}}	}
d d d        t        j                  d      5   t!        j"                        | }d d d        t        |      }t	        j$                        D cg c]'  \  }}|rt'        j(                  |      n
t        |      ) }}}t        |d       J j
                  t              d  D cg c]  }t+        j,                  |       }}t+        j.                  |j0                  d|j2                  g |||      }|t        |      d  }|D cg c]  }t5        |      t&        j(                  u  c}x}_        t9        ||      \  }}|S c c}}w # 1 sw Y   _xY w# 1 sw Y   6xY wc c}}w c c}w c c}w )NFrematted_computation)r&   r   r@  r   r%  r   rN   rF  r   knownr   r
   mutable_array_checkspartial_eval_jaxpr_nounitsr   extend_name_stackr   jaxpr_as_funr  r   r"  r   r  backward_passr   r   r   in_cts_zeror'   )r   ins_flatout_cts_flatins_iterr  linin_pvalsjaxpr_rematted	lin_jaxprout_uk	res_avalsr   out_cts_iterzeror  
dummy_argsin_ctsr  r  r  r/   cellin_linr   r  s                        r1   
transposedz$_transpose_jaxpr.<locals>.transposed  sE   '	CK#f+4M3NOHl H~H "%U^^V!<>D# 03%%d+##DN34 >H > $''' 
	$	$U	+ >

'
'vu
= 3ni> 
	+	+,B	C <0t  0(;f< %L!$U__i!@B4 &*w||D!tL/AA BG Bd#+++7@7I7I#f+,7WXt"$$T*XJXiooui6F6F444g?FCKL!F GM"M48w||#;"MMHt!(F3LIq3>> >< <
B Y #Ns1   AH8H>>I,II;"I#>IIr   )r%  r   r  r   r   with_unknown_namesr   r   rN   r   r   ClosedJaxprr  )r   r  r  rr  r  r  r   r  dbgtransposed_wrappedr  r/   r   r  r  s   ```           @r1   r  r     s     !$ENNV ?KWQsqK #EOOY ?LWQtqLM(	$@ 	113#||J3?!#!:!:("$Q%%&7@	4++	++S LLs   
C"C"	
C(C(c          	        |j                   rJ t        j                  t        j                  |      | |t        j
                  gt        |j                        z        \  }}|j                  |j                  }}|rt        j                  |      }|D 	cg c]  }	|	rdnd 
 }
}	t        j                  g ||d|i||
fS c c}	w )Nr   r   )r  r   batch_jaxpr_axesrN   r$  zero_if_mappedr   r   r   r   r   r   r   )	axis_datarC   dimsr   rD   jaxpr_batched_out_batchedjaxpr_batchedr   bout_dimss              r1   
remat_vmapr  /  s    __	 ( 9 9nnUY#emm"44!6.+ )..0E0E-..}=M(3411a$4(4		Dv	D	DM	DV	Dh	NN 5s   Cc           	     (   t        |       s0t        j                  |      sdgt        |j                        z  d fS t        j
                  |j                  d   |       \  }}|j                  d   }t        |t              rt        d t        ||      D              }t        |j                  ||      }t        |      s$t        |       st        |j                        r|d fS t        j                  t        |j                  |      D cg c]
  \  }}|s	| c}}t        |j                  |       D cg c]
  \  }}|s	| c}}|j                  |t!        j"                  |      |j$                  |j&                        }||fS c c}}w c c}}w )NFr   r|   c              3  ,   K   | ]  \  }}|s	|  y wr   r-   )r   r2  us      r1   r   zremat_dce.<locals>.<genexpr>C  s     IdaqIs   
r  )r   rN   has_effectsr   r   r?  rD   rq   r   r%  rG  _has_effectsr   r  r   r  r   r  r  r  )	used_outputsr  r  used_inputsr|   rl  r
  usedr  s	            r1   	remat_dcer  <  s=   	\	2>>##67S_$d**<<

7(;\J)[

=)+U#Ic+{&CIIKCJJi[I*
k
3|#49$$%cjj+6?wq$$?ckk<8Awq$DAz4#3#3I#>	"G
 	 	@As   
F
F
7
FFc                l    t         j                  t         j                  ft        fd| D              S )Nc              3  8   K   | ]  }t        |         y wr   )rq   )r   r   not_really_effectss     r1   r   z_has_effects.<locals>.<genexpr>S  s     DqA122Ds   )r   NamedAxisEffectInternalMutableArrayEffectr   )r   r  s    @r1   r  r  Q  s+    ,,d.M.MN	DGD	DDr3   c                t    | j                   rJ |r	|rt        }nd } t        j                  |d      |d| iS )Nc                0    t        j                  | dg| S r  r  r   rC   s     r1   r   z!remat_expansion.<locals>.<lambda>^  s    DOOE2,M,M r3   r   r<   r   )r  $_remat_translation_using_opt_barrierr	   
named_call)r   r|   r   rC   r/   translation_rules         r1   remat_expansionr  V  s@     __	;M	<(|	<d	P%	PPr3   c                Z    t        j                  |      }t        j                  | dg| S r  )r8   optimization_barrierr   r  r  s     r1   r  r  c  s(    		*	*4	0$		*T	**r3   c                  t        |t              r|ft        | j                        z  }t        |t              sJ |rt        |      rt        || j                        \  }}t        ||      \  }}	t        j                  t        j                  |	            }
t        j                  |
j                  t        t        j                  |            }t        |||      }t        j                   | j"                  || j$                  j'                  d      | j(                  dg|| j*                  | j,                  d\  }}| j/                  |       |S )Nr   r-   )dim_var_valuesconst_lowering)rq   rr   r   avals_inr   r   r'   r   OptimizationBarrierOpr   flatten_ir_valuesunflatten_ir_values_like_typesr	  rB  aval_to_ir_typer*   jaxpr_subcompmodule_context
name_stackextend	tokens_inr  r  set_tokens_out)r  r   r|   r   rS   rC   r/   barrier_avals
other_argsbarrier_args
barrier_opbarrier_resultsr/  
tokens_outs                 r1   _remat_loweringr  h  s&    T".3s||#44K	K	''	'K(%k3<<@A}-k4@J**|,.J99C 4 4mDFO{J@D''	%!6!6|!D	mmR))/2/A/A'')$
 Z 	+r3   c                L    t        t        t        j                  |      |       S )a  Identifies a value with a name within :func:`jax.checkpoint`.

  This function acts as an identity function at runtime (returning ``x``
  unchanged) but attaches a string name to the value in the JAX trace.
  These names can be targeted by specific checkpointing policies (see
  :ref:`checkpoint-policies`) to control which intermediate values
  are saved during the forward pass and which are recomputed during the
  backward pass.

  Args:
    x: array or PyTree of arrays to be named.
    name: A string name to associate with the value ``x``.

  Returns:
    The input ``x``, unchanged.

  See Also:
    - :func:`jax.checkpoint` (alias: :func:`jax.remat`): decorator to
      enable checkpointing.
    - :mod:`jax.checkpoint_policies`: a namespace containing policies
      that use names marked via ``checkpoint_name`` to determine behavior.

  Example:
    >>> import jax
    >>> import jax.numpy as jnp
    >>> from jax.ad_checkpoint import checkpoint_name

    >>> # Define a function where we explicitly name an intermediate value
    >>> def f(x):
    ...   y = jnp.sin(x)
    ...   z = checkpoint_name(y, "my_intermediate")
    ...   return jnp.cos(z)

    >>> # Use a policy that saves only the named value
    >>> policy = jax.checkpoint_policies.save_only_these_names("my_intermediate")
    >>> f_checkpointed = jax.checkpoint(f, policy=policy)

    For further examples, see the `remat example notebook
    <https://docs.jax.dev/en/latest/notebooks/autodiff_remat.html>`_.
  r  )r"   r   rX   r   r   r<   s     r1   checkpoint_namer
    s    R 
'&++D11	55r3   c                   | S r   r-   r	  s     r1   r   r     s    1 r3   c                   | S r   r-   r	  s     r1   r   r     s    A r3   c               F    | |c\  }\  }t         j                  ||      |fS Nr  rX   r   )r'  r(  r<   r   xdots        r1   name_jvpr    s)    8-$1	QT	"D	((r3   c                   |gS r   r-   )r  r   r<   s      r1   r   r     s    s r3   c               F    | |c\  }\  }t         j                  ||      |fS r  r  )rC   r  r<   r   r   s        r1   name_batcherr    s)    T*$1	QT	"A	%%r3   c               B   t        j                  |g       \  }\   t        j                  |d|i|}t	        |t        |j                        g      \  }}t        |      }	| D 
cg c]  }
t        |
t              rt        |	      nd ! }}
t        |	d       J ||fS c c}
w )Nr   )r   discharge_stater   r   r&   r   r   r   rq   r   r   )r   r  r   rC   rD   discharged_jaxprout_vals_ref_valsout_valsref_vals	ref_vals_rr  
new_invalss               r1   _remat_state_discharge_ruler    s     #225"=BllDK0@KFK!"3c%--6H5IJ(H8n)!# $.a#=Y4G #* #	i		&&	&	X	#s   $$B)returnrr   )r   r   r|   rr   rS   zCallable[..., bool] | Noner}   zint | tuple[int, ...]r~   zbool | DeprecatedArgr  r   )r   r   r}   zfrozenset[int]r   ztuple[WrapHashably, ...]r   r   )
r   r   r   r   r   zSequence[core.AbstractValue]r   zcore.DebugInfor  z+tuple[core.Jaxpr, Sequence[Any], PyTreeDef])r   r   r  $list[tuple[core.AbstractValue, str]])r   
core.Jaxprr   zSequence[str]r  r  )rS  zpe.JaxprTracerT  zcore.Tracerr   r   )r   r   r   r   r  r   )r   core.ClosedJaxprr  bool | Sequence[bool]r  r"  r  z#tuple[core.ClosedJaxpr, list[bool]])r   r!  r  Sequence[bool]r  r#  )r  z
list[bool]r  zcore.JaxprEqnr  z'tuple[list[bool], core.JaxprEqn | None])r   r   r|   rr   r   rr   )r   r   )r  zmlir.LoweringRuleContextr   r   r|   rr   r   rr   )
__future__r   collections.abcr   r   	functoolsr   rL  typingr   typesnumpyr~  jax._srcr   r	   r
   r   r   r   r   r   r   r   r   r   jax._src.interpretersr   r   r   r   rN   jax._src.laxr   r8   r   r:   jax._src.lib.mlir.dialectsr   jax._src.stater   jax._src.state.typesr   jax._src.traceback_utilr   jax._src.tree_utilr   r   r   r    r!   r"   jax._src.typingr#   jax._src.utilr$   r%   r&   r'   r(   r)   r*   r+   register_exclusion__file__rB  r%  	getLoggerr   rO  r2   r5   r>   rx   rH   rT   r  rX   r^   rb   rg   rn   rw   SimpleNamespacecheckpoint_policiesr   r   r   r   r   r   r   r   r   r   r  r   multiple_resultsr  r   def_implr  def_effectful_abstract_evalr  r5  primitive_jvpsru  custom_partial_eval_rulesrA  r  call_partial_eval_custom_rulepartial_eval_jaxpr_custom_rulesr  fancy_transposesr  r  r  fancy_primitive_batchersr  	dce_rulesr  r  r  r  register_loweringr
  def_abstract_evalr  r  primitive_batchersregister_discharge_ruler  r-   r3   r1   <module>rG     s   # .          !  &  % #  $ * & 4 , 7 * $ , 0  *E E E $  # #H - ! ! !( +			8	$0    
	0$ ,e+++%!%E'G#A%E5/3&GI " 	&675948790=w1w4w .w EMw 8wt 15/324+8?F,F/F )F @HF4 6G7G@CG(@IL ))?@  ?&?:? *? E	? ?0[(L[,' -'2V'R8 $..
" K + + 	$$B %B#  '  ' JB(JBV );  W %" "H	@ B,,g46  " "7 + ,  /  G E4E<E ,,+,, .,, ,,\	O .8 ! !' * : & "W E

Q
Q+/
QAE
Q+
	!  	
 4   w 0)6V $ %   - .) %  &    v: ;& '3  F # #""7+	 ,	r3   