
    ukiܸ              
         d dl mZ d dlmZmZ d dlmZ d dlmZ d dl	Z	d dl
mZ d dl
mZ d dl
mZ d d	l
mZ d d
l
mZ d dl
mZ d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&Z'ejP                  Z)ejT                  Z+	 	 dQ	 	 	 	 	 	 	 	 	 dRdZ,	 	 	 	 dS	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dTdZ-	 	 	 	 dUdZ.	 	 dQ	 	 	 	 	 	 	 	 	 dVdZ/	 	 dQ	 	 	 	 	 	 	 	 	 dVdZ0	 	 dQ	 	 	 	 	 	 	 	 	 dVdZ1	 	 dQ	 	 	 	 	 	 	 	 	 dVdZ2	 	 dQ	 	 	 	 	 	 	 	 	 	 	 	 	 dVdZ3	 	 	 	 	 	 	 	 	 	 	 	 	 	 dWd Z4	 	 	 	 	 	 	 	 	 	 dXd!Z5	 	 	 	 	 	 	 	 	 	 	 	 	 	 dYd"Z6d# Z7d$ Z8 ejr                  d%      Z:d& Z;e;ejx                  e:<   d'e:_=        e:j}                   eej~                  e:             e:j                  e7       e8ej                  e:<   	 	 dZd(ZB ej                  e:eB       d) ZDd* ZEd+ ZFd, ZG ej                  eDej                  d-eG eej                  d-      .      ZK ej                  eKeE        eeFe/      ej                  eK<   d/ ZMd0 ZN	 	 dQd1ZO ej                  eNej                  d2eG eej                  d2      .      ZP ej                  eP eeMej                                eeFe1      ej                  eP<    ej                  eNej                  d3eG eej                  d3      .      ZS ej                  eS eeMej                                eeFe2      ZU eeFe2      ej                  eS<   d4 ZV ej                  eK eeVe#j                  d5               ej                  eS eeVej                  ej                                ej                  eP eeVej                  ej                               d6 Z\d7 Z] ej                  e\ej                  d8e] eej                  d8      .      Z^	 	 	 	 	 	 d[d9Z_ ej                  e^e_       d: Z`d; Zad< Zbd= Zcd> Zd ej                  e`ej                  d?ea eej                  d?      .      Zeecej                  ee<   ebejx                  ee<   edej                  ee<   d@ Zg ej                  ee ej                   eegdAB      dAC              ej                  ee ej                   eegd'B      dAC      dDE        ej                  ee ej                   eegd'B      dAC      dFE       dG ZidH ZjdIdJ	 dZdKZkdL ZldM ZmdN ZndO Zo ej                  eiej                  dPej eej                  dP      .      Zpemejx                  ep<   enej                  ep<   eoej                  ep<    ej                  ep ej                  eldAC              ej                  epekdFE       y)\    )annotations)CallableSequence)partial)AnyN)ad_util)api_util)core)dispatch)dtypes)	tree_util)util)ClosedJaxprShapedArrayjaxpr_as_fun)ad)batching)mlir)convolution)lax)slicing)	logaddexp)ir)hlo)Arrayc                   t        j                  |       \  }}	t        j                  d|||fi       }
t        j                  |      \  }}|	|k7  rt	        d|	 d|       t        |      t        |      k7  r#t	        dt        |       dt        |             t        |      dk(  rt	        d      t        |t              rI||nt        j                  ||      }t        t        j                  |d   j                  |||            }nt        d |D              }|dt        |      z  }|dt        |      z  }|dt        |      z  }t        ||      }|r || |||||      S t        t        j                   |      }t        j"                  ||
t        |      |      \  }}|	|k7  rt	        d	|	 d|       t        j$                  | }t'        j(                  g |||j*                  t        |j,                        t        |      t        |      |t        |      t        |      d
}t        j.                  ||      S )Nzreduce_window compz;Operands must have the same tree structure as init_values: z vs. z9Must have same total number of operands as init_values:  r   z-reduce_window must have at least one operand.c              3  *   K   | ]  \  }}||f  y wN ).0xys      [/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/lax/windowed_reductions.py	<genexpr>z!_reduce_window.<locals>.<genexpr>Q   s     /tq!QF/s      zGreduce_window output must have the same tree structure as the operands jaxprconstswindow_dimensionswindow_stridespaddingbase_dilationwindow_dilation)r   tree_flattenr	   
debug_info
ValueErrorlen
isinstancestrr   _dilate_shapetuplepadtype_to_padsshape_get_monoid_window_reducermapr
   get_aval_variadic_reduction_jaxprstandard_insert_pvaryreduce_window_pbindr(   r)   tree_unflatten)operand
init_valuecomputationr*   r+   r,   r-   r.   flat_operandsoperand_tree
comp_debugflat_init_valuesinit_value_treedilated_window_dimsmonoid_reducerflat_init_avalsr(   out_treeout_flats                      r#   _reduce_windowrN   /   s    !* 6 6w ?-""#7$.
#;RA*&/&<&<Z&H#O_$
	$~U?*;	=  	3/00
	uS)9%:$;	= 
 	1
D
EE,4+_=  C''a 3^WN OG /w//GC 122N3011MS!233O-k;KL.'#4ng': : $--)9:O33Z!7OE8 x	>xj	*+ + ..>M## 
	
	
 kkU\\" 12^,M*o.
H ##Hh77    c           
     &    t        | |||||||      S )ac  Reduction over padded windows.

  Wraps XLA's ReduceWindowWithGeneralPadding_ operator.

  Args:
    operand: input array or tree of arrays.
    init_value: value or tree of values. Tree structure must match that
      of ``operand``.
    computation: callable function over which to reduce. Input and output must be
      a tree of the same structure as ``operand``.
    window_dimensions: sequence of integers specifying the window size.
    window_strides: optional sequence of integers specifying the strides, of
      the same length as ``window_dimensions``.  Default (``None``) indicates
      a unit stride in each window dimension.
    padding: string or sequence of integer tuples specifying the type of padding
      to use (default: "VALID"). If a string, must be one of "VALID", "SAME", or
      "SAME_LOWER". See the :func:`jax.lax.padtype_to_pads` utility.
    base_dilation: optional sequence of integers for base dilation values, of
      the same length as ``window_dimensions``. Default (``None``) indicates unit
      dilation in each window dimension.
    window_dilation: optional sequence of integers for window dilation values, of
      the same length as ``window_dimensions``. Default (``None``) indicates unit
      dilation in each window dimension.

  Returns:
    A tree of arrays with the same structure as ``operand``.

  Example:
    Here is a simple example of a windowed product over pairs in a 1-dimensional array:

    >>> import jax
    >>> x = jax.numpy.arange(10, dtype='float32')
    >>> x
    Array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.], dtype=float32)

    >>> initial = jax.numpy.float32(1)
    >>> jax.lax.reduce_window(x, initial, jax.lax.mul, window_dimensions=(2,))
    Array([ 0.,  2.,  6., 12., 20., 30., 42., 56., 72.], dtype=float32)

  .. _ReduceWindowWithGeneralPadding: https://www.openxla.org/xla/operation_semantics#reducewindow
  )rN   rA   rB   rC   r*   r+   r,   r-   r.   s           r#   reduce_windowrR   t   s*    f 
	
 	rO   c                   t        |      dk7  ry |\  }t        j                  |      }t        j                  |      r|j                  dk(  rt        j
                  |      }| t        j                  u r|dk(  xr t        S | t        j                  u r*|t        j                  |j                        k(  xr t        S | t        j                  u r*|t        j                  |j                        k(  xr t        S y )Nr&   r   r   )r2   r
   r;   is_concreter8   to_concrete_valuer   add_reduce_window_summax_get_max_identitydtype_reduce_window_maxmin_get_min_identity_reduce_window_min)	monoid_opxsr!   avalvals        r#   r9   r9      s     	W\	"!	q	$	aTZZ2-

 
 
#CCGGAX,,,	cgg	S**4::66 %$&	cgg	S**4::66 %$&	rO   c           
         |dt        |      z  }|dt        |      z  }t        j                  | t        |      t        |      t        |      t        |      t        |            S Nr%   )r*   r+   r,   r-   r.   )r2   reduce_window_sum_pr?   r6   rA   r*   r+   r,   r-   r.   s         r#   rW   rW      q    
 3011MS!233O		!	!'8!9>*E'N-(O,	 
" 
. .rO   c                   t        j                  | d      }t        j                  t         j                  t	        j
                  |            \  }}|dt        |      z  }|dt        |      z  }t        j                  | |||t        |      t        |      t        |      t        |      t        |      	      \  }	|	S )Nr&   r%   r'   )
r   _const_reduction_jaxprmulr
   r;   r2   r>   r?   r6   
rA   r*   r+   r,   r-   r.   rB   r(   r)   outs
             r#   _reduce_window_prodrn      s    
 zz'1%*&&swwj0IJ-%3011MS!233O			zv/0>*E'N-(O, 
 
.$# 
*rO   c           
         |dt        |      z  }|dt        |      z  }t        j                  | t        |      t        |      t        |      t        |      t        |            S rd   )r2   reduce_window_max_pr?   r6   rf   s         r#   r[   r[      rg   rO   c           
         |dt        |      z  }|dt        |      z  }t        j                  | t        |      t        |      t        |      t        |      t        |            S rd   )r2   reduce_window_min_pr?   r6   rf   s         r#   r^   r^      rg   rO   c                   t        j                  | t        j                         }t        j                  t
        t        j                  |            \  }}|dt        |      z  }|dt        |      z  }t        j                  | |||t        |      t        |      t        |      t        |      t        |      	      \  }	|	S )Nr%   r'   )r   ri   npinfrj   r   r
   r;   r2   r>   r?   r6   rl   s
             r#   _reduce_window_logaddexprv     s     zz'BFF7+*&&y$--
2KL-%3011MS!233O			zv/0>*E'N-(O, 
 
.$# 
*rO   c                \   t        j                  |t        j                  |            \  }}	t        j                  |t        j                  |            \  }
}t        j                  | ||      \  } }}t
        j                  | ||||	|
|t        |      t        |      t        |      
      S )N)select_jaxprselect_constsscatter_jaxprscatter_constsr*   r+   r,   )r   rj   r
   r;   r=   select_and_scatter_pr?   r6   )rA   selectr*   r+   r,   sourcerB   scatterrx   ry   rz   r{   s               r#   _select_and_scatterr     s    
 !$ 4 4
DMM*%!',"%"6"6T]]:&#(- $ : :vz!#'6:		"	"vz!#u=N7O>*E'N	 
# 
D DrO   c           
         t        j                  | |      \  } }t        j                  | ||t	        |      t	        |      t	        |            S )N)select_primr*   r+   r,   )r
   r=   select_and_scatter_add_pr?   r6   r~   rA   r   r*   r+   r,   s         r#   _select_and_scatter_addr   ,  sR    
 ..vw?/&'	!	&	&g;/0>*E'N 
' 
D DrO   c                    t        j                  | |      \  } }t        j                  | ||t	        |      t	        |      t	        |      t	        |      t	        |            S )a(  Extracts the tangent corresponding to the minimum or maximum element in
  each window of the `operand` array.

  Wraps XLA's `ReduceWindow
  <https://www.openxla.org/xla/operation_semantics#reducewindow>`_
  operator, which applies a reduction function to all elements in each window of
  the input multi-dimensional array. In this case, the input multi-dimensional
  array is built by packing each element in the `operand` array with its
  corresponding element in the `tangents` array.

  Args:
    tangents: an array
    operand: an array with the same shape as `tangents`
    select_prim: a reduction function (restricted to `ge_p` and `le_p`)
    window_dimensions: an array of integers for window dimension values
    window_strides: an array of integers for window stride values
    base_dilation: an array of integers for base dilation values
    window_dilation: an array of integers for window dilation values

  Returns:
    An array containing the elements in `tangents` corresponding to the output
    of the reduction of `operand` fin each window.
  )r   r*   r+   r,   r-   r.   )r
   r=   select_and_gather_add_pr?   r6   tangentsrA   r   r*   r+   r,   r-   r.   s           r#   _select_and_gather_addr   7  sa    < 007C(G	 	%	%[/0>*E'N-(O, 
& 
. .rO   c           	     t   t        j                  |t        |      dz  g      \  }}	t        d t	        ||	      D              rMd}
t        |
j                  |D cg c]  }|j                   c}|	D cg c]  }|j                   c}            t        d |	D              r4d}
t        |
j                  |	D cg c]  }|j                   c}            t        |d   |||||      t        |d   |||||      t        j                  dg| t        fd|D              S c c}w c c}w c c}w )	N   c              3  T   K   | ]   \  }}|j                   |j                   k7   " y wr   rZ   )r    oivs      r#   r$   z4_reduce_window_abstract_eval_rule.<locals>.<genexpr>i  s&      	#aagg	s   &(zsreduce_window got inconsistent dtypes for operands and init_values: got operand dtypes {} and init_value dtypes {}.c              3  L   K   | ]  }t        |j                        d k7    yw)r   N)r2   r8   )r    vs     r#   r$   z4_reduce_window_abstract_eval_rule.<locals>.<genexpr>p  s     3qQWW	3s   "$zPreduce_window expected init_values to be scalars but init_values have shapes {}.r   rR   c              3  P   K   | ]  }t        |j                           yw))shardingvmaN)r   rZ   )r    op	out_shapeout_shardingr   s     r#   r$   z4_reduce_window_abstract_eval_rule.<locals>.<genexpr>{  s+      ( 9bhh3OO (s   #&)r   
split_listr2   anyzip	TypeErrorformatrZ   r8    _common_reduce_window_shape_rulereduce_window_sharding_ruler
   standard_vma_ruler6   )r(   r)   r*   r+   r,   r-   r.   avalsoperand_avalsinit_val_avalsmsgr   r   r   r   r   r   s                 @@@r#   !_reduce_window_abstract_eval_ruler   ^  s<    #'//%#e*/9J"K- 	'*=.'I	 >C
CJJ?A?3ABRBD E E3N33C
CJJ@A@A
BB.A)>7_&) -A)>7_&, 	??#	 (&( 
( (  @B  As   D+5D0;D5c                  t        |       dz  }	t        j                  | |	g      \  }
}t        j                  ||	g      \  }}t        d |D              rt	        d      t        d t        |
|      D              }t        |
|      D cg c]  \  }}t        j                  |||       }
}}d|z   }d|z   }d|z   }d|z   }d|z   }t        j                  |
|z   |||||||d}|d|	z  fS c c}}w )	Nr   c              3  $   K   | ]  }|d u 
 y wr   r   )r    	init_bdims     r#   r$   z4_generic_reduce_window_batch_rule.<locals>.<genexpr>  s     A9$	As   z<reduce_window batching is not implemented for initial valuesc              3  D   K   | ]  \  }}||j                   |     y wr   r8   )r    r!   axs      r#   r$   z4_generic_reduce_window_batch_rule.<locals>.<genexpr>  s(      !ea ggbk !    r%   r   r   r'   r   )r2   r   r   r   NotImplementedErrornextr   r   bdim_at_frontr>   r?   )batched_args
batch_dimsr(   r)   r*   r+   r,   r-   r.   num_operandsoperandsinit_valuesoperand_bdimsinit_value_bdimssizeargbdimoutss                     r#   !_generic_reduce_window_batch_ruler     s/    \"a',//,G(K$(OOJ$O!-!A0@AA
 / 0 0 
 !s8]'C ! 
!$  #8];=#t $$S$5 =( =...(.'&-?*/			;uV).]%	
'$
 
tl"	""=s   !C-rR   c	           
        |}	t        |       dz  t        j                  | g      \  }
}t        j                  |g      \  }}t        d |D              st	        d      t        t        j                  |      }t        |	|      }t        j                  |dt        |      z  dgt        |      z        d   fd}t        g |
|g ||||||||      }t        j                  |t        |      dz  g      \  } }g | g |fS )	Nr   c              3  P   K   | ]  }t        |t        j                           y wr   )r3   r   Zero)r    ts     r#   r$   z$reduce_window_jvp.<locals>.<genexpr>  s     @Z277#@s   $&z?reduce_window jvp does not support non-zero init_value_tangent.)TFr   c                    t        j                  | g      \  }}t        j                  |g      \  }} t              g |||| S r   )r   r   r   )leftrightpltlprtrjvp_reductionns         r#   wrapperz"reduce_window_jvp.<locals>.wrapper  sS    __TA3'FB__UQC(FB&<&::R:":r::rO   rQ   )r2   r   r   allr   r:   r   instantiater   r   	jvp_jaxprrN   )primalsr   r*   r+   r,   r-   r.   r(   r)   reduction_jaxprrA   rB   operand_tangentinit_value_tangentc_reduction_jaxprr   jvp_primals_tangentsr   r   s                    @@r#   reduce_window_jvpr     s,    /	'la!!5':(,1#(F%/%	@-?@	@
U
VV7..0BC!/6:,,0'CM2IE7UXYkUlKlmnop-;
 (**/*3:3 23)#!%	 oo&:SAU=VZ[=[<\]'8	7[x[	  rO   Tc                   t        j                  |t        |      dz  g      \  }	}
t        j                   j                  t        |	      g      \  }}d fd}t	        j
                   d||	|
| j                  |||||      S )Nr   c                &   j                   rt        d      t        j                  j                  j
                  t        j                         g| j                  j                  j                  d\  }}t        j                  |      S )Nz'Cannot lower effectful `reduce_window`.dim_var_valuesconst_lowering)effectsr   r   jaxpr_subcompmodule_context
name_stackTokenSet	argumentsr   r   flatten_ir_values)reducer	out_nodes_r)   ctxr(   s      r#   reducer_bodyz2_generic_reduce_window_lower.<locals>.reducer_body  s    }} IJJ%%c&8&8%N")"3"3N))#:L:LNLIq !!),,rO   generic_reduce_window_reducerreducer_namer   r   r   init_values_avals	out_avalsr*   r+   r-   r.   r,   r   zir.BlockreturnzSequence[ir.Value])r   r   r2   avals_inr   rR   	avals_out)r   r(   r)   r*   r+   r,   r-   r.   argsr   r   r   init_value_avalsr   s   ```           r#   _generic_reduce_window_lowerr     s     //$Ta0@A(Ks8}oF!- 
			2()#!%
 rO   c                  t        j                  | j                  t        j                        sCd}t        |j                  t        j                  | j                        j                              t        | |||||      S )Nz=operand to reduce_window_sum must have a number dtype, got {})	r   
issubdtyperZ   rt   numberr   r   namer   rA   r*   r+   r,   r-   r.   r   s          r#   _reduce_window_sum_shape_ruler     sd    			7=="))	4
IC
CJJrxx6;;<
==	)'3D*8'*7
J JrO   c          	        t        j                  |      sJ |j                  j                  }t	        j
                  |||| j                  |||      }dgt        |      z  }	t        ||      D 
cg c]  \  \  }
}}|
||dz
  f }}}
}t        j                  | t        j                  |       |      }t        |||dgt        |      z  |	|      }|j                  |k(  sJ |j                  |f       |gS c c}}}
w )Nr&   r   )r-   r.   )r   is_undefined_primalra   r8   r   _conv_general_vjp_lhs_paddingr2   r   r   pad_zerorW   )	cotangentrA   r*   r+   r,   r-   r.   input_shapepadsoneslohistridepadding_configpad_cotangentresults                   r#   !_reduce_window_sum_transpose_ruler	  
  s    
			((	(""+		2	2$nioow_
&$ s;	$,/n,EG G(R& VaZ( G. G'')SYYy%9>J-m->%h[)99,0.=?& 
	$Av||[&AA	$
/Gs   2C8c                   |\  }|\  }	|	F|d |	 dz   ||	d  z   }|d |	 dz   ||	d  z   }|d |	 dz   ||	d  z   }|d |	 dz   ||	d  z   }|d |	 dz   ||	d  z   } | ||||||      }||	fS )Nr%   r   r   )
rR   r   bdimsr*   r+   r,   r-   r.   rA   r   s
             r#   _reduce_window_batch_ruler    s     ('
%$	%4 4'*;DE*BB #ET*T1N454IINetny(745>9G!%4(4/-2FFM%et,t3ode6LLO'#4ng':'	$rO   c                x    t        | j                  |||||      }t        j                  j	                  || d      S )NrR   )reduce_window_shape_tupler8   r   r   #_get_sharding_for_varying_out_shape)rA   r*   r+   r,   r-   r.   r   s          r#   r   r   /  s?    'mm&) 
	8	8/
+ +rO   reduce_window_sum)sharding_rulevma_rulec          
         | t         j                  u s| t         j                  u sJ | t         j                  u rt         j                  nt         j                  }t        ||||||||      S r   )r   max_pmin_pge_ple_pr   )	primgrA   r*   r+   r,   r-   r.   r   s	            r#   _reduce_window_chooser_jvp_ruler  ?  s[     
	dcii//	/ CII-388+	7K9J . /
1 1rO   c                   t        j                  dd|d       t        j                  dd|d       t        j                  dd|       t        j                  dd|       | j                  t        |      k7  r'd}t	        |j                  | j                  |            t        |      t        |      k7  rd	}t	        |j                  ||            t        |      t        |      k7  rd
}t	        |j                  ||            t        |      t        |      k7  rd}t	        |j                  ||            t        | j                  |||||      S )NrR   r*   T)non_zero_shaper+   r-   r.   ztreduce_window got the wrong number of window_dimensions for operand: got operand shape {} with window_dimensions {}.ztreduce_window got inconsistent window_strides and window_dimensions: got window_strides {} and window_dimensions {}.zrreduce_window got inconsistent base_dilation and window_dimensions: got base_dilation {} and window_dimensions {}.zvreduce_window got inconsistent window_dilation and window_dimensions: got window_dilation {} and window_dimensions {}.)r   _check_shapelikendimr2   r   r   r8   r  r   s          r#   r   r   I  sK    (;=N&*,(8.&*,G(9?K\\S*++	C  CJJw}}.?@
AAC 122PC
CJJ~/@A
BB3011OC
CJJ}.?@
AAS!233C CJJ0AB
CC	"7==2C#17M#2
4 4rO   c                    |t        j                  | |      } |t        j                  ||      }t        d t        | |      D              }t        t	        t
        j                  |||            S )Nc              3  8   K   | ]  \  }\  }}||z   |z     y wr   r   )r    dr   phs       r#   r$   z,reduce_window_shape_tuple.<locals>.<genexpr>v  s      THRR"Ts   )r   r5   r6   r   r:   r
   
stride_dim)operand_shaper*   r+   r,   r-   r.   operand_paddeds          r#   r  r  o  sh     %%m]CM ))*;_MTM78STT.	s4??N4E~V	WWrO   reduce_window_maxreduce_window_minc               B    |j                   \  }	|	j                  d|	j                  j                  d            }
t        j                  |d|
j
                   d fd|gt        j                  | ||
j
                        |
      g|
g|j                  |||||      S )Nr   specr8   r   reduce_window__reducerc                $     | j                    gS r   )r   )r   	reduce_ops    r#   <lambda>z&_reduce_window_lower.<locals>.<lambda>  s    Iw/@/@$A#B rO   r   )r   updater   r   rR   rZ   full_like_avalr   )r/  rB   r   rA   r*   r+   r,   r-   r.   operand_avalscalar_avals   `          r#   _reduce_window_lowerr5    s     ,,-,##..5525> $ @+ 
			#K$5$5#6h?By


c:k.?.?#@+
N %)#!%
 rO   c                     y)Nr   r   )r   s    r#   r0  r0    s    rO   c                   t        j                  dd|       t        j                  dd|       t        |      t        |      k7  rd}
t        |
j	                  ||            | j
                  S )Nselect_and_scatterr*   r+   zyselect_and_scatter got inconsistent window_strides and window_dimensions: got window_strides {} and window_dimensions {}.)r   r  r2   r   r   r8   )rA   r~   rB   rx   ry   rz   r{   r*   r+   r,   r   s              r#   _select_and_scatter_shape_ruler9    sk     +-@(*+-=~N	s>22PC
CJJ~/@A
BB	rO   c                   | j                   S r   r   )
rA   r~   rB   rx   ry   rz   r{   r*   r+   r,   s
             r#   !_select_and_scatter_sharding_ruler<         
		rO   r8  c               r   | j                   \  }}}| j                  \  }|j                  d|j                  j                  d            }t	        j
                  |      }t        j                  t	        j
                  |      |||t	        j                  |      t	        j                  |	      t        j                  j                  t        j                  |
t        j                        t        |
      df            }|j                   j"                  j%                  ||      }t        j&                  |      5  |j(                  rt+        d      t	        j,                  | j.                  || j0                  t	        j2                         |g|j4                  | j6                  | j8                  d\  }}t        j:                  t	        j<                  |             d d d        |j>                  j"                  j%                  ||      }t        j&                  |      5  |j(                  rt+        d	      t	        j,                  | j.                  || j0                  t	        j2                         |g|j4                  | j6                  | j8                  d\  }}t        j:                  t	        j<                  |             d d d        tA        |jB                  | j                        D cg c]  \  }}t	        jD                  | ||       c}}S # 1 sw Y   :xY w# 1 sw Y   ^xY wc c}}w )
Nr   r)  r+  r   r   )r*   r+   r,   z Cannot lower effectful `select`.r   z!Cannot lower effectful `scatter`.)#r   r   r1  r   r   aval_to_ir_typer   SelectAndScatterOpdense_int_arrayr   DenseIntElementsAttrgetrt   asarrayint64r2   r}   blocksappendInsertionPointr   r   r   r   r   r   r   r   r   return_r   r   r   resultslower_with_sharding_in_types)r   rA   r~   rB   rx   ry   rz   r{   r*   r+   r,   r3  source_avalinit_value_avalaval_outr4  scalar_typer   r}   r   r   r   rra   s                           r#   _select_and_scatter_lowerrQ    s   
 03||,,_mm)(##..5525> $ @+$$[1+

8$,,->?)).9%%))"**Wbhh*G14Wq0A * CD" 99"";<&	  	3 BCC%%c&8&8,&)nn&*mmo}I (.'7'7I 695G5G585G5GILIq KK&&y12	3 JJ$$[+>'	! 	3 CDD%%c&8&8-&)nn&*mmo~I (/'8'8I 695G5G585G5GILIq KK&&y12	3 RZZ7
9a 
+
+CD
9 
9 9+	3 	3	3 	3
9s    =B%L%B%L'5!L3L$'L0c                   |j                   S r   r   r   s         r#   "_select_and_scatter_add_shape_rulerS    s     
rO   c                   |j                   S r   r;  r   s         r#   %_select_and_scatter_add_sharding_rulerU    r=  rO   c                   | \  }}|\  }}	t        ||||||      }
~	t        |      t        j                  u r#t        j                  j	                  |
      }|
|fS t        ||||||      }|
|fS r   )r   typer   r   from_primal_value)r   r   r   r*   r+   r,   r~   rA   g_source	g_operandval_outtangent_outs               r#   _select_and_scatter_add_jvpr]  	  s     /&' (I#g{$5~' 	(^w||#,,009K
 
+	 *';(9!K 
+	rO   c          
        t        j                  |      rt        j                  |      rJ t        |       t        j                  u r!t        j                  |j
                        d gS dt        |      z  }t        | |||||||      }|d gS )Nr%   )r   r   rW  r   r   ra   r2   r   )	r   r~   rA   r   r*   r+   r,   r  source_ts	            r#   !_select_and_scatter_add_transposer`    s     
			'0F0Fw0OO	O	!WLL%t,,	%&	&$#Aw=N$2GT4I(
D	rO   c                   | \  }}|\  }}	t        d t        | |      D              }
t        j                  |||
      }t        j                  ||	|
      }d|z   }d|z   }d|z   }t	        ||||||      }|dfS )Nc              3  D   K   | ]  \  }}||j                   |     y wr   r   r    ar   s      r#   r$   z5_select_and_scatter_add_batch_rule.<locals>.<genexpr>*  )      #4! ggdm #r   r%   r   r   )r   r   r   r   r   )r   r   r   r*   r+   r,   r~   rA   s_bdimo_bdimr   rm   s               r#   "_select_and_scatter_add_batch_rulerh  %  s     !/&'.&&	 #3|Z+H # 
#$!!&&$7&""7FD9'...(.'>O .	9#	a-rO   select_and_scatter_addc                  | j                   }fd}|t        j                  k(  rt        j                  nt        j
                  }	|r|j                  }
|}t        j                  u rt        j                  nt        j                  }|D cg c]
  \  }}||df }}}t        j                  | ||      |      }|D cg c]  }d }}t        |||||| t        j                  |      |	      }|rQD cg c]  \  }}|	 }}}t        |
      D cg c]  \  \  }}}||z    }}}}t        j                  |||      }|S c c}}w c c}w c c}}w c c}}}w )Nc                (    j                  | |      S r   )r?   )r!   r"   r   s     r#   r0  z._select_and_scatter_add_impl.<locals>.<lambda>F  s    ((A. rO   r   r   )rZ   rt   bool_r   
bitwise_orrV   r8   r  rY   r]   r   r   r   r   r   slice)r~   rA   r   r*   r+   r,   expand_paddingrZ   r}   r   r$  original_paddingidentityr  r  r  r   rm   start_indicesr!  stop_indicess     `                  r#   _select_and_scatter_add_implrt  B  sM    ,,%.&#rxx/CNNSWW'MMM)4)@%%** (/0HRRQK0D0gggx5G&'!v'G'v(.'6	ii'	## (89HRR9M9034D4A1C D D}R!BF DL D
--]L
9C	* 1'
 :Ds   E 	E7EEF)ro  )multiple_resultscpu)platformgpuc                   | j                   |j                   k7  r1d}t        |j                  | j                   |j                               t        ||||||      S )NzLselect_and_gather_add tangents and operand shapes must match, got {} and {}.)r8   r   r   r   )	r   rA   r   r*   r+   r,   r-   r.   r   s	            r#   !_select_and_gather_add_shape_rulerz  g  sU     ^^w}}$C
CJJx~~w}}=
>>	) .'=
 rO   c                   | j                   |j                   k7  r0t        j                  d| j                    d|j                    d      t        ||||||      S )NzEselect_and_gather_add tangents and operand shardings must match, got z and .)r   r
   ShardingTypeErrorr   r   s           r#   $_select_and_gather_add_sharding_ruler~  r  sn     '***

 
 	  !w'7'7&8	;< < 
% .'=
 rO   @   )max_bitsc                    j                   \  }
} j                  \  }t        |t        j                        sJ |       |j
                  t        j                        t        j                        j                  |	k  sJ dz  |	k  }d  fd|rVt        j                     t        j                  dz     t        j                        fd}fdfd}not        j                  d       dz  t        j                        j                  z
  dz
  t        j                     xfd	}fd
fd}t        j                   u st        j"                  u sJ        t        j                   u rt$        j&                   nt$        j&                  }|j)                        }dfd}t        j*                   d| ||||      g | |       d      t        j                  d            gt        j                  d      g|g|||||      \  } |||      gS )Nr   c                V    t        j                  t        j                  ||             S Nr   )r   ir_constantrt   array)rZ   r!   s     r#   r0  z1_select_and_gather_add_lowering.<locals>.<lambda>  s    4++BHHQe,DE rO   c                X    t        j                   |j                  |       |d      S )Nr   )broadcast_dimensions)r   broadcast_in_dimrZ   )r!   rN  constr   s     r#   _broadcast_scalar_constz@_select_and_gather_add_lowering.<locals>._broadcast_scalar_const  s+      eHNNA&>!)68: :rO   c                   |j                        }|j                        }t        j                  t        j                  |      |       } t        j                  t        j                  |      |      }t        j
                  t        j                  |      |       } t        j
                  t        j                  |      |      }t        j                  |  |            } t        j                  | |      S r  )r1  r   bitcast_convertr   r?  convert
shift_leftor_)	rd  bab_avalword_type_ab_avaldouble_word_type_ab_avalr  double_word_dtypenbits
word_dtypes	        r#   packz-_select_and_gather_add_lowering.<locals>.pack  s    !..z.:!(6G!H


d223DEq
Ia


d223DEq
Ia
++d**+CDa
Ha
++d**+CDa
Ha
..
$U,DEGaWWQ]rO   c           	     `   t        j                  | j                        j                  rJ t	        j
                  |              }t	        j                  t         j                  j                  g       t	        j                  t         j                  j                  g       |            S r   )	r   RankedTensorTyperW  r8   r   shift_right_logicalr  rC  r  )r   str  r  etyper  	word_types     r#   fstz,_select_and_gather_add_lowering.<locals>.fst  s    $$QVV,2222""1e,=u&EFb  



!
!"e
,
++b))--b)<b
AC CrO   c           
         t        j                  t        j                  |j	                              t        j
                  t        j                  |j	                              |             S r  )r   r  r   r?  r1  r  )r   t_avalrZ   r  s     r#   sndz,_select_and_gather_add_lowering.<locals>.snd  sU      


v}}5}9
:
++d**6==z=+JKQ
OQ QrO   zUsing reduced precision for gradient of reduce-window min/max operator to work around missing XLA support for pair-reductions. This is likely from a second or higher derivative of a max-pooling operation.r&   c                *   |j                        }t        j                  | t        j                        t        j                              } t        j                  |t        j                        t        j                              }t        j
                  t        j                  |      |       } t        j
                  t        j                  |      |      }t        j                  | |            }t        j                  | |      S )Nr   )exponent_bitsmantissa_bits)	r1  r   reduce_precisionr   i32_attrr  r?  r  r  )	rd  r  r  r  r  nexpnmantr_nbitsr  s	       r#   r  z-_select_and_gather_add_lowering.<locals>.pack  s    !..z.:


qd0C.2mmE.BDa


qd0C.2mmE.BDa


d223DEq
Ia


d223DEq
Ia

!
!
$W.?@BaWWQ]rO   c                   t        j                  | j                        j                  rJ t	        j
                  |  dz  dz
  z              }t	        j                  t         j                  j                  g       |      S )Nr&   )r   r  rW  r8   r   and_r  rC  )r   r  r  r  r  r  s     r#   r  z,_select_and_gather_add_lowering.<locals>.fst  sl    $$QVV,222288AuZ1<1*<)HIJb  !4!4!8!8U!CRHHrO   c                    t        j                  t        j                  |j	                              t        j
                  |  |j	                                          S r  )r   r  r   r?  r1  r  )r   r  r  rZ   r  r  s     r#   r  z,_select_and_gather_add_lowering.<locals>.snd  sS      


v}}5}9
:
..3GV]]Q[]=\]
^` `rO   r   c                   | j                   \  }}t        j                  u st        j                  u sJ t        j                  u rdnd}t	        j
                  t        j                   |       |      |      ||      }|S )NGELE)r   r   r  r  r   SelectOpr   compare_hlo)r   r!   r"   cmp_oprm   r  r   s        r#   r   z5_select_and_gather_add_lowering.<locals>.reducer_body  sq     DAq#(("kSXX&=== CHH,T$F
,,t''AA?A
FCJrO   #reduce_window_select_and_gather_addr   r   r   r   )r   r   r3   r
   r   rZ   r   dtype_to_ir_typer   finfobitsr   _UINT_DTYPESwarningswarnr  r  r  rt   ru   r1  rR   ) r   r   rA   r   r*   r+   r,   r-   r.   r  r   r3  out_avaldouble_word_reductionr  r  initdouble_word_out_avalr   resr  r  r  rZ   r  r  r  r  r  r  r  r  s    `  `                @@@@@@@@@@@@r#   _select_and_gather_add_loweringr  }  s9   
 \\!\mm)(	L$"2"2	3A\A	3


%



&%
,,u

"
"%	(		!)x/
E%:
  !!%(J((3%%j1I	C CQ MM B C qjG<<##DdNQE%(%5%5e%<<

 
I`
 
	 K388$;H[H	;388+"&&$!/@A 
		C8Wh56eT*E%OT=M=MbRW=XYZ))".?@A%&)#!%
$# c'
(	))rO   c          
     |   fd}t         j                  u st         j                  u sJ        t         j                  u rt        j                   nt        j                  }	t        || ft        j                  |	|j                        t        j                  d|j                        f||||||      \  }
}|S )Nc                    | \  }}|\  }}j                  ||      }t        j                  |||      t        j                  |||      fS r   )r?   r   r}   )r!   r"   kxvxkyvywhichr   s          r#   r   zC_select_and_gather_add_using_variadic_reducewindow.<locals>.reducer  sL    FBFBR$EJJub"%szz%R'@AArO   r   r   )r   r  r  rt   ru   rR   r  rZ   )r   rA   r   r*   r+   r,   r-   r.   r   r  r   rm   s     `         r#   2_select_and_gather_add_using_variadic_reducewindowr    s    B 
	 K388$;H[H	;388+"&&$hXXd'--("((1GMM*JK	&!S
 
*rO   c          
         | \  }}	|\  }
}t        ||	||||||      }~t        |
      t        j                  u r#t        j                  j	                  |      }||fS t        |
|	||||||      }||fS r   )r   rW  r   r   rX  )r   r   r   r*   r+   r,   r-   r.   r~   rA   rY  rZ  r[  r\  s                 r#   _select_and_gather_add_jvpr    s     /&' (I"g{$5~}o/' 	(^w||#,,009K
 
+	 )';(9AK 
+	rO   c                  |t         j                  t         j                  fv sJ t        j                  |      rt        j                  |      rJ t        d |D              rd}	t        |	j                  |            t        |       t        j                  u r!t        j                  |j                        d gS t        d |D              }
|
ri|t         j                  u rt         j                  nt         j                  }t        j                  | ||j                        t!        d |D                    }t#        | |||||      }|
r8t%        j&                  |dt)        |j*                        z  |j*                  |      }|d gS )Nc              3  &   K   | ]	  }|d k7    ywr&   Nr   r    r!  s     r#   r$   z3_select_and_gather_add_transpose.<locals>.<genexpr>"  s     )Aa)   zaVJP not implemented for select_and_gather (MaxPool) with window dilation, got window_dilation={}.c              3  &   K   | ]	  }|d k7    ywr  r   r  s     r#   r$   z3_select_and_gather_add_transpose.<locals>.<genexpr>(  s     8Q!q&8r  c              3  ,   K   | ]  }d d |dz
  f  yw)r   r&   Nr   r  s     r#   r$   z3_select_and_gather_add_transpose.<locals>.<genexpr>-  s     AaQ1q5MAs   r   )r   r  r  r   r   r   r   r   rW  r   r   ra   rY   r]   r   rZ   r6   r   r   rn  r2   r8   )r   r   rA   r   r*   r+   r,   r-   r.   r   has_base_dilationselect_identityr  s                r#    _select_and_gather_add_transposer    sD    
388,	,,	,

 
 
*$$W-/ .)))/C
cjj9
::	!WLL'..8-880;sxx0Gs,,11 gggw}}=A=AACG"1g{<M#17<&]]64#fll*;#;V\\(*F
$rO   c          
        | \  }}	|\  }
}t        d t        | |      D              }t        j                  ||
|      }t        j                  |	||      }	d|z   }d|z   }d|z   }d|z   }d|z   }t	        ||	||||||      }|dfS )Nc              3  D   K   | ]  \  }}||j                   |     y wr   r   rc  s      r#   r$   z7_select_and_gather_add_batching_rule.<locals>.<genexpr>:  re  r   r%   r   r   )r   r   r   r   r   )r   r   r   r*   r+   r,   r-   r.   r   r!   t_bdimx_bdimr   rm   s                 r#   $_select_and_gather_add_batching_ruler  5  s     
$!Q.&&	 #3|Z+H # 
#$Q-!Q-!...(.'&-?*/q![2C-w.	0# q/rO   select_and_gather_add)NN)
r*   
core.Shaper+   Sequence[int] | Noner,   str | Sequence[tuple[int, int]]r-   r  r.   r  )NVALIDNN)rA   r   rB   r   rC   r   r*   r  r+   r  r,   r  r-   r  r.   r  r   r   )r`   zSequence[Array]r   zCallable | None)rA   r   r*   r  r+   Sequence[int]r,   Sequence[tuple[int, int]]r-   r  r.   r  r   r   )rA   r   r}   r   r*   r  r+   r  r,   r  r~   r   rB   r   r   r   r   r   )r~   r   rA   r   r   core.Primitiver*   r  r+   r  r,   r  r   r   )r   r   rA   r   r   r  r*   r  r+   r  r,   r  r-   r  r.   r  r   r   )r   mlir.LoweringRuleContext)r   r  rx   
core.Jaxprrz   r  )q
__future__r   collections.abcr   r   	functoolsr   typingr   r  jax._srcr   r	   r
   r   r   r   r   jax._src.corer   r   r   jax._src.interpretersr   r   r   jax._src.laxr   r   r   jax._src.lax.otherr   jax._src.lib.mlirr   jax._src.lib.mlir.dialectsr   jax._src.typingr   numpyrt   safe_mapr:   safe_zipr   rN   rR   r9   rW   rn   r[   r^   rv   r   r   r   r   r   	Primitiver>   r   primitive_jvpsru  def_implapply_primitivedef_abstract_evalprimitive_batchersr   register_loweringr   r	  r  r   standard_primitiveinput_dtyper   re   
deflinear2r  r   r  rp   defjvpr  rr   r  _reduce_window_min_batch_ruler5  rV   min_hlor]   max_hlorY   r9  r<  r|   rQ  rS  rU  r]  r`  rh  r   primitive_transposesrt  	lower_funrz  r~  r  r  r  r  r  r   r   rO   r#   <module>r     s}   # .           @ @ $ * & $    (   * ! 
mm
mm +/,0B8 "	B8
 )B8 -B8 (B8 *B8T ,0/6*.,0<<< < "	<
 )< -< (< *< 	<~". >B?C	.'4. 9. ';. )=	. IN	.$ ?C@D	(5!: (< *>	 JO	, >B?C	.'4. 9. ';. )=	. IN	.$ >B?C	.'4. 9. ';. )=	. IN	.& +/,0'1! ' (	
 *
 6;(D+5D(5D ";D EJD %*	D 5=	D BG	D"	D)7	D/9	D -:	D &?		D DI		D$.(6$..8$. ,9$. %>	$.
 +8$. -:$. ?D$.N(B"#J !$..1(!T &7  / "#'     !9!9?K L  ! !"C D/P  O ,#	!#L   (D EJ&$+ -c,,!3??4G-T++-@AC  !#D E3:/41  / 01$4N 6:.2X -c,,$coo7J-T++-@AC  
		
w'F'*yy 2 33:/41  / 0 -c,,$coo7J-T++-@AC  
		
w'F'*yy 2 3 !((A(:!< 3:/41  / 0 F   *G#'';-0 1   *G$,,(=(=-? @   *G$,,(=(=-? @

 .s--"COO5I3T++-ABD 
*9	!*9*9 *9X   +-F G

"	" 2311&7T++-EF	H  &   0 1.I  * +& 	  4 50   /(?2    /(>2&+-   /(>2&+-
		 	y*	!y*x$"2( 1#00%s+OT++-DEG  .H  ) *"   / 0 ' 	  3 4   .61    #rO   