ukiL xUddlmZddlmZmZddlZddlZddlmZddl Z ddl m Z ddl Z ddl ZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZddlmZddlmZ ddl!m"Z"ddl!m#Z$ddl%m&Z&m'Z'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z1ddl2m3Z3m4Z4m5Z5ddl6m7Z7ddl8m9Z9m:Z:e9e;cZ;Zej~Z@eAZB d ddZAdd dd ZCdd dd!ZDGd"d#e ZEGd$d%ejZGd&d&ddd' dd(ZHGd)d*e ZIeed+,d&d&dd- dd.ZJeed/,d&d&dd- dd0ZKeed1,d&d&dd- dd2ZLeed3,d&d&dd- dd4ZMeed5,d&d&dd- dd6ZNe jZPd7eQd8<d9d&d&dd: dd;ZRd<ZSd&d&dd- dd=ZTdd>ZU d dd?ZV d dd@ZW ddd ddAZX ddd ddBZYdd ddCZZdd ddDZ[dEZ\dFZ]dGZ^dHZ_dIZ`dJZadKZbe(e\e'dLe_eejdLMZdejede`eaejed<ebejed<edjdNZidOZjejedejdPZldQZmdRZndSZodTZpdUZqdVZrdWZsdXZtdYZudZZve(eleod[e&demeejd[en\Zwepejew<eqejew<erejew<etejew<eue jew<evejew<d]Z|ejewe|d^Z}d_Z~d`ZdaZdbZdcZddZdeZe(e}edfe~eejdfeegZeeje<eeje<eeje<dhZejeediZdjZdkZdlZdmZdnZdoZdpZdqZdrZddsZdtZduZdvZdwZdxZe(eedye&deeejdyzZej0eedeeje<eeje< dd{Zd|Zejeed}Zd~ZdZddZ ddZdZdZdZdZdZdZdZe(eede&deeejdeZeeeeje<eeeeje<eeeejPe<dejRe<e(eede&deeejdeZeeeeje<eeeeje<eeeejPe<dejRe<e(eede&deeejdeZdZej0eddeeeje<eeeejPe<dejRe<dZe(eede&deeejdeZeeeejPe<dejRe<eeeeje<e(eede&deeejdeZeeeejPe<dejRe<eeeeje<dZdZe(eede&deeejdeZeeje<eeje<eeeejPe<dejRe<dZ ddZejeeejeeejeeejeeejeeejeedZdZejeeee,jndejeeee,jpd ddZy)) annotations)CallableSequenceN)partial) NamedTuple)ad_util)api)core)dispatch)dtypes)source_info_util) api_boundary)util)mesh)ad)batching)mlir) partial_eval)lax)utils)_argnum_weak_type input_dtypestandard_primitive)ir)hlo) NamedSharding) PartitionSpec)Array ArrayLikeShape)ds)safe_mapsafe_zipc|tj|t|t||dSt|S)aWraps XLA's `Slice `_ operator. Args: operand: an array to slice start_indices: a sequence of ``operand.ndim`` start indices. limit_indices: a sequence of ``operand.ndim`` limit indices. strides: an optional sequence of ``operand.ndim`` strides. Returns: The sliced array Examples: Here are some examples of simple two-dimensional slices: >>> x = jnp.arange(12).reshape(3, 4) >>> x Array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]], dtype=int32) >>> lax.slice(x, (1, 0), (3, 2)) Array([[4, 5], [8, 9]], dtype=int32) >>> lax.slice(x, (0, 0), (3, 4), (1, 2)) Array([[ 0, 2], [ 4, 6], [ 8, 10]], dtype=int32) These two examples are equivalent to the following Python slicing syntax: >>> x[1:3, 0:2] Array([[4, 5], [8, 9]], dtype=int32) >>> x[0:3, 0:4:2] Array([[ 0, 2], [ 4, 6], [ 8, 10]], dtype=int32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.slice_in_dim` - :func:`jax.lax.index_in_dim` - :func:`jax.lax.dynamic_slice` N start_indices limit_indicesstrides)slice_pbindtuple)operandr&r'r(s O/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/lax/slicing.pyslicer.>sMf gU=-A$)-$8&-od  KK;@>  KKTallow_negative_indicesct|||}tj|}tj|g|^}}t j |g|dt |iS)aWraps XLA's `DynamicSlice `_ operator. Args: operand: an array to slice. start_indices: a list of scalar indices, one per dimension. These values may be dynamic. slice_sizes: the size of the slice. Must be a sequence of non-negative integers with length equal to `ndim(operand)`. Inside a JIT compiled function, only static values are supported (all JAX arrays inside JIT must have statically known size). allow_negative_indices: a bool or sequence of bools, one per dimension; if a bool is passed, it applies to all dimensions. For each dimension, if true, negative indices are permitted and are are interpreted relative to the end of the array. If false, negative indices are treated as if they were out of bounds and the result is implementation defined, typically clamped to the first index. Returns: An array containing the slice. Examples: Here is a simple two-dimensional dynamic slice: >>> x = jnp.arange(12).reshape(3, 4) >>> x Array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]], dtype=int32) >>> dynamic_slice(x, (1, 1), (2, 3)) Array([[ 5, 6, 7], [ 9, 10, 11]], dtype=int32) Note the potentially surprising behavior for the case where the requested slice overruns the bounds of the array; in this case the start index is adjusted to return a slice of the requested size: >>> dynamic_slice(x, (1, 1), (2, 4)) Array([[ 4, 5, 6, 7], [ 8, 9, 10, 11]], dtype=int32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.slice` - :func:`jax.lax.dynamic_slice_in_dim` - :func:`jax.lax.dynamic_index_in_dim` slice_sizes)_dynamic_slice_indicesr canonicalize_shapestandard_insert_pvarydynamic_slice_pr*r+)r,r&r3r1sizess r- dynamic_slicer9vs`p) }46-  ! !+ .% 66wOO'M   g P P5< PPr/c~t|||}tj||g|^}}}tj||g|S)aWraps XLA's `DynamicUpdateSlice `_ operator. Args: operand: an array to slice. update: an array containing the new values to write onto `operand`. start_indices: a list of scalar indices, one per dimension. allow_negative_indices: a bool or sequence of bools, one per dimension; if a bool is passed, it applies to all dimensions. For each dimension, if true, negative indices are permitted and are are interpreted relative to the end of the array. If false, negative indices are treated as if they were out of bounds and the result is implementation defined, typically clamped to the first index. Returns: An array containing the slice. Examples: Here is an example of updating a one-dimensional slice update: >>> x = jnp.zeros(6) >>> y = jnp.ones(3) >>> dynamic_update_slice(x, y, (2,)) Array([0., 0., 1., 1., 1., 0.], dtype=float32) If the update slice is too large to fit in the array, the start index will be adjusted to make it fit >>> dynamic_update_slice(x, y, (3,)) Array([0., 0., 0., 1., 1., 1.], dtype=float32) >>> dynamic_update_slice(x, y, (5,)) Array([0., 0., 0., 1., 1., 1.], dtype=float32) Here is an example of a two-dimensional slice update: >>> x = jnp.zeros((4, 4)) >>> y = jnp.ones((2, 2)) >>> dynamic_update_slice(x, y, (1, 2)) Array([[0., 0., 0., 0.], [0., 0., 1., 1.], [0., 0., 1., 1.], [0., 0., 0., 0.]], dtype=float32) See Also: - :attr:`jax.numpy.ndarray.at` - :attr:`lax.dynamic_update_index_in_dim` - :attr:`lax.dynamic_update_slice_in_dim` )r4r r6dynamic_update_slice_pr*)r,updater&r1s r-dynamic_update_slicer=sSn) }46-$($>$> v%'%%'!'6M  $ $Wf E} EEr/cLeZdZUdZded<ded<ded<dZded<dZded<y ) GatherDimensionNumbersa# Describes the dimension number arguments to an `XLA's Gather operator `_. See the XLA documentation for more details of what the dimension numbers mean. Args: offset_dims: the set of dimensions in the `gather` output that offset into an array sliced from `operand`. Must be a tuple of integers in ascending order, each representing a dimension number of the output. collapsed_slice_dims: the set of dimensions `i` in `operand` that have `slice_sizes[i] == 1` and that should not have a corresponding dimension in the output of the gather. Must be a tuple of integers in ascending order. start_index_map: for each dimension in `start_indices`, gives the corresponding dimension in the `operand` that is to be sliced. Must be a tuple of integers with size equal to `start_indices.shape[-1]`. operand_batching_dims: the set of batching dimensions `i` in `operand` that have `slice_sizes[i] == 1` and that should have a corresponding dimension in both the `start_indices` (at the same index in `start_indices_batching_dims`) and output of the gather. Must be a tuple of integers in ascending order. start_indices_batching_dims: the set of batching dimensions `i` in `start_indices` that should have a corresponding dimension in both the `operand` (at the same index in `operand_batching_dims`) and output of the gather. Must be a tuple of integers (order is fixed based on correspondence with `operand_batching_dims`). Unlike XLA's `GatherDimensionNumbers` structure, `index_vector_dim` is implicit; there is always an index vector dimension and it must always be the last dimension. To gather scalar indices, add a trailing dimension of size 1. ztuple[int, ...] offset_dimscollapsed_slice_dimsstart_index_mapoperand_batching_dimsstart_indices_batching_dimsN)__name__ __module__ __qualname____doc____annotations__rDrErCr/r-r?r?s1>''""+--133r/r?ceZdZdZej Zej Zej Zej Z e ddZ y)GatherScatterModea^ Describes how to handle out-of-bounds indices in a gather or scatter. Possible values are: CLIP: Indices will be clamped to the nearest in-range value, i.e., such that the entire window to be gathered is in-range. FILL_OR_DROP: If any part of a gathered window is out of bounds, the entire window that is returned, even those elements that were otherwise in-bounds, will be filled with a constant. If any part of a scattered window is out of bounds, the entire window will be discarded. PROMISE_IN_BOUNDS: The user promises that indices are in bounds. No additional checking will be performed. In practice, with the current XLA implementation this means that out-of-bounds gathers will be clamped but out-of-bounds scatters will be discarded. Gradients will not be correct if indices are out-of-bounds. ct|tr|S|dk(rtjS| |dk(s|dk(rtjS|dk(rtjS|dk(rtj St d|d)Nclipfilldroppromise_in_boundsone_hotzUnknown gather mode "") isinstancerLCLIP FILL_OR_DROPPROMISE_IN_BOUNDSONE_HOT ValueError)ss r-from_anyzGatherScatterMode.from_any4s!&' hF{  # ##yAK1;  + ++   0 00I~  & && .qc3 44r/N)rZstr | GatherScatterMode | NonereturnrL) rFrGrHrIenumautorUrVrWrX staticmethodr[rCr/r-rLrLsP( $,diik DIIK' 5 5r/rLFunique_indicesindices_are_sortedmode fill_valuec |tj}tj|}|tjk(r3|2t |} t j | tjrtj}nt j | tjr t j| j}nt j | tjr t j| j}nj| t jk(rd}nTt j | t j r tjdj}nt#d| d}t%j&||\}}t(j+|||t%j,|t/|t/|||S)aZ Gather operator. Wraps `XLA's Gather operator `_. :func:`gather` is a low-level operator with complicated semantics, and most JAX users will never need to call it directly. Instead, you should prefer using `Numpy-style indexing`_, and/or :func:`jax.numpy.ndarray.at`, perhaps in combination with :func:`jax.vmap`. Args: operand: an array from which slices should be taken start_indices: the indices at which slices should be taken dimension_numbers: a `lax.GatherDimensionNumbers` object that describes how dimensions of `operand`, `start_indices` and the output relate. slice_sizes: the size of each slice. Must be a sequence of non-negative integers with length equal to `ndim(operand)`. indices_are_sorted: whether `indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements gathered from ``operand`` are guaranteed not to overlap with each other. If ``True``, this may improve performance on some backends. JAX does not check this promise: if the elements overlap the behavior is undefined. mode: how to handle indices that are out of bounds: when set to ``'clip'``, indices are clamped so that the slice is within bounds, and when set to ``'fill'`` or ``'drop'`` gather returns a slice full of ``fill_value`` for the affected slice. The behavior for out-of-bounds indices when set to ``'promise_in_bounds'`` is implementation-defined. fill_value: the fill value to return for out-of-bounds slices when `mode` is ``'fill'``. Ignored otherwise. Defaults to ``NaN`` for inexact types, the largest negative value for signed types, the largest positive value for unsigned types, and ``True`` for booleans. Returns: An array containing the gather output. Examples: As mentioned above, you should basically never use :func:`gather` directly, and instead use NumPy-style indexing expressions to gather values from arrays. For example, here is how you can extract values at particular indices using straightforward indexing semantics, which will lower to XLA's Gather operator: >>> import jax.numpy as jnp >>> x = jnp.array([10, 11, 12]) >>> indices = jnp.array([0, 1, 1, 2, 2, 2]) >>> x[indices] Array([10, 11, 11, 12, 12, 12], dtype=int32) For control over settings like ``indices_are_sorted``, ``unique_indices``, ``mode``, and ``fill_value``, you can use the :attr:`jax.numpy.ndarray.at` syntax: >>> x.at[indices].get(indices_are_sorted=True, mode="promise_in_bounds") Array([10, 11, 11, 12, 12, 12], dtype=int32) By comparison, here is the equivalent function call using :func:`gather` directly, which is not something typical users should ever need to do: >>> from jax import lax >>> lax.gather(x, indices[:, None], slice_sizes=(1,), ... dimension_numbers=lax.GatherDimensionNumbers( ... offset_dims=(), ... collapsed_slice_dims=(0,), ... start_index_map=(0,)), ... indices_are_sorted=True, ... mode=lax.GatherScatterMode.PROMISE_IN_BOUNDS) Array([10, 11, 11, 12, 12, 12], dtype=int32) .. _Numpy-style indexing: https://numpy.org/doc/stable/reference/arrays.indexing.html NTuint32z(Unsupported dtype for gather fill_value dimension_numbersr3rbrcrdre)rLrWr[rV_dtyper issubdtypenpinexactnan signedintegeriinfominunsignedintegermaxbool_prng_keyrYr r6gather_pr*r5bool) r,r&rir3rbrcrdre parsed_modedtypes r-gatherrzDs_` \  . .D!**40+%222Woe   5"** -VV   UB$4$4 5\\%(,,   UB$6$6 7\\%(,, FLL    UFOO 4XXh'++ CE7KLLJ55g}M'=  }0A))+6.)01   r/cLeZdZUdZded<ded<ded<dZded<dZded<y ) ScatterDimensionNumbersa Describes the dimension number arguments to an `XLA's Scatter operator `_. See the XLA documentation for more details of what the dimension numbers mean. Args: update_window_dims: the set of dimensions in the `updates` that are window dimensions. Must be a tuple of integers in ascending order, each representing a dimension number. inserted_window_dims: the set of size 1 window dimensions that must be inserted into the shape of `updates`. Must be a tuple of integers in ascending order, each representing a dimension number of the output. These are the mirror image of `collapsed_slice_dims` in the case of `gather`. scatter_dims_to_operand_dims: for each dimension in `scatter_indices`, gives the corresponding dimension in `operand`. Must be a sequence of integers with size equal to `scatter_indices.shape[-1]`. operand_batching_dims: the set of batching dimensions `i` in `operand` that should have a corresponding dimension in both the `scatter_indices` (at the same index in `scatter_indices_batching_dims`) and `updates`. Must be a tuple of integers in ascending order. These are the mirror image of `operand_batching_dims` in the case of `gather`. scatter_indices_batching_dims: the set of batching dimensions `i` in `scatter_indices` that should have a corresponding dimension in both the `operand` (at the same index in `operand_batching_dims`) and output of the gather. Must be a tuple of integers (order is fixed based on correspondence with `input_batching_dims`). These are the mirror image of `start_indices_batching_dims` in the case of `gather`. Unlike XLA's `ScatterDimensionNumbers` structure, `index_vector_dim` is implicit; there is always an index vector dimension and it must always be the last dimension. To scatter scalar indices, add a trailing dimension of size 1. Sequence[int]update_window_dimsinserted_window_dimsscatter_dims_to_operand_dimsrCrDscatter_indices_batching_dimsN)rFrGrHrIrJrDrrCr/r-r|r|s2@$#%% --)++133r/r|zlax.scatter_add)repro_api_namercrbrdc 6tjtjtjtj |d\}}tj |||\}}}tj||||||||tj| S)aO Scatter-add operator. Wraps `XLA's Scatter operator `_, where addition is used to combine updates and values from `operand`. The semantics of scatter are complicated, and its API might change in the future. For most use cases, you should prefer the :attr:`jax.numpy.ndarray.at` property on JAX arrays which uses the familiar NumPy indexing syntax. Args: operand: an array to which the scatter should be applied scatter_indices: an array that gives the indices in `operand` to which each update in `updates` should be applied. updates: the updates that should be scattered onto `operand`. dimension_numbers: a `lax.ScatterDimensionNumbers` object that describes how dimensions of `operand`, `scatter_indices`, `updates` and the output relate. indices_are_sorted: whether `scatter_indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements to be updated in ``operand`` are guaranteed to not overlap with each other. If true, may improve performance on some backends. JAX does not check this promise: if the updated elements overlap when ``unique_indices`` is ``True`` the behavior is undefined. mode: how to handle indices that are out of bounds: when set to 'clip', indices are clamped so that the slice is within bounds, and when set to 'fill' or 'drop' out-of-bounds updates are dropped. The behavior for out-of-bounds indices when set to 'promise_in_bounds' is implementation-defined. Returns: An array containing the sum of `operand` and the scattered updates. Examples: As mentioned above, you should basically never use :func:`scatter_add` directly, and instead perform scatter-style operations using NumPy-style indexing expressions via :attr:`jax.numpy.ndarray.at`. Here is and example of updating entries in an array using :attr:`jax.numpy.ndarray.at`, which lowers to an XLA Scatter operation: >>> x = jnp.ones(5) >>> indices = jnp.array([1, 2, 4]) >>> values = jnp.array([2.0, 3.0, 4.0]) >>> x.at[indices].add(values) Array([1., 3., 4., 1., 5.], dtype=float32) This syntax also supports several of the optional arguments to :func:`scatter_add`, for example: >>> x.at[indices].add(values, indices_are_sorted=True, ... mode='promise_in_bounds') Array([1., 3., 4., 1., 5.], dtype=float32) By comparison, here is the equivalent function call using :func:`scatter_add` directly, which is not something typical users should ever need to do: >>> lax.scatter_add(x, indices[:, None], values, ... dimension_numbers=lax.ScatterDimensionNumbers( ... update_window_dims=(), ... inserted_window_dims=(0,), ... scatter_dims_to_operand_dims=(0,)), ... indices_are_sorted=True, ... mode=lax.GatherScatterMode.PROMISE_IN_BOUNDS) Array([1., 3., 4., 1., 5.], dtype=float32) r update_jaxpr update_constsrircrbrd) r_reduction_jaxpraddr get_aval_constr6 scatter_add_pr*rLr[ r,scatter_indicesupdatesrircrbrdjaxprconstss r- scatter_addrsX&&sww'+}}SZZ5K'LN-%&*&@&@ ')#'?G    e.?+N  % %d +  --r/zlax.scatter_subc 6tjtjtjtj |d\}}tj |||\}}}tj||||||||tj| S)aScatter-sub operator. Wraps `XLA's Scatter operator `_, where subtraction is used to combine updates and values from `operand`. The semantics of scatter are complicated, and its API might change in the future. For most use cases, you should prefer the :attr:`jax.numpy.ndarray.at` property on JAX arrays which uses the familiar NumPy indexing syntax. Args: operand: an array to which the scatter should be applied scatter_indices: an array that gives the indices in `operand` to which each update in `updates` should be applied. updates: the updates that should be scattered onto `operand`. dimension_numbers: a `lax.ScatterDimensionNumbers` object that describes how dimensions of `operand`, `start_indices`, `updates` and the output relate. indices_are_sorted: whether `scatter_indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements to be updated in ``operand`` are guaranteed to not overlap with each other. If true, may improve performance on some backends. JAX does not check this promise: if the updated elements overlap when ``unique_indices`` is ``True`` the behavior is undefined. mode: how to handle indices that are out of bounds: when set to 'clip', indices are clamped so that the slice is within bounds, and when set to 'fill' or 'drop' out-of-bounds updates are dropped. The behavior for out-of-bounds indices when set to 'promise_in_bounds' is implementation-defined. Returns: An array containing the difference between `operand` and the scattered updates. rr) rrsubr rrr6 scatter_sub_pr*rLr[rs r- scatter_subr/s\&& ggt}}SZZ34-%'+&@&@ ')#'?G     )+#  % %d +   r/zlax.scatter_mulc 6tjtjtjtj |d\}}tj |||\}}}tj||||||||tj| S)aScatter-multiply operator. Wraps `XLA's Scatter operator `_, where multiplication is used to combine updates and values from `operand`. The semantics of scatter are complicated, and its API might change in the future. For most use cases, you should prefer the :attr:`jax.numpy.ndarray.at` property on JAX arrays which uses the familiar NumPy indexing syntax. Args: operand: an array to which the scatter should be applied scatter_indices: an array that gives the indices in `operand` to which each update in `updates` should be applied. updates: the updates that should be scattered onto `operand`. dimension_numbers: a `lax.ScatterDimensionNumbers` object that describes how dimensions of `operand`, `start_indices`, `updates` and the output relate. indices_are_sorted: whether `scatter_indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements to be updated in ``operand`` are guaranteed to not overlap with each other. If true, may improve performance on some backends. JAX does not check this promise: if the updated elements overlap when ``unique_indices`` is ``True`` the behavior is undefined. mode: how to handle indices that are out of bounds: when set to 'clip', indices are clamped so that the slice is within bounds, and when set to 'fill' or 'drop' out-of-bounds updates are dropped. The behavior for out-of-bounds indices when set to 'promise_in_bounds' is implementation-defined. Returns: An array containing the product of `operand` and the scattered updates. r) rrmulr rrr6 scatter_mul_pr*rLr[rs r- scatter_mulroP&&sww'+}}SZZ5K'LN-%&*&@&@ ')#'?G    e.?+N  % %d +  --r/zlax.scatter_minc 6tjtjtjtj |d\}}tj |||\}}}tj||||||||tj| S)aScatter-min operator. Wraps `XLA's Scatter operator `_, where the `min` function is used to combine updates and values from `operand`. The semantics of scatter are complicated, and its API might change in the future. For most use cases, you should prefer the :attr:`jax.numpy.ndarray.at` property on JAX arrays which uses the familiar NumPy indexing syntax. Args: operand: an array to which the scatter should be applied scatter_indices: an array that gives the indices in `operand` to which each update in `updates` should be applied. updates: the updates that should be scattered onto `operand`. dimension_numbers: a `lax.ScatterDimensionNumbers` object that describes how dimensions of `operand`, `start_indices`, `updates` and the output relate. indices_are_sorted: whether `scatter_indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements to be updated in ``operand`` are guaranteed to not overlap with each other. If true, may improve performance on some backends. JAX does not check this promise: if the updated elements overlap when ``unique_indices`` is ``True`` the behavior is undefined. mode: how to handle indices that are out of bounds: when set to 'clip', indices are clamped so that the slice is within bounds, and when set to 'fill' or 'drop' out-of-bounds updates are dropped. The behavior for out-of-bounds indices when set to 'promise_in_bounds' is implementation-defined. Returns: An array containing the min of `operand` and the scattered updates. rr) rrrqr rrr6 scatter_min_pr*rLr[rs r- scatter_minrrr/zlax.scatter_maxc 6tjtjtjtj |d\}}tj |||\}}}tj||||||||tj| S)aScatter-max operator. Wraps `XLA's Scatter operator `_, where the `max` function is used to combine updates and values from `operand`. The semantics of scatter are complicated, and its API might change in the future. For most use cases, you should prefer the :attr:`jax.numpy.ndarray.at` property on JAX arrays which uses the familiar NumPy indexing syntax. Args: operand: an array to which the scatter should be applied scatter_indices: an array that gives the indices in `operand` to which each update in `updates` should be applied. updates: the updates that should be scattered onto `operand`. dimension_numbers: a `lax.ScatterDimensionNumbers` object that describes how dimensions of `operand`, `start_indices`, `updates` and the output relate. indices_are_sorted: whether `scatter_indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements to be updated in ``operand`` are guaranteed to not overlap with each other. If true, may improve performance on some backends. JAX does not check this promise: if the updated elements overlap when ``unique_indices`` is ``True`` the behavior is undefined. mode: how to handle indices that are out of bounds: when set to 'clip', indices are clamped so that the slice is within bounds, and when set to 'fill' or 'drop' out-of-bounds updates are dropped. The behavior for out-of-bounds indices when set to 'promise_in_bounds' is implementation-defined. Returns: An array containing the max of `operand` and the scattered updates. rr) rrrsr rrr6 scatter_max_pr*rLr[rs r- scatter_maxrrr/zweakref.WeakKeyDictionary_scatter_apply_cacherC) update_shapercrbrdc tj|d|j}fd} tj | } tj | tjtj|\} } tj|||\}}}tj|||| | |||tj| S#t $rYwxYw)aScatter-apply operator. Wraps `XLA's Scatter operator `_, where values from ``operand`` are replaced with ``func(operand)``, with duplicate indices resulting in multiple applications of ``func``. The semantics of scatter are complicated, and its API might change in the future. For most use cases, you should prefer the :attr:`jax.numpy.ndarray.at` property on JAX arrays which uses the familiar NumPy indexing syntax. Note that in the current implementation, ``scatter_apply`` is not compatible with automatic differentiation. Args: operand: an array to which the scatter should be applied scatter_indices: an array that gives the indices in `operand` to which each update in `updates` should be applied. func: unary function that will be applied at each index. dimension_numbers: a `lax.ScatterDimensionNumbers` object that describes how dimensions of `operand`, `start_indices`, `updates` and the output relate. update_shape: the shape of the updates at the given indices. indices_are_sorted: whether `scatter_indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements to be updated in ``operand`` are guaranteed to not overlap with each other. If true, may improve performance on some backends. JAX does not check this promise: if the updated elements overlap when ``unique_indices`` is ``True`` the behavior is undefined. mode: how to handle indices that are out of bounds: when set to 'clip', indices are clamped so that the slice is within bounds, and when set to 'fill' or 'drop' out-of-bounds updates are dropped. The behavior for out-of-bounds indices when set to 'promise_in_bounds' is implementation-defined. Returns: An array containing the result of applying `func` to `operand` at the given indices. rc|SNrC)x_funcs r-zscatter_apply..8s Qr/r)rfullryr setdefault TypeErrorrr r_zeror6 scatter_pr*rLr[) r,rrrirrcrbrdunused_applyrrs ` r- scatter_applyrs^ 88L!W]] 3& & ! , ,T6 :F&&vt}}SYYw=O/PQ-%%)%?%? &("'?F  U.?+N  % %d +  --   sC CCc|SrrC)rys r-rrHsar/c tj|||\}}}tj|||dd|||tj | S)a_ Scatter-update operator. Wraps `XLA's Scatter operator `_, where updates replace values from `operand`. If multiple updates are performed to the same index of operand, they may be applied in any order. :func:`scatter` is a low-level operator with complicated semantics, and most JAX users will never need to call it directly. Instead, you should prefer using :func:`jax.numpy.ndarray.at` for more familiary NumPy-style indexing syntax. Args: operand: an array to which the scatter should be applied scatter_indices: an array that gives the indices in `operand` to which each update in `updates` should be applied. updates: the updates that should be scattered onto `operand`. dimension_numbers: a `lax.ScatterDimensionNumbers` object that describes how dimensions of `operand`, `start_indices`, `updates` and the output relate. indices_are_sorted: whether `scatter_indices` is known to be sorted. If true, may improve performance on some backends. unique_indices: whether the elements to be updated in ``operand`` are guaranteed to not overlap with each other. If true, may improve performance on some backends. JAX does not check this promise: if the updated elements overlap when ``unique_indices`` is ``True`` the behavior is undefined. mode: how to handle indices that are out of bounds: when set to 'clip', indices are clamped so that the slice is within bounds, and when set to 'fill' or 'drop' out-of-bounds updates are dropped. The behavior for out-of-bounds indices when set to 'promise_in_bounds' is implementation-defined. Returns: An array containing the values of `operand` and the scattered updates. Examples: As mentioned above, you should basically never use :func:`scatter` directly, and instead perform scatter-style operations using NumPy-style indexing expressions via :attr:`jax.numpy.ndarray.at`. Here is and example of updating entries in an array using :attr:`jax.numpy.ndarray.at`, which lowers to an XLA Scatter operation: >>> x = jnp.ones(5) >>> indices = jnp.array([1, 2, 4]) >>> values = jnp.array([2.0, 3.0, 4.0]) >>> x.at[indices].set(values) Array([1., 2., 3., 1., 4.], dtype=float32) This syntax also supports several of the optional arguments to :func:`scatter`, for example: >>> x.at[indices].set(values, indices_are_sorted=True, mode='promise_in_bounds') Array([1., 2., 3., 1., 4.], dtype=float32) By comparison, here is the equivalent function call using :func:`scatter` directly, which is not something typical users should ever need to do: >>> lax.scatter(x, indices[:, None], values, ... dimension_numbers=lax.ScatterDimensionNumbers( ... update_window_dims=(), ... inserted_window_dims=(0,), ... scatter_dims_to_operand_dims=(0,)), ... indices_are_sorted=True, ... mode=lax.GatherScatterMode.PROMISE_IN_BOUNDS) Array([1., 2., 3., 1., 4.], dtype=float32) NrCr)r r6rr*rLr[)r,rrrircrbrds r-scatterrJs[T'+&@&@ ')#'?G  d*;+N  % %d +  --r/c ltj|Dcgc]}tj|dc}d}tjtj|Dcgc]}|j |c}t t|jdz }||z}t|j }|D]}d||< t td|j|j dz dz}t|t |t |} t||| t |Scc}wcc}w)Nrrr@rArBrir3) r concatenate expand_dimsrlarrayshaper+rangendimlistr?rz) srcidxsaxesiindicesaxmax_idxr3r@dnumss r- index_takers OOtD!S__Q5Da H' OOBHHd%Cciim%CD!% q(8"9: <' g 'SYY+ bKOeAsxx'--*::Q>?@+  ;Dk % W!+. 00E%Cs D,D1 cdg|jz}t|j}dg|jz}|j|}|tj|nd} |tj|n|} | dkr| |z} | dkr| |z} t |}| ||<| ||<tj|||<t ||||S)a Convenience wrapper around :func:`lax.slice` applying to only one dimension. This is effectively equivalent to ``operand[..., start_index:limit_index:stride]`` with the indexing applied on the specified axis. Args: operand: an array to slice. start_index: an optional start index (defaults to zero) limit_index: an optional end index (defaults to operand.shape[axis]) stride: an optional stride (defaults to 1) axis: the axis along which to apply the slice (defaults to 0) Returns: An array containing the slice. Examples: Here is a one-dimensional example: >>> x = jnp.arange(4) >>> lax.slice_in_dim(x, 1, 3) Array([1, 2], dtype=int32) Here are some two-dimensional examples: >>> x = jnp.arange(12).reshape(4, 3) >>> x Array([[ 0, 1, 2], [ 3, 4, 5], [ 6, 7, 8], [ 9, 10, 11]], dtype=int32) >>> lax.slice_in_dim(x, 1, 3) Array([[3, 4, 5], [6, 7, 8]], dtype=int32) >>> lax.slice_in_dim(x, 1, 3, axis=1) Array([[ 1, 2], [ 4, 5], [ 7, 8], [10, 11]], dtype=int32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.slice` - :func:`jax.lax.index_in_dim` - :func:`jax.lax.dynamic_slice_in_dim` rr)rrrr _canonicalize_dimensionintr.) r, start_index limit_indexstrideaxisr&r'r(len_axisstart_index_intlimit_index_ints r- slice_in_dimrsd# $-w}}%- C',, ']]4 (#/11+>56$/11+>5=q%0Oq%0O T$'-'-..v6'$- w }g >>r/c(tj|t|}}|j|}|dkr||zn|}d|cxkr|ks nd}t |j |||t |||dzd|}|r|Stj||fS)aConvenience wrapper around :func:`lax.slice` to perform int indexing. This is effectively equivalent to ``operand[..., index]`` with the indexing applied on the specified axis. Args: operand: an array to index. index: integer index axis: the axis along which to apply the index (defaults to 0) keepdims: boolean specifying whether the output array should preserve the rank of the input (default=True) Returns: The subarray at the specified index. Examples: Here is a one-dimensional example: >>> x = jnp.arange(4) >>> lax.index_in_dim(x, 2) Array([2], dtype=int32) >>> lax.index_in_dim(x, 2, keepdims=False) Array(2, dtype=int32) Here are some two-dimensional examples: >>> x = jnp.arange(12).reshape(3, 4) >>> x Array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]], dtype=int32) >>> lax.index_in_dim(x, 1) Array([[4, 5, 6, 7]], dtype=int32) >>> lax.index_in_dim(x, 1, axis=1, keepdims=False) Array([1, 5, 9], dtype=int32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.slice` - :func:`jax.lax.slice_in_dim` - :func:`jax.lax.dynamic_index_in_dim` rz2index {} is out of bounds for axis {} with size {}r) r rrr IndexErrorformatrrsqueeze)r,indexrkeepdims axis_size wrapped_indexmsgresults r- index_in_dimrs^,,U3SY%mmD!)',qy%)#e- m 'i ' >C SZZtY7 88  0A1d K& M ;;vw ''r/c tj|dg|jz}t|j}t |}dg|jz}|||<|||<t j|||<t||||S)a Convenience wrapper around :func:`lax.dynamic_slice` applied to one dimension. This is roughly equivalent to the following Python indexing syntax applied along the specified axis: ``operand[..., start_index:start_index + slice_size]``. Args: operand: an array to slice. start_index: the (possibly dynamic) start index slice_size: the static slice size axis: the axis along which to apply the slice (defaults to 0) allow_negative_indices: boolean specifying whether negative indices are allowed. If true, negative indices are taken relative to the end of the array. If false, negative indices are out of bounds and the result is implementation defined. Returns: An array containing the slice. Examples: Here is a one-dimensional example: >>> x = jnp.arange(5) >>> dynamic_slice_in_dim(x, 1, 3) Array([1, 2, 3], dtype=int32) Like `jax.lax.dynamic_slice`, out-of-bound slices will be clipped to the valid range: >>> dynamic_slice_in_dim(x, 4, 3) Array([2, 3, 4], dtype=int32) Here is a two-dimensional example: >>> x = jnp.arange(12).reshape(3, 4) >>> x Array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]], dtype=int32) >>> dynamic_slice_in_dim(x, 1, 2, axis=1) Array([[ 1, 2], [ 5, 6], [ 9, 10]], dtype=int32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.slice_in_dim` - :func:`jax.lax.dynamic_slice` - :func:`jax.lax.dynamic_index_in_dim` rFr0) rrrrrrr rr9)r,r slice_sizerr1r&r3allow_negatives r-dynamic_slice_in_dimr7sl%(JJ{A$>#?',,#N-W]]#+ T$7W\\)./.#-22:>+d w {.< >>r/cXt||d||}|r|Stj||fS)aConvenience wrapper around dynamic_slice to perform int indexing. This is roughly equivalent to the following Python indexing syntax applied along the specified axis: ``operand[..., index]``. Args: operand: an array to slice. index: the (possibly dynamic) start index axis: the axis along which to apply the slice (defaults to 0) keepdims: boolean specifying whether the output should have the same rank as the input (default = True) allow_negative_indices: boolean specifying whether negative indices are allowed. If true, negative indices are taken relative to the end of the array. If false, negative indices are out of bounds and the result is implementation defined. Returns: An array containing the slice. Examples: Here is a one-dimensional example: >>> x = jnp.arange(5) >>> dynamic_index_in_dim(x, 1) Array([1], dtype=int32) >>> dynamic_index_in_dim(x, 1, keepdims=False) Array(1, dtype=int32) Here is a two-dimensional example: >>> x = jnp.arange(12).reshape(3, 4) >>> x Array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]], dtype=int32) >>> dynamic_index_in_dim(x, 1, axis=1, keepdims=False) Array([1, 5, 9], dtype=int32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.index_in_dim` - :func:`jax.lax.dynamic_slice` - :func:`jax.lax.dynamic_slice_in_dim` rr0)rrr)r,rrrr1rs r-dynamic_index_in_dimrxs6f 47M O& M ;;vw ''r/ct|}tj|dgtj|z}|||<dg|jz}|||<t ||||S)ajConvenience wrapper around :func:`dynamic_update_slice` to update a slice in a single ``axis``. Args: operand: an array to slice. update: an array containing the new values to write onto `operand`. start_index: a single scalar index axis: the axis of the update. allow_negative_indices: boolean specifying whether negative indices are allowed. If true, negative indices are taken relative to the end of the array. If false, negative indices are out of bounds and the result is implementation defined. Returns: The updated array Examples: >>> x = jnp.zeros(6) >>> y = jnp.ones(3) >>> dynamic_update_slice_in_dim(x, y, 2, axis=0) Array([0., 0., 1., 1., 1., 0.], dtype=float32) If the update slice is too large to fit in the array, the start index will be adjusted to make it fit: >>> dynamic_update_slice_in_dim(x, y, 3, axis=0) Array([0., 0., 0., 1., 1., 1.], dtype=float32) >>> dynamic_update_slice_in_dim(x, y, 5, axis=0) Array([0., 0., 0., 1., 1., 1.], dtype=float32) Here is an example of a two-dimensional slice update: >>> x = jnp.zeros((4, 4)) >>> y = jnp.ones((2, 4)) >>> dynamic_update_slice_in_dim(x, y, 1, axis=0) Array([[0., 0., 0., 0.], [1., 1., 1., 1.], [1., 1., 1., 1.], [0., 0., 0., 0.]], dtype=float32) Note that the shape of the additional axes in ``update`` need not match the associated dimensions of the ``operand``: >>> y = jnp.ones((2, 3)) >>> dynamic_update_slice_in_dim(x, y, 1, axis=0) Array([[0., 0., 0., 0.], [1., 1., 1., 0.], [1., 1., 1., 0.], [0., 0., 0., 0.]], dtype=float32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.dynamic_update_slice` - :func:`jax.lax.dynamic_update_index_in_dim` - :func:`jax.lax.dynamic_slice_in_dim` rFr0)rrr_ndimrr=)r,r<rrr1r&rs r-dynamic_update_slice_in_dimrsk| T$$'JJ{A$>#?#))GBT#T-#-7W\\)./. gv}5C EEr/ct|}tj|tj|k7rGtj|dztj|k(sJtj||f}t |||||S)apConvenience wrapper around :func:`dynamic_update_slice` to update a slice of size 1 in a single ``axis``. Args: operand: an array to slice. update: an array containing the new values to write onto `operand`. index: a single scalar index axis: the axis of the update. allow_negative_indices: boolean specifying whether negative indices are allowed. If true, negative indices are taken relative to the end of the array. If false, negative indices are out of bounds and the result is implementation defined. Returns: The updated array Examples: >>> x = jnp.zeros(6) >>> y = 1.0 >>> dynamic_update_index_in_dim(x, y, 2, axis=0) Array([0., 0., 1., 0., 0., 0.], dtype=float32) >>> y = jnp.array([1.0]) >>> dynamic_update_index_in_dim(x, y, 2, axis=0) Array([0., 0., 1., 0., 0., 0.], dtype=float32) If the specified index is out of bounds, the index will be clipped to the valid range: >>> dynamic_update_index_in_dim(x, y, 10, axis=0) Array([0., 0., 0., 0., 0., 1.], dtype=float32) Here is an example of a two-dimensional dynamic index update: >>> x = jnp.zeros((4, 4)) >>> y = jnp.ones(4) >>> dynamic_update_index_in_dim(x, y, 1, axis=0) Array([[0., 0., 0., 0.], [1., 1., 1., 1.], [0., 0., 0., 0.], [0., 0., 0., 0.]], dtype=float32) Note that the shape of the additional axes in ``update`` need not match the associated dimensions of the ``operand``: >>> y = jnp.ones((1, 3)) >>> dynamic_update_index_in_dim(x, y, 1, 0) Array([[0., 0., 0., 0.], [1., 1., 1., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]], dtype=float32) See Also: - :attr:`jax.numpy.ndarray.at` - :func:`jax.lax.dynamic_update_slice` - :func:`jax.lax.dynamic_update_index_in_dim` - :func:`jax.lax.dynamic_index_in_dim` rr0)rrrrr)r,r<rrr1s r-dynamic_update_index_in_dimrsx~ T$YYv#))G,, 99V q CIIg$6 66 6 __VdW -F $ vud3 55r/ctjdd|tjdd||jt|k7r'd}t |j ||j t|t|k7rd}t |j ||tttj|j |s'd}t |j ||j td|Dsd}t |j |tttj||sd }t |j ||tttj||}|$t|d t|j zk(r|Stjdd |t||jk7r'd }t |j ||j td |Dsd}t |j |tdt||DS)Nr.r&r'z{slice start_indices must have length equal to the number of dimensions of the operand, got indices {} for operand shape {}.zjslice limit_indices must have the same length as start_indices, got start_indices {} and limit_indices {}.zkslice limit_indices must be less than or equal to operand shape, got limit_indices {} for operand shape {}.c3&K|] }|dk\ ywrNrC).0sis r- z$_slice_shape_rule..Qs -R1W -zSslice start_indices must be greater than or equal to zero, got start_indices of {}.znslice limit_indices must be greater than or equal to start_indices, got start_indices {} and limit_indices {}.rr(zuslice strides must have length equal to the number of dimensions of the operand, got strides {} for operand shape {}.c3&K|] }|dk\ ywrrCrrZs r-rz$_slice_shape_rule..bs %Q!V %rz&slice strides must be positive, got {}c3RK|]\}}tj|d|!yw)r) window_size window_strideN)r stride_dim)rdrZs r-rz$_slice_shape_rule..es,/1aqaqAA/%')r_check_shapelikerlenrrrallmapoperatorger+rzip)r,r&r'r(rdiffs r-_slice_shape_rulerBsw?w? \\S'' MC CJJ}gmm< ==3}-- 8C CJJ}m< == Sgmm] ; < 8C CJJ}gmm< == -} - - &C CJJ}- .. Sm] ; < 8C CJJ}m< == s8<< > ?$ _g$W]]1C*CC Kw 73\W\\! CC CJJw 6 77 %W % % 2C CJJw' (( /tW-/ //r/c~t|trtjfd|DSj|S)Nc3<K|]}j|ywrr)rrZrs r-rz%_get_sub_spec_size..js5qTZZ]5)rTr+mathprodr)rsub_specs` r-_get_sub_spec_sizer hs1% 995H5 55 H r/c :|jj}t|j||jjD]L\}}}||k7s ||t ||zdk7s#t j|d|dt ||d|d|jS)zFReturns a sharding when out_shape may not be the same as operand shaperz on sharded dims where out dim (z!) is not divisible by mesh axes (z ) with spec (z) is not implemented.)shardingrr#rspecr r ShardingTypeError) out_shaper,namerop_shout_shop_specs r-#_get_sharding_for_varying_out_shaperms     $ ( mmY 0 0 5 5!7eVW G/#D'22a7  " " F26(;+D':;<i, .  "   r/c:t||||}t||dS)Nr%slicing)rr)r,r&r'r(rs r-_slice_sharding_rulers' }.;WN) ,Y KKr/c|jj}|)tjtj|dr.t |tj ||dt|z}ntj|tjtj|jdk(dtjdtjtj |jd|}t |tj ||tj |d}tj|tj|d|}|j|k(sJd|jd||gS)Nrrrz result.shape=z operand_shape=)avalrrlrequalrsubtractrrwherermultiplyrpadr) tr,r&r'r( operand_shapepads real_limitsrs r-_slice_transpose_ruler&s(,,$$- _rxx34 }bkk-Gc-(( *D&&hhrxx A%qvvaR[[!%,> NM**,dFF &,,'1,a2;;r{{6<<'CWMN PQk GLL,>,> LWa(*d MM#''&#**VQ"7>?r/c|\}|\}t|}|j|dt|}|j||j||d} nt|} | j|dt|||| } | |fS)Nrr)rinsertrr.) batched_args batch_dimsr&r'r(r,bdimnew_start_indicesnew_limit_indices new_stridesouts r-_slice_batching_ruler<s (' %$=)4#=)4t!45 _Kw-KtQ g(*;[I# dr/r.) sharding_rulevma_rulec|tjt||||Stt j |t j |z }tjt |g|d|iS)Nr%r3)r apply_primitiver)r+rlrr7)rr&r'r(r3s r- _slice_implrAsp   # # q !7 44bhh}-0GGH+  ! !/1 ;} ;.9 ;;r/c|xsdgt|z}|j\}tj||||||}tj|||gS)Nrr%)r avals_outrslice_oplower_with_sharding_in_types)ctxrr&r'r(aval_outr;s r- _slice_lowerrHsW  /sS//'mm)( c1hm$17 D#  + +Ch ? @@r/c^tttj|j|s'd}t |j ||jtd|Dsd}t |j |td|Drt d|t|S)Nzgslice slice_sizes must be less than or equal to operand shape, got slice_sizes {} for operand shape {}.c3&K|] }|dk\ ywrrC)rsszs r-rz,_dynamic_slice_shape_rule..s -#SAX -rzOslice slice_sizes must be greater than or equal to zero, got slice_sizes of {}.c3:K|]}|jdk7ywrrridxs r-rz,_dynamic_slice_shape_rule..03Q0zGstart_indices arguments to dynamic_slice must be scalars, got indices ) rrrrrrranyr+)r,r3r&rs r-_dynamic_slice_shape_rulerSs Sgmm[ 9 : 6C CJJ{GMM: ;; - - - $C CJJ{+ ,,0-00 $$1?4 55 { r/c:t|g|d|i}t||dS)Nr3r9)rSr)r,r3starts_and_dyn_sizesrs r-_dynamic_slice_sharding_rulerVs.' ?$?2=?) ,Y QQr/c|j|jj|jjjS)Nreducedr)r<rrrY)out_sr,r3rUs r-_dynamic_slice_reduced_ruler\s@ 5::,,##++-- ..r/ctj||jg\}tfdDr2d}t |j dj dD|jS)Nc3K|]Q}|jdjk7xs/tj|jtj Sywrryr rkrlintegerrrr&s r-rz,_dynamic_slice_dtype_rule..T L89 M!$** * 4   qww 3 3 4 LAAzKindex arguments to dynamic_slice must be integers of the same type, got: {}, c3HK|]}|jjywrryrrrs r-rz,_dynamic_slice_dtype_rule..(M!(M ")r split_listrrRrrjoinry)r,r3rUdynrr&s @r-_dynamic_slice_dtype_rulermsk';gll^L- L=J LL C CJJtyy(M}(MMN OO r/c|d}t|tjurtj|g|ddd|i}tj|dg|ddd|i|fS)Nrrr3)r)rr*r7r*)primalstangentsr3 tangent_outs r-_dynamic_slice_jvprrsg + +gll*!&&{ZWQR[ZkZK   gaj P712; PK PR] ]]r/cNtj|sJtd|DsJt|tj ur0t j |j gdgt|zzStj|j jd|j j|j j}tj|t|j j }t#j$||g|gdgt|zzS)Nc3HK|]}tj| ywrris_undefined_primalrs r-rz0_dynamic_slice_transpose_rule..  Bq''* * Brirr)rrvrr)rr*rrrrrryrr pvaryr+vmar;r*)r"r,r3r&zeross r-_dynamic_slice_transpose_ruler| s   (( ( BM B BB B !W  LL & '4&3}3E*E EE HHW\\''GLL,>,>%ll33 5E JJueGLL$4$45 6E # ( ( BM B C FS' ' ()r/c t|tjsJtd|DsJt |t j uryt|tjr5|jj|ttt||ytj|jj d|jj"|jj$}t'j(|t|jj*}|j-t/j0||g|y)Nc3RK|]}t|tj !ywr)rTrr(rs r-rz1_dynamic_slice_transpose_fancy..s DAr||, , Drrrx)rTrr(rr)rr*r+r-r.r+rr!rrrrryrr ryrzr/r;r*)r0r,r3r&r{s r-_dynamic_slice_transpose_fancyrs GR\\ ** * Dm D DD D &\W\\!6% KK&%B {(K"LM HHW\\''GLL,>,>%ll33 5E JJueGLL$4$45 6E MM(--eVLmLMr/c Rt|dk(rtjgddfSt}t dt ||D|}t dt ||Dd}||ur:t j|Dcgc]}t j|dc}ddfS|dnt|dt|dd}t jt ||Dcgc]$\}}t j||df|dnd| &c}}d }|dfScc}wcc}}w) Nrint32c3FK|]\}}| |j|ywrrrrrs r-rz/_batch_dynamic_slice_indices..%s KdaQ]qwwqzKs !!c3K|]^\}}|Wtj|jjtj|jj|f`ywr)r typeofrrrrs r-rz/_batch_dynamic_slice_indices..'sT@!Q{{1~&&++T[[^-D-D-I-I!-LM@sA$A&rrrrC)broadcast_dimensions out_sharding) dimension) rrlrobjectnextrrr broadcastrPbroadcast_in_dim)rbdims empty_markersizer;rr[rs r-_batch_dynamic_slice_indicesr!s-\Q 88B $ &&, Kc'5&9K $ @gu-@AE G# \ ??GDqCMM!T2Da H$ NN+$=Q3q64#I% OOWe$& A  D!9AMDr&23  4' !E&s D')D# c t|^}}|^}}|j|tjurdndz }tt |}t j ||g\} } t j ||g\} } t|d|} t| | \}}t||g| ||g| | |ddtjdS)NrrrCrTrh) rr not_mappedr+rrrjr?r_gather_batching_rulerLrW)r5r6r3r,start_indices_and_dyn operand_bdstart_idx_and_dyn_bdsndimsdimsr&dyn_slice_sizes start_idx_bdsdyn_slice_size_bdsrr index_bdims r-_dynamic_slice_batching_ruler3s%1!' !'1$*$ ,,zX-@-@@!a H% uU| $#'??3H5'#R -&*oo6KeW&U#-#  % 3=-P%  e&o&101UDT  , ,  ??r/cP|jt|g|t||S)N)r3) source_info)default_process_primitiver7dict)tracerrr3r&s r-_dynamic_slice_staging_rulerLs1  ( (*M*D[,I ) r/cftj|jgd|Dd|i\}}|g|fS)Nc34K|]}|jywr)rrrs r-rz0_dynamic_slice_typecheck_rule..Ts.1.sr3)r7 abstract_evalr)rrr3r&out_avaleffectss r-_dynamic_slice_typecheck_rulerRsC%33ffI. .I reduced_rulec|j^}}|j\}tj||||}tj|||gSN)r&)avals_inrCrr9rE)rFrr3r&x_avalrrGr;s r-_dynamic_slice_lowerrdsH||*&1mm)( 3!=I#  + +Ch ? @@r/c>|j|jk7r1d}t|j|j|j|jt |k7r'd}t|j||jt t tj|j|js1d}t|j|j|jtd|Drtd||jS)Nzidynamic_update_slice update must have the same rank as operand, got update shape {} for operand shape {}.zvdynamic_update_slice start_indices must have length equal to the rank of operand, got indices {} for operand shape {}.zodynamic_update_slice update shape must be smaller than operand shape, got update shape {} for operand shape {}.c3:K|]}|jdk7ywrrMrNs r-rz3_dynamic_update_slice_shape_rule..zrPrQzMstart_indices arguments to dynamic_update_slice must be scalars, got indices ) rrrrrrrrrrRr,r<r&rs r- _dynamic_update_slice_shape_rulerms \\V[[  7C CJJv||W]]; << \\S'' CC CJJ}gmm< == SgmmV\\ : ; >C CJJv||W]]; <<0-00 ,,9?< == r/c|j|jk7r.rbrczQindex arguments to dynamic_update_slice must be integers of the same type, got {}rdc3HK|]}|jjywrrfrgs r-rz3_dynamic_update_slice_dtype_rule..rhri)rcheck_same_dtypesrRrrrkryrs ` r- _dynamic_update_slice_dtype_rulersb.@ L=J LL C CJJtyy(M}(MMN OO r/c|dd\}}|dd}|dd\}}tj||g|}t|tjur>t|tjur#tjj |}||fSt j|}t j|}tj||g|}||fS)N)r;r*r)rr*from_primal_valuerinstantiate_zeros) rorpr,r<r& g_operandg_updateval_outrqs r-_dynamic_update_slice_jvprsBQK/'6!"+- ! )X " ' ' H- H' )_ $h7<<)G,,009K + $$Y/I##H-H(--iRMRK + r/ctd|DsJtj|r|jjn |j}|jj }t |tjurmtj|rtj|jnd}tj|rtj|jnd}nvtj|||}tj|rtj||g|nd}tj|rtj|g|d|ind}||gdgt|zzS)Nc3HK|]}tj| ywrrurrs r-rz7_dynamic_update_slice_transpose_rule..rwri)rrr3)rrrvrrrr)rr*r_zerosr;r*r7r) r"r,r<r&rupdate_sharding operand_tupdate_tr{s r-$_dynamic_update_slice_transpose_rulers, BM B BB B')'='=f'E&++##,,KK((/ !W .0.D.DW.M W\\*SWI,.,B,B6,Jw||FKK(PTH JJq  GE**73(,,QF F9=))&1 $$QQQLQ7; X $#m*unreduced_rulerc|j\}tj|||||}tj|||gSr)rCrr=rE)rFrr<r&rGr;s r-_dynamic_update_slice_lowerrs@mm)( !!#xF0= ?#  + +Ch ? @@r/c tj|jtjs t d|jS)N!indices must have an integer type)r rkryrlr`rY)r,rrekwargss r-_gather_dtype_rulers0   7=="** 5 8 99 r/c,t|jSr)rr)arrs r-rrsC Nr/cztdt|D]#}||||dz kst|d|d|y)Nr in z op must be sorted; got )rrr)rop_namerrs r- _is_sortedrsO CI La Awa!e d7)+CD6J KKLr/c V|D]$}|dks||k\std|d|d|d|d y)NrInvalid  set in  op; valid range is [0, ); got: r)r)rrankrrdims r-_dims_in_rangersN 3c Qw#+ hwi8!F(3%q2 333r/c t|dk(ryd}|ddkr|d}n |d|k\r|d}|rtd|d|d|d|d y)Nrrrrrr)rr)rrrr invalid_dims r-_sorted_dims_in_rangersvY!^ + !Wq[q'K Bx4r(K htfHWI6& Q8 99r/cltt|t|k7rt|d|d|dy)Nrz op must not repeat; got: r)rsetr)rrrs r-_no_duplicate_dimsr s:T^s4y tfD )CD6K LL!r/c ~t|jt|st|d|d|d|d|d y)N and rz op must be disjoint; got: r)r isdisjointr)dims1dims2rname1name2s r-_disjoint_dimsrsQ U  s5z * ugU5'gY7!7%wa1 22 +r/c ~|j}|j} |j} |j} |j} t dz } t | ks| dkrt dt d| dt|ddt|ddt|}t|t zdz }t|D]&}||}|dks||k\st d|d |d |d t| j| k7r+t d t| d | dj| dtt| D]8}| |}|dks|t k\st dt d|d|dt| ddt| ddt| t ddt| ddt| ddt| ddt| t ddt| | dddt| | dddt| ddt| t dd| | vrt d| dt| t| k7rt d| d| dtfd| D}tfd| D}t!j"||st d|d|dt t|k7r#t dt|dt t|t|t| zt| zk7r!t d t|d!|d"| d#| d tt|D]U}||}j|}t!j$jr7|dk\r||k\rBt d$|d%|d&|dtt| D]'}|| |}|dk7st d'|d(| |d)|dtt| D]'}|| |}|dkDst d*|d(| |d)|dt'||S)+aFValidates the well-formedness of the arguments to Gather. The code implements the checks based on the detailed operation semantics of XLA's `Gather `_ operator and following the outline of the implementation of ShapeInference::InferGatherShape in TensorFlow. rrzTGather index leaf dimension must be within [0, rank(indices) + 1). rank(indices) is z$ and gather index leaf dimension is rrzr@zOffset dimension z$ in gather op is out of bounds; got z, but should have been in [0, )zGather op has zI elements in start_index_map and the bound of dimension index_vector_dim= of indices is z". These two numbers must be equal.z'Invalid start_index_map; domain is [0, ), got: ->rBrArDrEzNGather op cannot have the index vector dimension as a batching dimension; got z_Gather op requires equal numbers of operand_batching_dims and start_indices_batching_dims, got z andc3<K|]}j|ywrrrrr,s r-rz%_gather_shape_rule..~N1gmmA.Nrc3<K|]}j|ywrrrrrs r-rz%_gather_shape_rule..gmmArzkGather op requires operand batching dimensions and indices batching dimensions to have the same shape, got rzTGather op must have one slice size for every input dimension; got: len(slice_sizes)=z, input_shape.rank=zAll components of the offset index in a gather op must either be a offset dimension or explicitly collapsed/batching; got len(slice_sizes)=z, output_slice_sizes=z, collapsed_slice_dims=z, operand_batching_dims=zSlice size at index z2 in gather op is out of range, must be within [0, z + 1), got zBGather op can only collapse slice dims with bound 1, but bound is z for index z at position zKGather op can only have operand batching dims with bound 0/1, but bound is )r@rArDrErB_rankrrrrrrrrrr+r definitely_equal_shapeis_empty_shape_gather_shape_computation)r,rrir3rbrcrdrer@rArDrErBindex_vector_dimoutput_offset_dim_countoutput_shape_rankr offset_dimoperand_dim_for_start_index_ioperand_batch_shapeindices_batch_shapercorresponding_input_sizebounds`` r-_gather_shape_rulers"--+*??+AA 1 M M%55/7^a' 7^&&*:Q*> 77.s. !Q & &14I+Isc3RK|]}|vr tn t ywr)r)rrindices_shape_genr@slice_sizes_gens r-rz,_gather_shape_computation..s5N/0()K'7d?#)*+N$') r@rArDrrrrappendpopiter enumerater+r) rrir3rrexpanded_indices_shapeansrAr$r@rDr%s @@@@@r-rrs!--+*??+AA+&w7!;7^a' .  $44!!!$-.12K(/ N49:K4LN N# *r/c|j}|j|j|j|jt ||j zdz }|j dz }jjt|jjtdDrtdDr tStfdDsJtfdD}tfdtD}tfdtD} |r|r| rtD]+\} } | | k(s | | J| xs| | <-fd tD} t} g}t|D]\}||vr. *!t) *c3$K|]}|du ywrrCrs r-rz+_gather_spec_computation..r/r0c3&K|]}|v ywrrC)rrrBs r-rz+_gather_spec_computation..s @aQ/ ! @sc3ZK|]"}|j|k(xs|du$ywrr)rrr, operand_specr3s r-rz+_gather_spec_computation..s<4 !n a((CLOt,CC4 s(+c3^K|]$\}}||k(xs|duxs|du&ywrrC)r operand_dim indices_dim indices_specr4s r-rz+_gather_spec_computation..sW, %; ;< #<<+ k "d *+ k "d *+,s*-c34K|]\}}|vr|duywrrC)rrrZrEs r-rz+_gather_spec_computation..s**Aq - -4i*Nc3<K|]\}}|vr |vr||fywrrCr"s r-rz+_gather_spec_computation.."s4M1a ( (Q6K-K AMr)r@rBrArDrErrrrrrrrr*r)rrrr')r,rrir3r@rr+operand_index_dims_full_slice_or_replicated#batching_dims_resolve_unambiguously!indices_non_batch_dims_replicatedr6r7r%indices_spec_genout_specrrrAr8rDr4rBrEs` ` @@@@@@r-_gather_spec_computationrAsQ:"--+%55/*??+AA 1 M M+&59\\A%!!&&,g&&++,, *\ ** *\ ** 3J @+? @ @@ @144 4 1 -),,), !<)> ,)%'**%l3*'# 2 - +$':%<Y [;'< +DDk*2k*243 #/{";"X|K?Xl;YM$[1MOL)H $ % 0 k #'"7 ZgmmK88 ,4 65 [12 -./ 0 h< r/c|Dchc]}|jr|}}t|dkDrtjd||st j S|\}|Scc}w)z'Resolves the mesh between given meshes.rz0Conflicting meshes received. Got: unique_meshes=)emptyrr rmesh_libget_abstract_mesh)meshesr unique_meshess r- _resolve_meshrH:sk$*=D$**4=-=!  ;],<=    % % '' %$ +>s AAct|jj|jj}t||||} | t j d|d|t || S)NzUse `.at[...].get(out_sharding=)` to provide output PartitionSpec for the gather indexing as out sharding could not be resolved unambiguously (or would require collectives on inputs). Got operand= , indices=)rHrrrAr rr) r,rrir3rbrcrdreout_meshr@s r-_gather_sharding_rulerLGs{7++00'2B2B2G2G H( %gw8I&13(     :[z # $$ x **r/cf|}tj|jd} ttj | } t j|jj| } t j|| }t|jdz } | | |jt j|| |jj| z } t jt j|| jdt j|t j | t#t%| }t j&|| g}| t|j(z}t j*t j,||j(}t/|||||t0j2}t j4t j6||||j8j:|t j<||S) zHLowers a FILL_OR_DROP gather as a PROMISE_IN_BOUNDS gather with masking.Tsignedryrr)rcrd)r)re) lax_utilsint_dtype_for_shaperrrlrrshape_as_valueastypeconvert_element_typerrB bitwise_andrr)lerr+r reduce_andr@rarangerzrLrWselectrrr full_like)r,rrir3rbrcre output_shaper index_dtypeintarray operand_dimsnum_batch_dims upper_boundmask output_ndimsbatch_dims_in_output gather_outs r- _gather_fillrfUs %--gmmDI+ RXX[ 1(##GMM299+F,  $ $Wk :'w}}%).u$$% %hu/D/D&EFMM+  ffWk&&q)* ffWcook5~9N3OPQ S$ ~. /$ #e&7&7"88,299\#:#(#4#46 gw{);,>>@* |-A&0oo&>&>@ jZ@ BBr/c (t|||||||dS)Nrra)rz) gr,rrir3rbrcrdres r-_gather_jvp_ruleri{s% 7-{-#5D r/c ~tj|sJt|tjur#tj|j } | dgSt j|j jdtj|j|j j} tj| t|j j} t!|j"|j$|j&|j(|j*} t-| ||| |||} | dgS)Nrrxr~rrrDr)rbrcrd)rrvr)rr*rrrrr rryrryr+rzr|r@rArBrDrEr) r"r,rrir3rbrcrdrer;r{ scatter_dnumss r-_gather_transpose_rulerms   (( ( !W  ,,w|| $C t HHW\\''DKKN,@,@%ll33 5E JJueGLL$4$45 6E+,88.CC%6%F%F/EE&7&S&S M eWa%3); !C tr/c |\}} |\} } | | tj|| d}d} |jdf|z}dtt j d|j z} tt j d|j} tt j d|j}tt j d|j}t| | |||j}| }t|| |||||||fS| | tj| | d} td|j D} tt j d|j}t| |j|j|j|}t|| ||dd||dfStj|| d}tj| | d} tj|jddrd|z}nd|z}tt j d|j} dtt j d|jz}dtt j d|jz}tt j d|j } tt j d|j}t| | |||}t|| ||||||dfS) Nrrrr@rArBrDrErhc3&K|] }d|z yw)rNrCrs r-rz(_gather_batching_rule..sE!AErFr)rmoveaxisrr+rlrr@rArDrBr?rErzr definitely_equal)r5r6rir3rbrcrdrer,r operand_bdim indices_bdimr@rArDrBrbdim_outrEs r-rrs"'7), ,"6q9GL==#% 3Krvva):)F)FGHHK +<+Q+Q!RS! q#99:BFF1&7&G&GHIO "1'3$5$Q$Q  EH E%-D   !)  )) 8q9GE'8'D'DEEK"' q#??@# #.CC)99/EE$?  E '7e)%%** NOP QQ q9Gq9G W]]1-q1;&k;&k  q#889!5 q#99:$#' q#??@*#q"3"?"?@AKBFF1&7&G&GHIO "1'3$?  E '7e).%7d' )+, ,,r/rz)rr=r>c |j\} } |j\} tj| jj } t t| D cgc]} | j| z}} |jg|j|}g|| }tt||||||}tj||||tj| | gtj| g\}|Scc} w)N)r@rhrrC)rrCr physical_element_avalryrrrr_replacer@r _gather_lowerrdelegate_lowering physical_aval)rFr,rrir3rbrcrdreaval_x aval_indicesaval_y elt_shapertrailing_offset_dims gather_lowerress r-_gather_lower_opaquers&, MM'&((6<<)38Y3HIa&++/II'00I%11I4HI1K*+* *+'8n+$:O,    <'""6*L9##F+, .$# *JsC;c b|j\} } |j\} tj| jtj rt ||||||||| gS|tjk(r6tjtd} | ||||||||| j S|tjtjfvsJ|tj j#t%|j&t%|j(t%|j*t-| jdz t%|j.t%|j0} t3j4|stj6||}tj8| g}|||g}| t:j<j#|d}tj>jA|||jBSt3jD| jr/tjF|d | }tjH||| gStjJ||| tjL|t:j<j#| }tjH||| gS) NrhFmultiple_results)rir3rbrcrer\r)rArDrErr@rB)rirc)resultsoperands attributesr)rc)'rrCr rkryextendedrrLrVr lower_funrfrrWrUrr?getrrArDrErr@rBr is_constant_shapeeval_dynamic_shape_as_tensoraval_to_ir_typerBoolAttrDynamicGatherOp build_genericrrfull_like_avalrErzdense_int_array)rFr,rrir3rbrcrdrer indices_avalrGgather_fill_fnrrrrr;s r-rzrz skLL/!\mm)( x~~v7  Wg1B-D      + ++^^L5IN  Wg+%:LHNN  << #55#((* *0+/0 * $ $ ( ( 1 F FG !2!H!HI"&  7 7#<--.2(445,<<= ) %    ,33CEK ##H-.G+.H" kkoo.@AJ    , ,(z - CCJ7K 8>>*   c1h /C  - -c3 A BB **Wgud.B.B;.O(* 8J(K MC  - -c3 A BBr/c tj|jtjs t dt jd|||jS)Nrr)r rkryrlr`rYrr)r,rrrs r-_scatter_dtype_rulerC sB   7=="** 5 8 99 7G4 r/ctt||D]}j|dtt|k(sJt fd|DS)Nc3@K|]}j|ywr)r)rrscatter_dim_in_updatess r-rz-_get_updates_batching_dims..O sN1%++A.Ns)rrr4rr+)indices_batching_dimsr~r updates_shaperrs @r-_get_updates_batching_dimsrI s^&6 78 +a!!!T*+ # $M(: :: : N8MN NNr/c  |j} |j} |j} |j} |j} t dz }t |ks|dkrt dt d|dtj}t||k(r|jdt|dz t| z}t |k7rt d|dt dt| dd t| dd t| t dd t| dd t| dd t| t dd t| dd t| dd t| t dd t| | dd d t| | dd d t| dd t| t dd || vrt d| dt| t| k7rt d| d| dt!fd| D}t!fd| D}t#j$||st d|d|dt'| | |j}t!fd|D}t#j$||st d|d|dt| t| zt| z}t |k7rt d|dt dt| j|k7r+t dt| d|dj|dt)t| D]8}| |}|dks|t k\st dt d|d|dt| dd t)tjDcgc]+}|t+| vr|t+| vrj|-}}t)t| D]A}| |}||j|kst d|d j|d!||dt)t Dcgc]}|t+| vs|}}d}|D]\}||k(r|dz }t#j,j|||s%t d"|d j|d#||d|dz }^jScc}wcc}w)$aaValidates the well-formedness of the ``dimension_numbers`` argument to Scatter. The code implements the checks based on the detailed operation semantics of XLA's `Scatter `_ operator and following the outline of the implementation of ShapeInference::InferScatterShape in TensorFlow. rrzUScatter index leaf dimension must be within [0, rank(indices) + 1). rank(indices) is z% and scatter index leaf dimension is rzUpdates tensor must be of rank z; got rr~rrDrrzOScatter op cannot have the index vector dimension as a batching dimension; got zbScatter op requires equal numbers of operand_batching_dims and scatter_indices_batching_dims, got rc3<K|]}j|ywrrr s r-rz&_scatter_shape_rule.. rrc3<K|]}j|ywrrrs r-rz&_scatter_shape_rule.. rrzlScatter op requires operand batching dimensions and indices batching dimensions to have the same shape, got c3<K|]}j|ywrr)rrrs r-rz&_scatter_shape_rule.. rrzlScatter op requires operand batching dimensions and updates batching dimensions to have the same shape, got zScatter op has window of size z ; doesn't match operand of rank zScatter op has zV elements in scatter_dims_to_operand_dims and the bound of dimension index_vector_dim=r z!. These two numbers must be equalz <*!- .. ,@A'4HI*E'NI,. !9.DE)96LM,eGni.0 "I/FG*I7NOU7^Y8O%' Scatter offset_dims <-> update_window_dims collapsed_slice_dims <-> inserted_window_dims start_index_map <-> scatter_dims_to_operand_dims rc3$K|]}|du ywrrCrs r-rz,_scatter_spec_computation..G r/r0c3$K|]}|du ywrrCrs r-rz,_scatter_spec_computation..H r/r0c3$K|]}|du ywrrCrs r-rz,_scatter_spec_computation..I r/r0c3RK|]\}}}t||| ywr)r)rodidudr8r4 updates_specs r-rz,_scatter_spec_computation..V s9P "b"\"%|B'7b9IJPr&c3K|]E\}}j|j|k(xr ||k(xs|duxr|duGywrr)r update_dimr6r,r4rrs r-rz,_scatter_spec_computation..] su24 "*k}}Z GMM+$>>=J< #<<R z "d * P|K/HD/PR24sA Ac34K|]\}}|vr|duywrrC)rrrrs r-rz,_scatter_spec_computation..c s*3$q$ " " dl3$r:c3<K|]\}}|vr |vr|duywrrC)rrrr~rs r-rz,_scatter_spec_computation..j s43Iq$ ' 'A5G,G dl3Irc34K|]\}}|vr|duywrrC)rrrrs r-rz,_scatter_spec_computation..m s*3)q$ ' ' dl3)r:N)rDrr~rrrrrrrrrrr*r)r,rrrirDrroperand_window_dimsbatch_dims_resolvable)update_and_operand_window_dims_resolvable*inserted_window_dims_replicated_in_operand*unbatched_query_dims_in_updates_replicated*unbatched_query_dims_in_indices_replicatedindex_vector_dim_is_replicatedr@r6r7 updates_dimrrr8rr4r~rrs` ` @@@@@@@r-_scatter_spec_computationr sz,AA+II(;;*??\\A%!!&&,!!&&,!!&&, *\ ** *\ ** *\ ** 3J4/1A7==R w||$K & &14I+IKK P !68MOPP/224&) 1&3 24/4+ 033$"+L"93$0$,033I"+L"93I0I,033)"+L9J:J,K"L3)0), $00@#AT#I /000$L!H2546K2MD- [+ +  &  % Bk)B D $''9;N#O9 K +  & ,Z 89 h< sKs:Hc|jSr) memory_space) r,rrrrrircrbrds r-_scatter_memory_space_ruler s   r/ctd|||fD} t||||} | tjd|d|d|t | | S)Nc3HK|]}|jjywr)rrrs r-rz)_scatter_sharding_rule.. s=A =rizUse `.at[...].set/add/mul/...(out_sharding=)` to provide output PartitionSpec for the scatter update as out sharding could not be resolved unambiguously (or would require collectives on inputs). Got operand=rJz , updates=)rHrr rr) r,rrrrrircrbrdrKr@s r-_scatter_sharding_ruler su='7G!<=?( &w'8:(     :[zG: / 00 x **r/c gd}ttjD]`}||jvs||jvrj d1j |j|j ||dz }btfd|jD}tj|}tj||jjttj|jj tj|jj }tj"||j}tj$||jt|jdz f}tj&|jjd||S)z.Clamps `indices` to be in-range for a scatter.rrc3HK|]}j||z ywrr)rrr,r3s r-rz)_clamp_scatter_indices.. s-#P'($+==#3k!n#D#Ps")rrrrrDr'r~r+rrrSrqryr)rlrprsrUrclamp) r,rrrposr upper_boundsrar3s ` @r-_clamp_scatter_indicesr sp+ # W]]# $aE & &&!u/J/J*Ju'?'?'DEF Qhc ##P,1,N,N#PP,""<0+ bhh{(()--rxx /F/J/J K+ ((gmmD+$$['--&)'--&81&<%>@+ 7==%%a('; ??r/c |\} } } |\} } }~ |j| | | |||||| }t| tjur>t|tjur#tjj |}||fSt j | } t j |}|j| | ||||||| }||fS)Nr)r*r)rr*rrr)primrorprrrircrbrdr,rrr g_indices g_updatesrrqs r-_scatter_addsub_jvpr s&'7G$,!)Y  II wl!5F+N   '  )_ $iGLL)H,,009K + $$Y/I$$Y/I))7IL#7H-n  K + r/ctj|rJtj|r|jj} n |j} t |t j urntj|rt j |jnd} tj|rt j |jnd} ndx} } tj|r|} tj|rt|j|j|j|j|j}g}d}tt|jD]V}||jvs||jvr|jd1|j| |j||dz }Xt!||||| d} |t"urt%j&| } | d| gS)Nrorrrir3rdre)rrvrrr)rr*r?r~rrrDrrrr'rzrrneg)rr"r,rrrrrircrbrdrrr gather_dnumsr3rrs r-_scatter_addsub_transpose_ruler s # #G ,, ,G$LL&&MMMM !W .0.D.DW.M W\\*SWI-/-C-CG-Lw||GLL)RVHI g&i g&+'::0EE+HH 1 G G&7&U&U lk cS\"! "77 7%;;;   Q    ]+<+O+OPS+TU V (#7l$/dqJh  778$ T8 $$r/c tj|rJtj|r|jj} n |j} t |t j urntj|rt j |jnd} tj|rt j |jnd} n/dx} } tj|rt||||||| } tj|r|s tdt|j|j|j|j|j} g}d}tt!|jD]V}||jvs||jvr|j#d1|j#| |j||dz }Xt%t'j(|||| || d} | d| gS)NrCscatter_mul gradients are only implemented if `unique_indices=True`rorrr)rrvrrr)rr*rNotImplementedErrorr?r~rrrDrrrr'rzrr)r"r,rrrrrircrbrdrrrrr3rrs r-_scatter_mul_transpose_ruler s # #G ,, ,G$LL&&MMMM !W .0.D.DW.M W\\*SWI-/-C-CG-Lw||GLL)RVHI g& Wg1B/i g& ! OQ Q+'::0EE+HH 1 G G&7&U&U lk cS\"! "77 7%;;;   Q    ]+<+O+OPS+TU V (#7+W*6K!a1h T8 $$r/c |\} } } |\} }}tdt||D}tj| | ||j} tj| |||j} |t t jd|j}dt t jd|jz}t t jd|j}t t jd|j}t|||||j}|j| | | |||| || dfStj| |||j} t t jd|j}t t jd|j}dt t jd|jz}dt t jd|jz}t t jd|j}t|||||}|j| | | |||| || dfS)Nc3DK|]\}}||j|ywrr)rrrs r-rz)_scatter_batching_rule..4 s(!eaggbk!s rrrk)rircrbrdrrr)rrr bdim_at_frontexplicit_mesh_axisr+rlrrr~rDrr|rr*) scatter_op axis_datar5r6rrrircrbrdr,rrrsrt updates_bdimrrr~rDrrrs r-_scatter_batching_ruler, s+'7G-7*, l !s<'D! !$  " "7L$#,#?#? A'  " "7L$#,#?#? A' +<+Q+Q!RSbffQ0A0T0T&U VV! q#99:$)3D3a3a)b#c #-1%A3&7&U&U  E ?? w5+N l-  IKL LL  " "7L$#,#?#? A'RVVA'8'K'KLMrvva):)O)OPQffQ!778"::"&ffQ!??@*B#B!&ffQ!>>?"A !+/#?1$A  C%  w5+N l-  IKL LLr/z scatter-addr)rr=r>memory_space_rulecyNrCrCrs r-rrm r/z scatter-subrcyrrCrs r-rry rr/z scatter-mulrc |s tdtj|tt j |||||||S)Nrr)rrrrrzeros_like_jaxval) rhrrrrircrbrdkws r-_scatter_mul_jvp_rhsr sN  K MM K "Aqt|tjur#tjj |}||fSt j| } t j|}t|j|j|j|j|j}g}d}tt| jD]V}||jvs||jvr|j!d1|j!||j||dz }Xt#| | |t%j&|}t#|| |t%j&|}| |k(}||k(}t#t)t+j,| | t+j.|t+j0| t+j,| || |t%j&|}t#t)t+j,| | t+j0| || |t%j&|}t+j.|d|dzz d|z }t+j.||t+j,| }t+j.|d|dzz t+j,|}d|z|z } t#| | |t%j&|}!|!| z||zz}"t)| | |"||||}||fS)Nrrorrg?gr)rr*r)rr*rrrr?r~rrrDrrrr'rzrlrrrrrZ_ones)#rrorprrrircrbrdr,rrrrrrlrrrqrr3rr initial_vals target_valssuccessful_updatesretained_values num_updatesnum_refsupdates_normalizer updates_coefoperand_normalizer operand_coeftarget_tangentstangent_updatess# r-_scatter_extremal_jvpr sL&'7G$,!)Y #---- OO wl!]+#$  0'  )_ $iGLL)H,,009Kr + o$$Y/I$$Y/I*!44*??%BB+AA$1$O$O LK C 3w}}% & }11 1 -55 51=)I)I#)NOP q,(=?L,(=?K"[0#{2O JJw  JJ)399W+=zz'* ,     KCJJw'IIg&! #  HO$';?$;$'+$57::00!jj13LO$';?$;$'JJ{$;=--9L7L"((;*?AO'5 </0Oi%-+1C-;#' )K + r/z scatter-minrcyrrCrs r-rr rr/z scatter-maxrcyrrCrs r-rr rr/c | td|\}} } |\} } } |}t| tjurZt| tjur?tj || | |||||| }|tjj |fStj| } tj| } |r@tj || | ||||d| }tj | | | ||||d| }||fSt| j}|jD]}d||< tj|}tj|d}t!j"t!j$t!j&|||t!j(| |}t+t!j,|jd|| ||||| }t/|j|j0|j2|j4|j6 }g}d}t9t;|jD]`}||j0vs||j4vr|j=d1|j=| j|j||dz }bt?|| || }t!j@t!jB|t!jD||t!jD|}t!j@t!jB||| t!jD| }t!j@t!jB|t!jD|| t!jD| }t!j@t!jB||| t!jD| }tG|| ||||| }tG|| ||||| }||fS) Nz!scatter_apply JVP not implementedrTrFrNrPrrrorr)$rr)rr*rr*rrrrrr~r r rQint_dtype_for_dimrrreshapeiotarrrr?rrrDrrrr'rzrZeqrr) rorprrrircrbrdr,rrrrrrrrq ids_shapernum_idsid_dtype update_ids scattered_idsrr3rrgathered_update_idsmasked_operandmasked_updatesmasked_g_operandmasked_g_updatess r- _scatter_jvpr s A BB%'7G$,!)Y % )_ $iGLL)Hnn wl!U+N  G GLL227; ;;""9-)""9-)nn wl!U+DtMG..),!U+Dt!MK K 7==!),,jIj IIi '  ( ( ?(wws{{388Hg#> Jyy9;*#((7==!X>!:u-?)7dD- (** 5588!77"'"E"E ,+ # ](() *aE & &&!u/J/J*Ju'?'?'DEF Qhc  }g1=+68 ::cff]CJJ}4MN%szz'':<.::cffZ2EF%szz'':<.ZZ}cjj6O P )3::i+@BZZz3F G )3::i+@B */+='5D B',g7G.3/A+9F+ + r/c |s tdtj|rJtj|r-|jj} |jj } n+|j} t j|j } t|tjurntj|rtj|jnd} tj|rtj|jnd} n/dx} } tj|r3t||tj| d|j| ||d| } tj|rt|j |j"|j$|j&|j(}g}d}t+t-|jD]V}||j"vs||j&vr|j/d1|j/| |j ||dz }Xt1||||| d} | d| gS) Nz?scatter transpose is only implemented where unique_indices=Truer)ryrTrrorr)rrrvrrrr rr)rr*rrrryr?r~rrrDrrrr'rz)r"r,rrrrrircrbrdrupdates_shardingrrrr3rrs r-_scatter_transpose_rulerw s  4 55  # #G ,, ,G$LL&&M||,,MMM{{7+44 !W .0.D.DW.M W\\*SWI-/-C-CG-Lw||GLL)RVHI g& W ((=!177=M N-/4 Qi  g&+'::0EE+HH 1 G G&7&U&U lk cS\"! "77 7%;;;   Q    ]+<+O+OPS+TU V (#7l$/d#$&h T8 $$r/rcyrrCrs r-rr rr/c |j\} } } |j\} tj| jj }t t|Dcgc]}| j|z}}|jg|j|}tt|||||| }tj|||||tj| | tj| gtj| g\}|Scc}w)N)r~rrrirbrcrdrw)rrCr rxryrrrrryr~r_scatter_lowerrr{r|)rFr,rrrrrirbrcrdr}r~ aval_updatesrrrtrailing_window_dims scatter_lowerrs r-_scatter_lower_opaquer  s(+||$&, MM'&((6<<)9>s9~9NOA,++a/OO'001,??1/112<}).+$8-    ='7G""6*L""<02##F+,  .$# *PsD c 4|N|rJ|jdj} tjtt j d| \}}|j\} tj| jtjrt||||||||||  gS| tjk(r:tjt d} | |j#d||||\}|} t$j&j)t+| j,t+| j.t+| j0t+| j2t+| j4t7|jdj8dz }tj:| }|g}|g}t%j<|f||||t>j@j)|t>j@j)| }tj:t j d| j}|jBjDjG||}t?jH|5tKjL}|jNr tQd tjR|jT||tjV||jXd|jXd|jZ|j\ \}}t%j^tj`|dddtc|jd|jDcgc]\}}tjf|||c}}S#1swYQxYwcc}}w) NrrCrFrrCrrr~rinput_batching_dimsrscattered_dims_to_operand_dimsrrcrbz!Cannot lower effectful `scatter`.)dim_var_valuesconst_lowering)4rryrr_scatter_reduction_computationr ShapedArrayrCr rkrr rLrUrrrreplacerr|rrr~rrDrrrrr ScatterOprrupdate_computationblocksr'InsertionPointr new_name_stackrr jaxpr_subcompmodule_contextTokenSet argumentsr(r)return_flatten_ir_valuesr#rrE)rFr,rrrrrircrbrd operand_dtyperGclip_fnrrlrop scalar_typer< name_stack out_nodesrrrs r-rr s LLO))M"%"6"6&(8(8](K#ML-mm)( x~~v7 ! Wgw!+N-D  : ;;   # ##nn3eLGt4gw!):a  + +CD 9 >>33 >s9B?N#!NNc@tj|jSr)rlfinforyrPs r- _real_dtyperA srxx444r/c 4 j\} } } | jtjk7rt ||||||  S| t j k(r:tjtd}|jd|||\j\|}tjjt|j t|j"t|j$t|j&t|j(t+jdj,dz t/jtj0t3j4j, f d}|tj6|tj6|}|tj8|tj8|}tj:||gS) NrFrr"r#rr$c R |g}|g}tj f|| tjj tjj }t j tjd }|jdjj||}tj|5tj |jjgdddt j |jS#1swY*xYw)Nr'rCr)rr-rrrrrr r+regionsr/r'r0r6r5rrE) operand_part updates_partrr;reducerrGrFrrcoperand_type_part real_dtype reduce_oprlrbs r-_scatterz+_scatter_addsub_lower_gpu.._scatter* s >L >Lmm lG\=;;??+=>{{~68G&&t'7'7J'GHKooa ''..{KHG  7 #: kk9g//07789:  , ,S'..( KK::s .DD&)rryrl complex128rrLrUrrrr,rCrr|rrr~rrDrrrrrArr r+realimagcomplex)rFr,rrrrrircrbrdrJoperand_aval_inrupdates_aval_inr9rrKrMrNrGrHrIrls` ` `` ` @@@@r-_scatter_addsub_lower_gpurR s), %/1obmm+ #w'3(5,=-?)7d  DD  # ##nn3eLGs{{T{2GWg.0HGmm)( %--11e667 : :;u::;$()L)L$M%)%*L*L%M3<<?001A5 2-8>>**** x~~z24 L L #((7#SXXg%6 7$ #((7#SXXg%6 7$ ++dD ! ""r/)rJgpu)platformc t||jk7r0d}t|jt||jt |t tfs>|jdk7r$tdj|jt|}g}t |tr|gt|z}t||j|D]\}}}tjt|tjr|j|Gt |t tj"frt%j&|r|r9|jt)j*|dkr||zn|t|nE|dkrtd|d|d|jt)j*|t|t%j,|}t |t tj"fre|r9|j|dkr"|t)j*|t|zn|n(|dkrtd|d|d|j||rMt)j*|t|}|jt)j.|dk||z||j||S)NzJLength of slice indices must match number of operand dimensions ({} vs {})rz+Slice indices must be a 1D sequence, got {}rzIndex z is out of bounds for dimension z if allow_negative_indices=False)rrrYrrrTr+rrwrr rkrjrlrrr'rr`r is_constant_dimrrUdimension_as_valuerZ) r,r&r1rrrrallow_negative_indexd_arrs r-r4r4I sl  7<<' C SZZM 2GMMB CC ME4= 1Q D}223 55'M&&-45M8JJ$'W]]$:% a B$6$67 mmA!c2::&'D,@,@,C  c..Aq1u1fQiPQ q56!$DQCH99: :  c..q&)<= "A!c2::&'  AEa#221fQi@@qQ q56!$DQCH99: :  a&&q&)4e mmCJJq1ua%i34 mmA?@ -r/r) r,rr&r}r'r}r(zSequence[int] | Noner]r) r,Array | np.ndarrayr&(Array | np.ndarray | Sequence[ArrayLike]r3r r1bool | Sequence[bool]r]r) r,rZr<rr&zArray | Sequence[ArrayLike]r1r\r]r)r,rr&rrir?r3r rbrwrcrwrdr\r]r)r,rrrrrrir|rcrwrbrwrdr\r]r)r,rrrrzCallable[[Array], Array]rir|rr rcrwrbrwrdr\r]r)rrrrrr}r]r)rr) r,rZr int | Nonerr]rrrrr]r)rT) r,rZrrrrrrwr]rr) r,rZrrrrrrr1rwr]r) r,rZrrrrrrwr1rwr]r) r,rZr<rrrrrr1rwr]r) r,rZr<rrrrrr1rwr]r)r]zmesh_lib.AbstractMesh)r]zir.Value)rz str | Noner]rw)r]zP | None)rFzmlir.LoweringRuleContextrz core.Jaxpr)r,rZr&r[r1r\r]zlist[ArrayLike]) __future__rcollections.abcrrr^r functoolsrr typingrweakrefnumpyrljax._srcrr r r r r jax._src.traceback_utilrrrrDjax._src.interpretersrrrrpe jax._src.laxrrrQjax._src.lax.utilsrrrjax._src.lib.mlirrjax._src.lib.mlir.dialectsrjax._src.named_shardingrjax._src.partition_specrrjax._src.typingrrr jax._src.state.indexingr! jax._src.utilr"r#r unsafe_mapr unsafe_zipryrjr.r,r9r=r?EnumrLrzr|rrrrrWeakKeyDictionaryrrJrr*rrrrrrrrrr rrr&r2r<standard_vma_ruler) deflinear2fancy_transposesprimitive_batchersdef_implrArHregister_loweringrSrVr\rmrrr|rrrrrr7primitive_jvpsprimitive_transposescustom_staging_rulescustom_typechecksrrrrrrrrrr;rrrrrrrrrrrArHrLrfrirmrrvdefjvprrzrrrrrrrrrrrrrfancy_primitive_batchersskippable_batchersrrrrrrrrrr rrArRAddOp SubtractOpr4rCr/r-rs #.  %0%$*&4+ !*1633&,CZCZ   +/5K&5K'5K385Kz59 <Q <Q;<Q<Q 2 <Q  <QF59 ;F ;F)2;F.;F2 ;F  ;F|$4Z$4N'5 '5\#(&+26k4kk k $ k 0 k"'k\%4j%4N &78#5)- S- S-'0S-;DS-,S-S-59S- ' S-38 S-9S-j &78 % +/< <<</ <  << )< <9<~ &78#5)- /- /-'0/-;D/-,/-/-59/- ' /-38 /-9/-b &78#5)- /- /-'0/-;D/-,/-/-59/- ' /-38 /-9/-b &78#5)- /- /-'0/-;D/-,/-/-59/- ' /-38 /-9/-d3L'2K2K2M/M "5)- =- =-#(=- =--=- =-  =- 59 =- ' =- 38 =-@"0 #5)- P- P-'0P-;DP-,P-P-59P- ' P-38 P-d0,/0H?(H?H?(+H?49H?VGH"&9(9(+09(|78>>9=>>&/>>%(>>03>>26>>BG>>F:>8(9= 8( )8("8(268(26 8(BG 8(~@D DE(1DE-6DE>ADE9= DEIN DET@DE5(1E5:CE5&)E59=E5INE5P$/L ,L @(( . W+?&-d.D.Dg&N P g,-5G';G$  ;;Aw - R . ^ ) N$?2  %8/$Q'. T++_ =, . &8/"+H('EO$/KO,+F(*G'A (<=$2. = +0,$&F*M T++-C D73 5 -F()(./( 23A -/JK #L 3 9M2 _LD 8cL  +#BL.Q,f *H$Q'7L T++X 6 8  ($d+$:!(=H% BJ (5Cnx/ OfR< I5=IX +@80'%R+%\7Lr#,m$Q'7M T++] ;0 2 $++> #N- )01OQ^)_ &3:;QS`3a!!-0-9M*",m$Q'7M T++] ;0  $++> #N- )01OQ^)_ &3:M4+!!-0-9M*",m$Q'7M T++] ;0 2   - D   *E & -0 !!-0-9M*iV#,m$Q'7M T++] ;0 2  -0 !!-0-9M*#*+@-#P- ",m$Q'7M T++] ;0 2  -0 !!-0-9M*#*+@-#P- bH1%f ,i$Q'7M T++Y 70  2 ,)%< " ), !!),)5I& ,1>!+1>fy.1}n5}n5}n5}n5}n550#f %;   %@ 3 3;323 3r/