ukiddlmZddlmZddlmZmZddlZddlZddlm Z ddl Z ddl Z ddl mZmZmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZddlmZddlmZddlm Z ddlm!Z!ddl"m#Z#m$Z$m%Z%ddl&m'Z'ddl&m(Z(ddl&m)Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0m1Z1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;ddlZ>m?Z?m@Z@ddlAmBZBmCZCmDZDmEZEmFZFddlGZHeBeIcZIZJeKeLdfZMe*jZNeKeOdfZPeZQdCd!ZRGd"d#ZSd$ZTeFd%d&'d(ZUeFd%d&'d)ZV dDd*ZWGd+d ejZYeseDe*jeYZYdEd,ZZ dFd-Z[e\eYd.de\eYd/d0 dG dHd1Z] dGd2Z^ dG dId3Z_dd4 dJd5Z`d6ejeY<d7ZbebejeY<d8Zde jeYedejjrejjeYdKd9Zjd:Zkd;ZleFd%d&'d<ZmeFd%d&'d=Znd>Zoeoe!jeY<d?Zqeqe!jej<d@Ztete!jej<dAZveve!jej<dBZxexe!jej<y)L) annotations) defaultdict)CallableSequenceN)partial)Any TYPE_CHECKINGcast)api) basearray)config)core)dispatch)dtypes)errors)literals)profiler)util) xla_bridge)are_hlo_shardings_equal)mlir)pxla) AutoLayoutFormatLayout)_jax)jaxlib_extension_version) xla_client)empty_concrete_mesh)Sharding)broadcast_prefix tree_flattentree_unflatten) PmapShardingSingleDeviceSharding NamedShardingdevice_replica_id_map hashed_indexnum_addressable_indiceslocal_to_global_shape_internal_use_concrete_mesh) ArrayLike DLDeviceType DTypeLike ExtendedDType)safe_zipunzip3 use_cpp_classuse_cpp_methodcache. ArrayImplc|jj}t|dk7rtdt|d|dS)NzpWhen making an array from single-device arrays the input arrays must have one shard each. An argument array had z shard(s).r)sharding_internal_device_listlen ValueError)adevicess I/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/array.py _get_devicer?>sN JJ , ,'\Q  669'l^: O PP ceZdZdZ d d dZdZejd dZejd dZ e dZ e dZ y) SharduA single data shard of an Array. Attributes: device : Which device this shard resides on. index : The index into the global array of this shard. replica_id : Integer id indicating which replica of the global array this shard is part of. Always 0 for fully sharded data (i.e. when there’s only 1 replica). data : The data of this shard. None if ``device`` is non-local. Nc<||_||_||_||_yN)_device _sharding _global_shape_data)selfdevicer8 global_shapedatas r>__init__zShard.__init__SsDLDN%DDJr@c  d|jd|jd|jd|jd S#t$r d|jd|jdcYSwxYw)Nz Shard(device=z, index=z , replica_id=z, data=))rJindex replica_idrLr;rIs r>__repr__zShard.__repr__Zss@dkk_HTZZLA OO,GDII;aAB @T[[O7499+Q ??@s69&A"!A"c |jj}||j|j }|J|S#t$rtd|jddwxYw)Nz(Cannot calculate indices from sharding: z<. Please create a device to index mapping for your sharding.)rFdevices_indices_mapAttributeErrorr;rGrJ)rIdevice_indices_map_fnrPs r>rPz Shard.indexas{?"nn@@ "$"4"4 5dkk BE    L ? A..)*44 5:>??s =$A!c\t|j|j|jSrD)r'rFrGrJrRs r>rQzShard.replica_idms" 1C1C DT[[ QQr@c|jSrD)rErRs r>rJz Shard.deviceqs <<r@c|jSrD)rHrRs r>rLz Shard.dataus ::r@rD)rJDevicer8r rKShaperLzNone | ArrayImpl | PRNGKeyArray)returnIndex)r]int) __name__ __module__ __qualname____doc__rMrS functoolscached_propertyrPrQpropertyrJrLr@r>rBrBGs} 8<4@    RR    r@rBc||}|j|tj|}|jjdi||_|S)z=Method to reconstruct a device array from a serialized state.rg) __setstate__r device_putavalupdate)funargs arr_state aval_statenp_value jnp_values r>_reconstruct_arrayrszsH $Z(  "nnX&)(9>>((6:6). r@F)max_sizetrace_context_in_keyc|j|}t}g}t|jD]9\}}t |}||vs|j ||j ||f;|SrD)addressable_devices_indices_mapset enumeratevaluesr(addappend)sshapemap_seen_h_indicesl array_indexrPh_indexs r>_cached_index_calcrsu * *5 1$5.!%dkkm4%k55!Gn$!hh U#$ % (r@cLtjdk(ryt|j|j Dchc] }t |c}}t|j |j Dchc] }t |c}}||k(Scc}wcc}w)Nr7F)r process_countr:rUr{r(rx)r~rvnum_unique_indicesnum_addressable_unique_indicess r> _process_has_full_value_in_mcjaxrs1$  ! 5 5e < C C EF1|AFH#& ! A A% H O O QR1|AR$T = ==GRs B>B!c |j}|D]m}|j|k7rtd|jd|d||j|k7s@td|d|jd|jd|y) z:Validates that per-device arrays are valid and consistent.z8Input buffers to `Array` must have matching dtypes. Got z , expected z for buffer: zExpected shard shape z- doesn't match the single device array shape z. Shape of Array is z with sharding N)dtyper;r str_short)arraysr8rkexpected_shapeexpected_dtypedbs r>/_validate_shape_and_dtype_for_per_device_arraysrs::.  b xx>!   +n%5]2$ H  xx>!  !.!12 "z)= ^^  ohZ 9  r@ceZdZUded<ded<ded<ded<ded <d ed <e dA dBd Zd ZedCdZedZ edZ edZ edZ edZ edZedDdZdZdZdZdZdZdZdZdZdEdZdZd Zd!Zd"ZedDd#Zd$Zd%ZedDd&Z dFd(Z!d'd'd'd'd) dGd*Z"dHd+Z#d,Z$ed-Z%eed.Z&ed/Z'dId0Z(ed1Z)ed2Z*dJd3Z+e,jZdKd4Z.ed5Z/edKd6Z0ed7Z1ed8Z2d9Z3ed:Z4edLd;Z5ed<Z6e7jpd=Z9ee,jte7jpd>?dMd@Z;y')Nr5core.ShapedArrayrkr rFzlist[ArrayImpl]_arraysbool _committed _skip_checksznp.ndarray | None _npy_valuec||_||_||_d|_|Dcgc]}|jd}}|rt j jr'|j||j|j}||_ycc}wNr) rkrFrrrr enable_checksvalue_check_and_rearrange)rIrkr8r committedrr<s r>rMzArrayImpl.__init__sv DIDNDODO$* +qaiil +F + 6//55((KfDL,sBc|Dcic]}t|j|}}|j}t|t|k7r#t dt|dt|t |j }|Dchc]}|j} }t|t|k7r0|Dcgc]}t|j} }t d| || z } | r-| |z } || z } d}| r |d| dz }| r |d| dz }t |t||||j|j |j}|Dcgc]}||jc}Scc}wcc}wcc}wcc}w) Nz Expected zW per-device arrays (this is how many devices are addressable by the sharding), but got zsWhen making an array from single-device arrays, the input arrays must be from distinct devices, but got device IDs z?Addressable devices and per-device arrays devices do not match.z Sharding contains devices z+ that are not present in per-device arrays.z# Per-device arrays contain devices z& that are not present in the sharding.)r8rkr) r?idaddressable_devicesr:r;rykeysr shard_shaper_addressable_device_assignment)rIrr8rkrdevice_id_to_bufferaddressable_devarray_device_idsdaddressable_device_idsbuffer_device_idsdiffdev_in_sharding_not_in_arraysdev_in_arrays_not_in_shardingerr_msgaddressable_darJs r>rzArrayImpl._check_and_rearranges?";r?--??    " ## 4 4D & IsE(E-7E2E7c.|jjSrD)rkrrRs r>rzArrayImpl.shape 99??r@c.|jjSrD)rkrrRs r>rzArrayImpl.dtyperr@c,t|jSrD)r:rrRs r>ndimzArrayImpl.ndims tzz?r@c@tj|jSrD)mathprodrrRs r>sizezArrayImpl.size s 99TZZ  r@c|jSrD)rFrRs r>r8zArrayImpl.sharding s >>r@c|jt|jtr"t |jj dS|jSr)_check_if_deleted isinstancer8r%list device_setrRs r>rJzArrayImpl.devicesA$--!56 $--** +A .. ==r@c.|jjSrD)rk weak_typerRs r>rzArrayImpl.weak_types 99  r@c|jSrD)rrRs r>rzArrayImpl.committeds ??r@c\ |jdS#t$r}td|d}~wwxYw)Nrzlen() of unsized object)r IndexError TypeError)rIerrs r>__len__zArrayImpl.__len__ s3: ZZ] : / 0c9:s + &+cVtj|t|jSrD)rcheck_bool_conversionr_valuerRs r>__bool__zArrayImpl.__bool__&st$  r@c`tj||jjSrD)rcheck_scalar_conversionr __float__rRs r>rzArrayImpl.__float__*s#  & ;; ""r@c`tj||jjSrD)rrr__int__rRs r>rzArrayImpl.__int__.s#  & ;;    r@c`tj||jjSrD)rrr __complex__rRs r>rzArrayImpl.__complex__2s#  & ;; " " $$r@cVtj|t|jSrD)rcheck_integer_conversionhexrrRs r>__hex__zArrayImpl.__hex__6!!$' t{{ r@cVtj|t|jSrD)rroctrrRs r>__oct__zArrayImpl.__oct__:rr@cjtj|tj|jSrD)rroprPrrRs r> __index__zArrayImpl.__index__>s#!!$' 88DKK  r@c8|jj|SrD)rtobytes)rIorders r>rzArrayImpl.tobytesBs ;;  u %%r@c6|jjSrD)rtolistrRs r>rzArrayImpl.tolistEs ;;   r@c~|jdk(rt|jd|St|j|S)Nrrg)rformatr)rI format_specs r> __format__zArrayImpl.__format__Hs3 yyA~ DKKO[ 11 DKK --r@cddlm}ddlm}|j t |j trtjjrWt |tr|n|f}td|Dtdft|jt|z zz}nkt |ts*|ftdft|jdz zz}n1|tdft|jt|z zz}t|j j|jj!} |j#|}||j&|}t)t+|} tjjr.tdt-D} |j/|| }t1|j2| |gdd S|j5||S#t$$rd}YwxYw) Nr)lax)indexingc3`K|]&}t|trt||dzdn|(yw)r7N)rr_slice).0is r> z(ArrayImpl.__getitem__..Xs0 CDZ3%7E!QUD !Q > s,.r7c3HK|]\}}t|ts|ywrD)rr_)rrxs r>rz(ArrayImpl.__getitem__..msKTQ 1c8JqKs"") dimensionsFTrr) jax._src.laxrjax._src.numpyrrrr8r$r pmap_no_rank_reductionrtuplerr:rrUr{rPr;rr%r?rzsqueezer5rkrewriting_take) rIidxrrcidx padded_cidxindicesarr_idxoutr8dimss r> __getitem__zArrayImpl.__getitem__Os '$--.  & & , , e,s3& HL  4[Nc$**oD 9 :; #u%%+3tzz?Q3F!GG+uT{nDJJ#c(0JKK+dmm77 CJJLMg-- ,  ll7#' C(89  ( ( . .KYt_KK$ CD 1# HHhTK K  " "4 --!sG<< H  H cjdk(r tdjsjsJt j j s jrdjdDStj tr!fdtjdDSfdtjdDS)Nrziteration over a 0-d arrayc3JK|]}|jD]}|ywrD)_unstack)rchunksls r>rz%ArrayImpl.__iter__..|s#Qu@PQ"QQs!#dc3(K|] }| ywrDrgrrrIs r>rz%ArrayImpl.__iter__..~s6AQ6sc3bK|]&}tjj|(ywrD)r rjrr s r>rz%ArrayImpl.__iter__..s"M1t{{1~.Ms,/) rris_fully_replicatedis_fully_addressableris_single_device_shardingr8 _chunk_iterrr$rangerrRs`r>__iter__zArrayImpl.__iter__vs yyA~ 2 33  % %)B)BB B  + +DMM :d>V>VQ 0 0 5QQ dmm\ 26tzz!}!566Ndjjm8LMMr@c.|jjSrD)r8r rRs r>r zArrayImpl.is_fully_replicateds == , ,,r@c`d}|j1|jjrd|jjd}nd|jj}t |j t r@|j jjr d|jd|d|j dS|js |jrtjd}|jd k(rd |j}nJ|j jsd |j}n$tj |j"|d d| }t%||j'dz dz}d}|t%|zdz|kDrdt%|z}||d ||dS|d |jd|dS)NzArray(zdtype=z, weak_type=Truez Array(shape=z, z , sharding=rO linewidthrz [], shape=zshape=,)prefixsuffix separatormax_line_width r7 )rkrrnamerr8r&spec unreducedrr r npget_printoptionsrhas_addressable_devices array2stringrr:rfind)rIr dtype_str line_widthr~ last_line_lenseps r>rSzArrayImpl.__repr__s F yy!4!44::??++;&>&&(5j a %}}44TZZL ! OODKKs&*: G!fqwwt},q0m c Y '! +j 8CKxs!C5 1 --xvdjj\I;a 88r@ct|jtr+|jjjr t |S|j s |jrt|jSt |SrD) rr8r&rrreprr r strrrRs r>__str__zArrayImpl.__str__sV$--/DMM4F4F4P4P $Z " "d&>&>   $Zr@c.|jjS)aIs this Array fully addressable? A jax.Array is fully addressable if the current process can address all of the devices named in the :class:`Sharding`. ``is_fully_addressable`` is equivalent to "is_local" in multi-process JAX. Note that fully replicated is not equal to fully addressable i.e. a jax.Array which is fully replicated can span across multiple hosts and is not fully addressable. )r8r rRs r>r zArrayImpl.is_fully_addressables == - --r@NcV|ind|i}tj|jfd|i|S)Ncopyr)rasarrayr)rIrcontextr.kwdss r> __array__zArrayImpl.__array__s.2FD>D ::dkk 7 7$ 77r@)stream max_version dl_devicer.cddlm}|jj}t |dkDrt dt |d|\}|||||||S)Nr) to_dlpackr7znto_dlpack can only pack a dlpack tensor from an array on a singular device, but an array with a Sharding over z devices was provided.)r3r4 src_devicer5r.)jax._src.dlpackr7r8rr: BufferError)rIr3r4r5r.r7rrJs r> __dlpack__zArrayImpl.__dlpack__so*))J :  558_4EF   GF T&!, &(   r@ct|jdk7r tdddlm}|j dk(r|j dfS|j dk(rut|jj}d|vr |j}nd|vr |j}ntd |t|j}| td ||fStd |j ) Nr7z/__dlpack__ only supported for unsharded arrays.r)r-cpugpucudarocmz%Unknown GPU platform for __dlpack__: z-Couldn't get local_hardware_id for __dlpack__z@__dlpack__ device only supported for CPU and GPU, got platform: ) r:rr:r9r-platformkDLCPUr?clientplatform_versionkDLCUDAkDLROCMlocal_hardware_id)rIr-rDdl_device_typerGs r>__dlpack_device__zArrayImpl.__dlpack_device__s 4<<A I JJ, }}%  ! ## E !$T*11BB # #%-- % %%--A,-/0 0&d+==  "IJJ . ..  L ]]_   r@c|jj\}}}d|jji}t||||ffS)Nr)r __reduce__rkrrs)rIrmrnrorps r>rKzArrayImpl.__reduce__sC;;113Cytyy223J dIz B CCr@ct|jdk7r td|jdjS)Nr7z?unsafe_buffer_pointer() is supported only for unsharded arrays.r)r:rr;unsafe_buffer_pointerrRs r>rMzArrayImpl.unsafe_buffer_pointers< 4<<A " ## <<? 0 0 22r@czt|jdk7r td|jdjS)Nr7zB__cuda_array_interface__() is supported only for unsharded arrays.r)r:rr;__cuda_array_interface__rRs r>rOz"ArrayImpl.__cuda_array_interface__s; 4<<A + ,, <<? 3 33r@cr|jd}|j}||jjzS)z>Returns the total global on-device size of the array in bytes.r)ron_device_size_in_bytesr8 num_devices)rIarrper_shard_sizes r>rQz!ArrayImpl.on_device_size_in_bytess4 ,,q/C002N DMM55 55r@cN|j|jjSrD)rr8rrRs r>r=zArrayImpl.devices s == # ##r@ctd)NzBarr.device_buffer has been deprecated. Use arr.addressable_data(0)rVrRs r> device_bufferzArrayImpl.device_buffers J LLr@ctd)NzTarr.device_buffers has been deprecated. Use [x.data for x in arr.addressable_shards]rWrRs r>device_bufferszArrayImpl.device_bufferss \ ^^r@cx|j|jr|jS|j|SrD)rr _fully_replicated_shardr)rIrPs r>addressable_datazArrayImpl.addressable_datas5   ) ) ++ << r@c |jg}|jD]<}|jtt ||j |j |>|SrD)rrr}rBr?r8r)rIrr<s r>addressable_shardszArrayImpl.addressable_shardssP C \\F jj{1~t}}djj!DEF Jr@c|jrtd|jS ttj|j |jS#t j$rQ}|j^}}t|tur,|jdrtd|jcYd}~Sd}~wwxYw)N UNIMPLEMENTED) is_deletedrr8rfrom_pjrt_layout _pjrt_layoutrJaxRuntimeErrorrntyper* startswith)rIemsg_s r>rzArrayImpl.format's  D$-- (( F++D,=,=>MM##   gcA cc cnn_=dDMM** s$2AB?.AB:3B?9B::B?c |j|jr |jSg}|jDcic]}t |j |}}|j jD]a}|j|j d||j }nd}|jt||j |j|c|Scc}w)zReturns list of all `Shard`s of the Array across all devices. The result includes shards that are not addressable by the current process. If a `Shard` is not addressable, then its `data` will be `None`. N) rr r_rr?rr8rgetr}rBr)rIrr<rglobal_darrays r> global_shardszArrayImpl.global_shards6s     $ $$ C9=FA;q>,,a/FFMM,,D  d 3 ?#HKK0 jjx EBC D JGsCc||jy|jD]}|jd|_d|_yrD)rdeleter)rIbufs r>rqzArrayImpl.deleteKs9 || || jjlDLDOr@cT|jytd|jDS)NTc3<K|]}|jywrD)rb)rrrs r>rz'ArrayImpl.is_deleted..[s8Cs~~8s)ranyrRs r>rbzArrayImpl.is_deletedTs& ||  84<<8 88r@cr|jr'td|jjdy)Nz"Array has been deleted with shape=.)rb RuntimeErrorrkrrRs r>rzArrayImpl._check_if_deleted]s:   .tyy/B/B/D.EQ G IIr@ch|j|jD]}|j|SrD)rrblock_until_ready)rIrs r>rzzArrayImpl.block_until_readybs2ll Kr@cyrDrgrRs r>)_single_device_array_to_np_array_did_copyz3ArrayImpl._single_device_array_to_np_array_did_copyisr@c>|jdjyr)rcopy_to_host_asyncrRs r>'_copy_single_device_array_to_host_asyncz1ArrayImpl._copy_single_device_array_to_host_asyncmsLLO&&(r@c.|j|jy|jr'|jjr|j yt |j|jD]"\}}|j|j $yyrD) rrr r8r!rrrr)rIrrjs r>r~zArrayImpl.copy_to_host_asyncqsy  ! !dmm&K&K 446$T]]DJJ?B$!Q Q??AB r@znp.asarray(jax.Array))rc|j|jd|jrE|jjr/|j \}}d|j _|r||_|S|js+t|j|js tdt|j|jD]"\}}|j|j$tj |j|j"}t|j|jD](\}}|j|j \||<}*||_d|jj _|jS)NFa%Fetching value for `jax.Array` that spans non-addressable (non process local) devices is not possible. You can use `jax.experimental.multihost_utils.process_allgather` to print the global array or use `.addressable_shards` method of jax.Array to inspect the addressable (process local) shards.)rrr r8r!r|flags writeabler rrrxrrrremptyr)rI npy_valuedid_copyrrjinds r>rzArrayImpl._value{sG   ! !dmm&K&K"LLN 8$) ! %$/ ''.t}}djjI ?  %T]]DJJ?B$!Q Q??AB((4::tzz2i&t}}djjAX&!S LLOUUW #X!do(-doo% ??r@)F) rkrr8r rzSequence[ArrayImpl]rrrr)r]r\)r]r)C)NNN)r3zint | Any | Noner4ztuple[int, int] | Noner5ztuple[DLDeviceType, int] | Noner.z bool | None)r]ztuple[enum.Enum, int])r]z set[Device])rPr_r]r5)r]zSequence[Shard])r]ztuple[np.ndarray, bool])r] np.ndarray)r5r5s>  6;*.2&,I\       ! !         : #!%!& .%.N N - -94  .  .8 6:7;>B%) 4 ; # (>D 33  4 4 66 $ L L ^ ^        (99I  )) BB 9X//6MN!O !r@c:tdt||DS)Nc3K|]8\}}t|tr#|jxs||jxsdz :yw)rN)rrstopstart)rr~dims r>rz(_get_shape_from_index..s; !S Au  vv}A&s>A)rr0)slcrs r>_get_shape_from_indexrs# S%( r@c |"|r|dj}|Std|dtj||}|r4|dj|k7r"td|d|d|djd|S)NrzbIf the Array has no addressable shards, `dtype` must be provided via the `dtype` argument to `jax.z`.zIf `dtype` is provided to `jax.z@`, it must match the dtype of the addressable shards. Got dtype=z and shard dtype=)rr;r!check_and_canonicalize_user_dtype)rrfnames r>_get_and_check_dtypers  ] Qiooe ,  ..3WB 8 99  4 4UE BE &)//U*  +E732279!9??#2 ' (( ,r@__hash____array_priority__rc (t|tr |jnd}t|trt d|t|tr |j n|}t|t st d|dt| d fd }|jrKt|jj}|tdft|zgt|z}nB|j|}t|j}|D cgc] } ||| }} t!||d}|j#|} t%j&t%j(|||} t+|| | |d} |rv|d} t| t,ra| j.rU|jrI| jr=| j j0t3|k(r| j4j|k(r| St7j8| j:t6j<r?t?j@||} | j:jBjE| || d S|G|Dcgc]}t|tG|}}t?j@||} t-| || d St| t,r2t| jId kDrt?j@||}tKjL| |||Scc} wcc}w) aReturns a ``jax.Array`` via data fetched from ``data_callback``. ``data_callback`` is used to fetch the data for each addressable shard of the returned ``jax.Array``. This function must return concrete arrays, meaning that ``make_array_from_callback`` has limited compatibility with JAX transformations like :func:`jit` or :func:`vmap`. Args: shape : Shape of the ``jax.Array``. sharding: A ``Sharding`` instance which describes how the ``jax.Array`` is laid out across devices. data_callback : Callback that takes indices into the global array value as input and returns the corresponding data of the global array value. The data can be returned as any array-like object, e.g. a ``numpy.ndarray``. dtype: The dtype of the output ``jax.Array``. If not provided, the dtype of the data for the first addressable shard is used. If there are no addressable shards, the ``dtype`` argument must be provided. Returns: A ``jax.Array`` via data fetched from ``data_callback``. Examples: >>> import math >>> from jax.sharding import Mesh >>> from jax.sharding import PartitionSpec as P >>> import numpy as np ... >>> input_shape = (8, 8) >>> global_input_data = np.arange(math.prod(input_shape)).reshape(input_shape) >>> global_mesh = Mesh(np.array(jax.devices()).reshape(2, 4), ('x', 'y')) >>> inp_sharding = jax.sharding.NamedSharding(global_mesh, P('x', 'y')) ... >>> def cb(index): ... return global_input_data[index] ... >>> arr = jax.make_array_from_callback(input_shape, inp_sharding, cb) >>> arr.addressable_data(0).shape (4, 2) Nzp`Layout.AUTO` cannot be used in place of a device-local layout when calling `jax.make_array_from_callback`. Got z6sharding should be an instance of `jax.sharding`. Got z of type c|J|}t|tjrtjdt j |}t|tjtjtjfr8tjtj||jd}|St|tr:tjtj|tj d}|S)NzFjax.make_array_from_callback cannot be called within a traced context.rFr)rrTracerrUnexpectedTracerErrorrcanonicalize_valuerTypedInt TypedFloat TypedComplex TypedNdArrayrr/rrbool_)rPr data_callbacks r>get_dataz*make_array_from_callback..get_datas   eA!T[[!  ( (   !!!$A!h'')<)<++-.    1AGG < Na H At     1BHH = Oa Hr@make_array_from_callback)rrkr8rTrr7)rP Index | Noner]z.ArrayImpl | literals.TypedNdArray | np.ndarray)'rrlayoutrrr8r rfr rr9addressable_device_listrr:rxrrrrupdate_aval_with_sharding ShapedArrayrr5r_device_assignmentrrr issubdtyperextendedr rj_rulesmake_sharded_arrayr%r=rbatched_device_put)rr8rrdllrr=per_device_valuesdevice_to_index_maprJrrk first_valuerrs ` r>rrs\&h7T#Z  DDL: O PP#-Xv">X  H( Hh '  @ Kh  " ##  5 ,!!811IIJG!5;.3u:"=>?#g,N"BB5I&++-.G>> from jax.sharding import PartitionSpec as P >>> mesh_rows = 2 >>> mesh_cols = jax.device_count() // 2 ... >>> mesh = jax.sharding.Mesh(np.array(jax.devices()).reshape(mesh_rows, mesh_cols), ('x', 'y')) >>> sharding = jax.sharding.NamedSharding(mesh, P(('x', 'y'),)) >>> rows_per_device = 2 >>> feature_length = 32 >>> per_device_shape = (rows_per_device, feature_length) >>> per_host_shape = (rows_per_device * len(mesh.local_devices), feature_length) >>> per_host_generator = lambda : np.arange(np.prod(per_host_shape)).reshape(per_host_shape) >>> per_host_data = per_host_generator() # replace with your own per-host data pipeline that outputs numpy arrays >>> global_shape = (rows_per_device * len(sharding.device_set), ) + per_device_shape[1:] >>> output_global_array = jax.make_array_from_process_local_data(sharding, per_host_data, global_shape) ... >>> assert output_global_array.addressable_data(0).shape == per_device_shape >>> assert output_global_array.shape == global_shape NB: While most shardings are uniform, It is possible to design an exotic sharding mesh where each process's devices will be arranged in a non-grid like pattern in some dimensions, or for indices to overlap non-trivially. Such sharding is called "non-uniform" in those dimensions. In that case, the global shape along those directions must match local shape as there is no meaningful way to represent all needed per-process data in non-overlapping fashion. For example for global_shape 4x4 if sharding looks like this:: 0123 2103 4675 4567 with 4 processes, containing devices (0,1), (2, 3), (4, 5), (6, 7) respectively. Then the data for each host look like:: xx.. ..xx .... .... .xx. x..x .... .... .... .... x..x .xx. .... .... xx.. ..xx the sharding is uniform on rows (each host requires either rows 1-2, or rows 3-4) and non-uniform on columns (hosts require overlapping but not matching set of columns). Thus local data must have the shape 2x4 or 4x4 for all hosts, even though each host can potentially fit into 2x2 shape. In this case user must provide global_shape explicitly and for local_shape=(2, 4), potentially valid global shapes are (2, 4) and (4, 4). On the other hand for sharding:: 0213 x.x. .x.x. .... .... 0213 x.x. .x.x. .... .... 4657 .... .... .x.x x.x. 4657 .... .... .x.x x.x. for local_shape=(2, 2) this function can accept a choice of 2x2, 2x4, 4x2 and 4x4 global shapes. Setting global_shape to None, is equivalent to setting it to (4, 4) in this case. Args: sharding: Sharding of the global array. local_data: Data on the host to be placed on local devices. Each dimension should either match global_shape, or match num_addressable_indices(dim). global_shape: The target shape of the global array. If None, will infer from local_data and sharding. Returns: Tensor that will have sharding=sharding and of shape global_shape. "make_array_from_process_local_datac.|duxst|tSrD)rrrs r>z4make_array_from_process_local_data..sT 9Z5%9r@)is_leafr7zWhen calling `make_array_from_process_local_data` on a single process, global_shape should be None or equal to local_data.shape.Got global_shape=z and local_data.shape=rw)r"r!maprr pspec_to_shardingrrziprr;rj_array_from_process_local_datar#) r8 local_datarKlocal_data_flattreedef sharding_flatglobal_shape_flatr~rrLrrs r>rrRs#h**5/7"8Z8- c##%IJ-'J9;1$%7 1 1< 2235!!"  ,    >>*h // #?MCTU W W T1e (a 7 W# W  %% WsC6c j}|#t||}|Jd|vr"td|dd|vrtd|dgtt j|D]L\}\}}j ||k(||k7s#t |||}||k7s6td|d|d|d|d |j|}tt|j} | Jt| ||D cgc]} gc} |jD];} | Jt| D]'\}} |j | jxsd )=tt|D]}tt!||<t#j$d fd dfd } t'||| Scc} w)Nz{Unable to compute global_shape due to non-uniform sharding. Specify global shape directly. Partially computed global_shape=rwz global_shape=z" has Nones. This is not supported.z\Invalid host data, each dimension should match either global or process shape. In dimension z, the process data has z elements. Process addresses z elements and global_shape=rrt)maxsizech|j|xsd|z}||z}t||Sr)rPr)rrendrslices_for_each_dims r> local_slicez3_array_from_process_local_data..local_slicesB  " ( (! 4{1~ EE +a. C  r@cR|Jfdt|D}t|S)Nc3XK|]!\}}|r|n||j#ywrD)r)rrrfull_dimrs r>rz=_array_from_process_local_data..cb..s3 As{ Asyy 99s'*)rzr)rP data_slicerrrs r>cbz*_array_from_process_local_data..cbs5   &J eJ' ((r@)rPrr]r,)rr*r;rzrr}r)rxnextiterr{rrrr:sortedryrd lru_cacher)rr8rK local_shaperdata_dim global_dim process_slicer_shardrj shard_indexrrrrrrs` @@@@r>rrs2   +(;?L  ## # |  N@La Q  |  &HI JJ (#, *  L)$  a (J OOH *+:-h<Hm ( " ++,#-DXJO++8/:oQ      ?? M t&--/0 1%   %e\:+6B)C")C'..04k  "" "K(43!##CIIN344 \" #Aa#C(;A(>$?@A t$%) "," ==1*Ds Grc t|tr t||d}tjtj ||d|}t j|jt jr)|jjj|||dSt|tr t|n|} t||ttt|dS#t $rat|tst!dt#|dt%d|Dr%t'd t)t+t"|wxYw) aReturns a ``jax.Array`` from a sequence of ``jax.Array``\s each on a single device. Every device in input ``sharding``\'s mesh must have an array in ``arrays``\s. Args: shape : Shape of the output ``jax.Array``. This conveys information already included with ``sharding`` and ``arrays`` and serves as a double check. sharding: Sharding: A global Sharding instance which describes how the output jax.Array is laid out across devices. arrays: `list` or `tuple` of ``jax.Array``\s that are each single device addressable. ``len(arrays)`` must equal ``len(sharding.addressable_devices)`` and the shape of each array must be the same. For multiprocess code, each process will call with a different ``arrays`` argument that corresponds to that processes' data. These arrays are commonly created via ``jax.device_put``. dtype: The dtype of the output ``jax.Array``. If not provided, the dtype of the first array in ``arrays`` is used. If ``arrays`` is empty, the ``dtype`` argument must be provided. Returns: A global ``jax.Array``, sharded as ``sharding``, with shape equal to ``shape``, and with per-device contents matching ``arrays``. Examples: >>> import math >>> from jax.sharding import Mesh >>> from jax.sharding import PartitionSpec as P >>> import numpy as np ... >>> mesh_rows = 2 >>> mesh_cols = jax.device_count() // 2 ... >>> global_shape = (8, 8) >>> mesh = Mesh(np.array(jax.devices()).reshape(mesh_rows, mesh_cols), ('x', 'y')) >>> sharding = jax.sharding.NamedSharding(mesh, P('x', 'y')) >>> inp_data = np.arange(math.prod(global_shape)).reshape(global_shape) ... >>> arrays = [ ... jax.device_put(inp_data[index], d) ... for d, index in sharding.addressable_devices_indices_map(global_shape).items()] ... >>> arr = jax.make_array_from_single_device_arrays(global_shape, sharding, arrays) >>> assert arr.shape == (8,8) # arr.shape is (8,8) regardless of jax.device_count() For cases where you have a local array and want to convert it to a global jax.Array, use ``jax.make_array_from_process_local_data``. $make_array_from_single_device_arraysFrTrz\jax.make_array_from_single_device_arrays `arrays` argument must be a list or tuple, but got rwc3PK|]}t|tj ywrD)rrr)rrSs r>rz7make_array_from_single_device_arrays.._s :C:c4;; ' :s$&zdjax.make_array_from_single_device_arrays requires a list of concrete arrays as input, but got types )rrrrrrrrrrrrrrr5r rrfrur;ryr)rr8rrrks r>rrsE^! = ?E  ' ' ueu5x A$ tzz6??3 ::   / /h:> 0 @@%fe44<&&  T8T(9*=v%F" $$   fd # Cfa) ** :6 ::  --0T61B-C,D F GG   s ?#C##A*E c|SrDrgrs r>rres!r@cVtj|j|jSrD)rrrkr8rRs r>_get_aval_arrayrgs ' ' 4== AAr@c tj|jS#t$r:}dt |vr'td|j j |d}~wwxYw)Nz9Fetching value for `jax.Array` that spans non-addressablezClosing over jax.Array that spans non-addressable (non process local) devices is not allowed. Please pass such arrays as arguments to the function. Got jax.Array: )r ir_constantrrxr*rkr)valrkrhs r>_array_mlir_constant_handlerrlst    CJJ ''  Cc!fL  ..1hh.@.@.B-C E FLMM  s! A$5AA$cJdg|jz}t|j}g}t|tr|n|f}t |D]\}}t|t r|||<|dz||<|j|5|tdk(rDt|tsJ|t|jt sJ|t|jt sJ||j||<|j||<t |t |t |fS)z2Returns start_indices, limit_indices, removed_dimsrr7N) rrrrrrzr_r}rrr)rSr start_indices limit_indices removed_dims tuple_idxrsub_idxs r>as_slice_indicesrs#.-syy/-,U+c#) * (lc7'3"mC"Q;mC# E$K   '00 '  s +4W4 +  c *3G3 *"==mC"<rz%shard_device_array..s63#&q#63s) r1r r:r+r _multi_slicershaped_abstractifyrr) rr=rr8rrrshardsrks ` r>shard_device_arrayrs/563*16303,-  !!S3w< F %%8 9J~~m]LIfJ  #$ x AAJJs BBctt}|jDcgc]}|j}}t |j j |jj}t||D]"\}}|t|j|$g}t||D]\}} |t|} | sEtj|gdgtjj g|j"gddcS| D])}|j%| hk(s|j||j| dtj&|j(|||Scc}w)NF) canonicalizer)rrr_rLrr8rUrr{r0r(r}r shard_argsxcArrayCopySemantics REUSE_INPUTrr=rrk) rr=rr8 candidatesrrbufs arr_indicesrrJcandidates_lists r>$shard_sharded_device_array_slow_pathrsW4 *22 3s#(( 3$ 3ajj44QWW=DDFG+4-.hc3|C !((-. $gw/'kc6 c!23O  __hZ$ 33??@188**/11244 ' 6( " C '  kk/"%&'  4 AA+ 4sE/c|j|j}|j|j}t|t|k(SrD)rxr{r) src_sharding dst_shardingr src_indices dst_indicess r>_fallback_check_via_indicesrsI<_sharding_indices_and_eqr sc #L$E$Ed$K$0$E$Ed$K M'  22JJ K  22JJ K L&  Vr@c tjdg}ggggf\}}}}g} tt||||D]\} \} } } }| j  t | j | t| j}| dn| jj| k(}| js!|r|r|j| td| jj }|rj|rh|jd|j| |j||j| |j| | j||s1|jt#j$| t'| | F| j)| jj+}t-j.| j r|jt1| ||| |jt3| ||| tjdt5j6|||| }t||D]\} }|| J||| <|S#t$r%t| j | | j}YwxYw)N_array_shard_argTz5Cannot reshard an input that is not fully addressablebatched_copy_array)r test_eventrzr0rr r8r:rNotImplementedErrorrrrr r}r9rr rjrrxr{rr rrr+batched_copy_array_to_devices_with_sharding)xs shardingslayoutscopy_semanticsresultsbatch_xs batch_devsbatch_shardings batch_indicesbatch_csrrr8rcs same_sharding same_layoutr=r copy_outscopy_outs r>r r sJ//$% '9;RR6(J (&/r9g~6'8#S"a "!XvrQ.qzz8S\Rm!.$ahhoo.GK ! ! ;q! CE E..FFg ;t'"x(Qs~~a)ABC::177CJJL  - -ajj 9 ..+AwJ K ..21gwQ SE#SJ//&'<< OX7)mY7ka 1:  GAJ .M Q1!**hPmQs*I*JJcjtjk(rMfd}tdk\r=t j }t j|||dj|S|Stjjtjr'jjj||St j||dS)Nc`tjjtjSrDrzerosrrfloat0)r global_avals r>handlerz-_array_global_result_handler..handler s XXk'' 77r@Tr) rrr$rr physical_avalrarray_result_handlerwraprrrglobal_sharded_result_handler)r% out_shardingrr& phys_avals` r>_array_global_result_handlerr.s&--'83&$$[1i  $ $Y 2688<W Fn {((&//:    # # A A\9 .. <94 r@cjtjk(rMfd}tdk\r=t j }t j||ddj|S|Stjjtjr'jjj||St j|ddS)Nc`tjjtjSrDr")rrks r>r&z,_array_local_result_handler..handlers XXdjj&-- 00r@r'Tr) rrr$rrr(rr)r*rrrlocal_sharded_result_handler)rkr8rr&r-s` r>_array_local_result_handlerr2s ZZ6== 13&$$T*i  $ $YD2688<W Fn tzz6??3 ::   9 9 h !!  H4 r@c >g}t|||D]\}}}|J|jtjgt}t j |}|j} |jtj|||gt| z| |S)Nr) r0rzrrnrrget_avalrr}rrr:) rrrrrrr8rrkr=s r>_token_shard_argr5/s '%b)W=6a6 >> 4 A == D55G NN4** hc'l*G56 6 .r@cttj||tdk\rd}j |Sfd}|S)Nr'c,tj|SrDrToken)rns r>wrapperz-_token_global_result_handler..wrapperAs ZZ r@c>|i|}tj|SrDr8)rnkwargsout_buf array_handlers r> old_wrapperz1_token_global_result_handler..old_wrapperEs!t.v.g ZZ  r@)r.rget_token_avalrr*)r%r,rr:r?r>s @r>_token_global_result_handlerrA=sH. \96-$   g &&! r@)r<r5r]r[)rz8Sequence[ArrayImpl | np.ndarray | literals.TypedNdArray]r8r rkrrr\)rr^rr\r]r\)rz>Sequence[basearray.Array | np.ndarray | literals.TypedNdArray]rz DTypeLike | ExtendedDType | Nonerr*rD) rr\r8zSharding | Formatrz#Callable[[Index | None], ArrayLike]rDTypeLike | Noner]r5)rrr8r rKz Shape | Noner]r5) rr\r8r rzSequence[basearray.Array]rrBr]r5)rSrrr^r]z8tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]])z __future__r collectionsrcollections.abcrrenumrdrroperatorrtypingrr r jax._srcr r r rrrrrrrrjax._src.op_shardingsrjax._src.interpretersrrjax._src.layoutrrr jax._src.librrrr jax._src.meshrjax._src.shardingr jax._src.tree_utilr!r"r#jax._src.sharding_implsr$r%r&r'r(r)r*r+jax._src.typingr,r-r.r/ jax._src.utilr0r1r2r3r4numpyrr unsafe_ziprr_r\r[rr^ PRNGKeyArrayr?rBrsrrrArrayr5rrsetattrrrrrcanonicalize_value_handlersrpytype_aval_mappingsrregister_constant_handleruse_simplified_jaxpr_constantsrliteralable_typesr|rrrrr r shard_arg_handlersr.global_result_handlersrr2local_result_handlersr5r9rA AbstractTokenrgr@r>rbs##.  ++9&&661)-&MM888NMPPCZ c3h  eSj 00f51  2  51 >2 > D  ,k kb)mBLL))4) J + . :t$ '-#CM CM-CM6CM CM(1CMRJ&\"&:>:>&.:>:>*3:>~#H H $H .GH H H T1<""9-B'6)$  y*FG ((..Y' I2 BB451222 5121d&6 " 1MD,,- 0K4++, '7 # 3OD../r@