ukijUddlmZddlZddlmZddlZddlmZddlZddl Z ddl Z ddl m Z ddl mZddl mZddl mZdd l mZdd l mZdd l mZdd l mZdd l mZddl mZddl mZddl mZddlmZddlmZddlmZddlmZddlmZddlm Z ddl!m"Z"ddl#m$Z$m%Z%ddl&m'Z(ddl)m*Z*m+Z+ddl,m-Z-m.Z.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5m6Z6m7Z7m8Z8ddl9m:Z:ddl;Zd"Z?ejeAe(jZCeCjZEe(jZFe(jZGe(jZHejeJcZJZKejeMcZMZNejePZQd#ZRd$ZSejdPd%ZUd&ZVe ZWGd'd(e jZYeYZZd(e[d)<ejd*Z]Gd+d,Z^e^Z_dQd-Z`dRd.ZaebZcd/e[d0<ejdSd1Zeej dTd2ZfdUd3ZgdVd4ZhdWd5ZidXd6ZjdYd7Zkd8Zl dZd9Zmejd:d#;d<Znejd=>Gd?d@Zp d[dAZq d\dBZr d]dCZs d^dDZtejdEZvd=ev_wevjetdFZyeyejev<dGZ{dHZ|evje|dIZ~eejevejev<e~ejev<dJZeejev<dKZej evedLMej evedNMdOZej evey)_) annotationsN)Sequence)partial)Any)api)array) basearray)config)core)dtypes)literals)pjit)traceback_util)util) xla_bridge) array_types)ad)batching)mlir) partial_eval)pxla)InternalFloatingPointError)LayoutFormat) xla_client) AbstractMeshMesh) record_scalarrecord_event_duration_secsrecord_event_time_span) PartitionSpec)Sharding) NamedShardingSingleDeviceSharding GSPMDShardingis_single_device_sharding) SourceInfoz&/jax/core/compile/jaxpr_trace_durationz//jax/core/compile/jaxpr_to_mlir_module_durationz*/jax/core/compile/backend_compile_durationFct|fi|}tjjd} ||}tjj ||S#tjj |wxYw)zEImpl rule that compiles and runs a single primitive 'prim' using XLA.F)xla_primitive_callabler disable_jit swap_local set_local)primargsparamsfunprevoutss L/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/jax/_src/dispatch.pyapply_primitiver4Tsht.v.#    & &u -$' :D   & +   &s A!A4c tjdfd}j|_j|_d|_t j|S)N!xla_primitive_callable_cache_missc~tjd5j|icdddS#1swYyxYw)NF)r eager_constant_foldingbind)r.r/r-s r3prim_funz(xla_primitive_callable..prim_funhs9  & &u -( TYY ' '(((s3<T)r test_eventname__name__ __qualname___apply_primitiverjit)r-r/r:s`` r3r)r)esF//56(ii())("(  cB|jtt|yN)def_implrr4)r-s r3 simple_implrEqs--./rAc\eZdZUdZded<ded<dZ d dZddZdd Zd Z d Z y )RuntimeTokenSetzJSee docstring for effects.py module for the calling convention for tokens.zdict[core.Effect, core.Token]current_tokenszdict[Device, RuntimeToken]output_runtime_tokensc i|_i|_yrCrHrIselfs r3__init__zRuntimeTokenSet.__init__D!#DrAc |jj|tjdtj}t |t jr3tj|tt|dtdStj|}t jtj |gdgt"j$j&g|gd}||j|<|S)Nrx)rHgetnpzerosbool_ isinstancer Tokenr device_putr#rr!r%get_replicatedr shard_argsxcArrayCopySemantics REUSE_INPUT)rMeffdevicestoks sharded_toks r3get_token_inputzRuntimeTokenSet.get_token_inputs    ! !#rxx288'< =C#tzz" ^^ }T'3/s1CDFF $$W-A**  C$"//;;__doc____annotations__rNrcrfrirkrnrAr3rGrGvsKR0/43$'30%/ $rArGruntime_tokensc,tjyrC)rxrnrwrAr3wait_for_tokensrzs""$rAc*eZdZgdZdddZdZdZy)LogElapsedTimeContextManager)fmtfun_nameevent start_timeNc.||_||_||_yrC)r}r~r)rMr}r~rs r3rNz%LogElapsedTimeContextManager.__init__sDHDMDJrActj|_|j-t|j|j|jyy)Nr~)timerrrr~rLs r3 __enter__z&LogElapsedTimeContextManager.__enter__s9iikDO zz **doo rAc$trytj}||jz }tjj rt jnt j}tj|r;tj||jj|j||jPt!|j||jt#|j|j||jyy)N)r~ elapsed_timer)_on_exitrrr log_compilesvalueloggingWARNINGDEBUGlogger isEnabledForlogr}formatr~rrr )rMexc_type exc_value tracebackend_timer log_prioritys r3__exit__z%LogElapsedTimeContextManager.__exit__s yy{Hdoo-L&,&9&9&?&?7??W]]L <( jjtxx==| / => zz **lT]] **doox$-- rArC)r}strr~rrz str | None)r=rtr> __slots__rNrrrwrAr3r|r|s8) rAr|c|dk(r|dkDSy)NtpuiFrw)num_argsplatforms r3should_tuple_argsrs d? rAc|jD]}||jjvsytj|D]}t ||syy)zCWhether there is a primitive given by user anywhere inside a Jaxpr.TF)eqns primitiver<r subjaxprsjaxpr_has_primitive)jaxpr prim_nameeqnsubjaxprs r3rrsV ZZcCMM&&& ..'h8Y/  rAzset[core.Primitive]%prim_requires_devices_during_loweringc|jD]}|jtvsytj|D]}t |syy)NTF)rrrr r jaxpr_has_prim_requiring_devices)rrrs r3rrsO ZZc }}== ..'h'1  rAcddlm}g}|jD]0}|jtj urx|j d}t|trt|jtrZt|j|jj|j|f|jtjurzt|j|jj|j!fd|j dD|j!fd|j dD.|j|j"ur|j dttrgt|j|jj|j!fd g|j d |j d D|jt$ust|j|jj|j!fd |j d D3t'j(|D]}|j!t+||S)Nr) shard_mapshardingc3&K|]}|f ywrCrw).0i source_infos r3 z-get_intermediate_shardings..sFa![!F in_shardingsc3&K|]}|f ywrCrw)rors r3rz-get_intermediate_shardings..sGa![!Gr out_shardingsmeshc3:K|]}t|fywrC)r#)rspecrrs r3rz-get_intermediate_shardings..s'R d+[9Rsin_specs out_specsc3^K|]$}t|tr|j|f&ywrC)rVr" memory_kind)rrars r3rz-get_intermediate_shardings..s2Kaq(+ 0I[!Ks*-r_)jax._srcrrrrsharding_constraint_pr/rVr#rrr'rr<appendjit_pextend shard_map_p device_put_pr rget_intermediate_shardings)rroutrrarrrs @@r3rrs! # ZZKc }}222 **Z a A} %*QVV\*Js 0B0BCk jj![!" $** $s 0B0BCk jjF3::n+EFF jjG3::o+FGG )// / ZZ d D, 's 0B0BCk jjRPszz*5P ;8OPRR , &s 0B0BCk jjK3::i+@KK)K,..'5hJJ)(345 *rAct|tjs1tj|st d|dt |dyy)N Argument ' ' of type z is not a valid JAX type.)rVr Tracer valid_jaxtype TypeErrortype)args r3 check_argr sM S$++ &$*<*s (rAc X|j|j}t|tsJ|tj k(rdnd}|j |j k(r"tjt|||S|jrd}n@tj|j jtg|j }t|j j"j%|j j&|j j(}t||j*|j,|dnt/|j1}t3j4||tj6}tjt|||S)Nrrdonate_argnums)otypes)r_logical_device_ids)_check_if_deletedrrVr#r\ DONATE_INPUT_device_assignmentrr@ris_fully_replicatedrS vectorizeindexintrrr_reshape axis_sizes axis_namesrrtupletolistr[reorder_shardsr]) rQtarget_shardingcopy inp_shardingr permute_ordernew_meshnew_snew_xs r3_different_device_order_reshardrBsr, L- 00 0 2 ? ??1T.$$(J(JJ 2377<"0 223 55%%M0BLL!C!C!I!I*-00<0O0OQM ""**<+<+<+G+GH""$(  !!/J/J#0#84!-"6"6"89 <%  Au&8&D&D E% /_- //4 66rAi)max_sizetrace_context_in_keyc|jj|jjk7ry|j||j|k7ry|jj|jjk7}t j }|rMt |dds@tjjs&td|jd|jd|S)zdst is a supported cross-host transfer.Fsupports_cross_host_transfersz The backend (z, z) does not support cross-host device transfers via ICI/NCCL. Please set jax_cross_host_transfer_socket_address and (optionally) jax_cross_host_transport_addresses flags to enable DCN-based cross host device transfers.) _internal_device_list device_kind_to_xla_hlo_shardingprocess_indicesr get_backendgetattr"CROSS_HOST_TRANSFER_SOCKET_ADDRESSr ValueErrorrplatform_version)ndim src_sharding dst_shardingdifferent_process_indsbackends r3!_is_supported_cross_host_transferr_s((44((445 ''-''-.  ((88((889  " " $' ':E B  7 7 = =  (()G,D,D+EF! ! "" rAT)frozencJeZdZUdZded<ded<ded<ded <d ed <d Zy )_DeferredShardArgzDeferred call to `pxla.shard_args`. Per-array impls return this object instead of a result array to indicate a deferred `shard_args` call. `_batched_device_put_impl` then batches all `_DeferredShardArg` objects into a single `shard_args` call. rrQr"razcore.AbstractValueavalbool committedr\copy_semanticscxtj|j|j|j|SrC)rglobal_aval_to_result_handlerrrar)rMshard_arg_results r3result_handlerz _DeferredShardArg.result_handlers4 +4 - - 4664>> ++; ==rAN)r=rtr>rurvr!rwrAr3rr|s) & + /$$=rArc |ddlm}t|tjrd}|j |j }}nd}d\}}t|tr|}|j } t|dd|k(r"t|ddr|tjk(r|S| sA|r?|s=|j|jk(r$t|tsJ|t|||S| ri|rg|re|jdkDrV|j|jk7r=|j|jk(r$t|tsJ|t|||S|rc|j rWt#j$dkDr@t'|j(||r)t+j,|g|jg|g|gdS| s|r|s|j|jk7r|j.D cgc]} | j0} } |j.dj2j5} |j.D cgc]} | j0} } |j.dj2j5}t7d | d | d | d |d |r |j r,t9|t:vst9|t<j>vrnt#j$tA|jjBk(r+|jE|t9|d t9|dtG|||d|St7d|dtG|||d|S|r|st7d|jI|.|tjk(r|StG||||jJ|StM|rn||j.dn|}|tjNk(r%t+jP|tS||g|gddStUjP|tS||g|gStS|tUjVn|}tG||||du|Scc} wcc} w)Nr)multihost_utilsTF)NNr _committedzFor a cross-host reshard in multi-controller JAX, input and target sharding should have the same set of devices. Got input's device set ids: z on platform z' and target sharding's device set ids: z. There is experimental support for cross-host transfers with different device sets, when input/output shardings have the same indices and layouts, in the TFRT TPU runtime only.zc passed to device_put is not the same on each process. Make sure you are passing the same value of z on each process.) fail_messagezjdevice_put's second argument must be a Device or a Sharding which represents addressable devices, but got zF. Please pass device or Sharding which represents addressable devices.zZdevice_put's first argument must be a fully addressable array, but got value with devices ),jax.experimentalr#rVr ArrayImplis_fully_addressablerr"r r\r] device_setr#r num_devicesrr$r process_countrrr[+batched_copy_array_to_devices_with_shardingridrupperrrrr python_scalar_typeslenr  assert_equalrr_rr& ALWAYS_COPYbatched_device_putr$rget_default_device)rQrrhrr#x_is_jax_arrayx_is_fully_addressable x_shardingras_is_fully_addressabledinp_idsinp_plat target_ids target_platshs r3_device_put_sharding_implr@s> /5??#N)*)?)?JN)3&J!A33:t$)gau.M &22 2 h "5 --- = ),1, ) ,Q4 88>1==1#4 :#C#CC --- = ),1, ) ,Q4 881<!>?A144??003<<BBD$%$8$89qadd9 9**1-66<<>   xj9 *|= FA AB B !,, q'[ DGv/I/I$I  # # %Q-D-D-T-T)U U  & &!WI1gY/1 ' 3 !AtT488  667S9< < == Q4t 44 !  $$%IIK= 2 33~ #// / Jakk4HH ": .39>z,,Q/vf #// /$$T+?+G!&,XtT; ;  $ $T+?+G!&,X//%~!33539;" 1b$d(:D AAu@9s P4P9c |" tj|}t||}t |tj rtt||f|f|fdSt |tr1|}|j}t|dr|jjnd}|$|jt|||j|St |jtrt |t t dfst#d|d|j%t'|dd|k(r"t'|d d r|t(j*k(r|S||t|||j|St-j.t0||t(j2k(rdnd |St||||S#t$r!}td|dt |d|d}~wwxYw) Nrrz is not a valid JAX typer_srcsrrrzIsharding and layout in `Layout` instance should be concrete. Got layout: z for input r$Fr)r abstractifyupdate_dp_avalrrrV MemorySpacer4rrlayouthasattrrrr@r"rr str_shortr r\r]rr@rr) rQrhsrcrrerrldllx_dlls r3_device_put_implrOs \P   a d D& )d (() <VISF+/' 334 66A ((C&q(3AHHOOE {qzz) &q$ D AA qzz8 , sVT$Z0 1  $$%3k$..2B1C E FF 8T"a'GA|U,K "... h } &q$ D AA 377!%7%D%DD$    #1dFD 99= P  qcDG9,D E GLOPPs!F)) G2GGc `g}ggggf\}}}} tt|||||D]\} \} } } }}t| | | ||}t|trb|j | |j |j |j |j| j |j|j ||retj|dgt|z| |}t||D]1\} }t|| tsJ|| j||| <3|S)N)rhrJrr) enumerateziprOrVrrrQrarrrZr1r!)r_rCr dst_avalsxsys dsa_indicesdsa_xs dsa_shardingsdsa_copy_semanticsrrQrhrJcpryshard_arg_resultsr s r3_batched_device_put_implr]$s=  ";=r2r>8+v}&8'0 "gt^Y7(9#a #!VS"d6s$GA!&' mmACC133 0 01IIaL  vF 7K(:FD";0AB5  10 11 1e""#34be5 )rAc 8t||||dgt|zdS)N)r_rCrrS)r]r1)r_rCrrTs r3batched_device_put_implr_Bs( " 7nW% ''rArXctd|dDr)td|Drtd|dDr|Sy)Nc3$K|]}|du ywrCrwrrQs r3rz+_device_put_folding_rule..Ts /!t) /sr_c3PK|]}t|tj ywrC)rVr TypedNdArrayrbs r3rz+_device_put_folding_rule..Us CqjH112 Cs$&c3BK|]}|tjk(ywrC)r\r]rbs r3rz+_device_put_folding_rule..Vs Taa%111 Tsr)all)constsr/ out_avalss r3_device_put_folding_ruleriQsA /VI. // CF C C T6BR;S T T M rAc t|tjs|St|trt|trK|j |j j |jj|jn|j d}|j/|j tj|j}|St|tjr|j |S|S)N)rr)r) memory_space) rVr ShapedArrayr"r#updaterr abstract_meshrrmem_kind_to_spacerF)rr:s r3rErE]s D$** + K8!]+ KK!5!5166;O;O;<66"6"CK D15d1K  }} [[d&<&|rtt|\} } }}}g}|D]} | tjk(r td| tjk(r |j tjT| tjk(sJ|j tjtj| ||t|d}t| |D] \}}|||< |S)Nztdonate=True is not allowed during tranposition of device_put. Please file an issue if you want this to be supported.rB)r1rQrRrrZerorlistr\rrr]r3rr9r)ctsr_rCr_resultsdp_argsrctrhrJrZindicesr.new_copy_semanticsrUr[s r3_device_put_transposer|os^ FSX ' '"+CWdN,S"T/a "fc2 Bxrww nnaVS"-./ 37W 3F0GT7D. B !.. . FG G #// /!!"4"@"@A'33333!!"4"@"@A B   D$W*/0B*C EBGR 1gaj .rAc |Dcgc]}|tjus|c}rtfdddDsJ|tj|i||fScc}w)Nc3.K|] }d|k(yw)rNrw)rbdmapped_batch_dimss r3rz&_device_put_batcher..s!&%'b &sr%)r not_mappedrfrr9) batched_args batch_dimsr/rrs @r3_device_put_batcherrst$.Pb"H.lowers FH %&*<*<*H64++, FH %  ( ( . .((1h%%dii02!((1h))$))4==?A! (8(89((0?E?Q?Q  $ $Q( ;a h HrA)module_contextall_default_mem_kindrtmapavals_in avals_out)rr_rCrrTrs` r3_tpu_gpu_device_put_loweringrs@ ,, I " c%WcllCMMB CCrAr)rgpuc|SrCrw)rr_rCrrTs r3_common_device_put_loweringrs )rA)r-zcore.Primitive)rrrrrpr)r core.Jaxprrrrpr)rrrpr)rrrpz%Sequence[tuple[Sharding, SourceInfo]])rr)rpr)r<rrzSequence[basearray.Array]rpNone)r<rrarray.ArrayImplrpr)r<rrznp.dtyperzbasearray.Arrayrpr)rQrrr#rr\rpr)rQrrzcore.ShapedArrayrhzDevice | Sharding | Nonerr\)rh!Device | Sharding | Format | NonerJrrr\)r_+Sequence[Device | Sharding | Format | None]rCrrSequence[ArrayCopySemantics]rSz!Sequence[core.ShapedArray | None])r_rrCrrr) __future__ratexitcollections.abcr dataclasses functoolsrr threadingrtypingrrrrr r r r r rrrrjax._src.abstract_arraysrjax._src.interpretersrrrrrjax._src.api_utilrjax._src.layoutrr jax._src.librr[ jax._src.meshrrjax._src.monitoringrrr jax._src.partition_specr!jax._src.shardingr"jax._src.sharding_implsr#r$r%r&jax._src.stagesr'numpyrSJAXPR_TRACE_EVENTJAXPR_TO_MLIR_MODULE_EVENTBACKEND_COMPILE_EVENTregister_exclusion__file___xlaxeClientBackendrrr\CompileOptionssafe_mapr unsafe_mapsafe_ziprR unsafe_zip getLoggerr=rrr4cacher)rErslocalrGrxrvregisterrzr|log_elapsed_timerrsetrweakref_lru_cacherrrrrrrrrr dataclassrr@rOr]r_ Primitivermultiple_resultsrDriconst_fold_rulesrErqdef_abstract_evalr| linear_jvpprimitive_jvpsprimitive_transposesrprimitive_batchersrregister_loweringrrwrAr3rs # $ #0$*&.&8*),aa1&'=ND!!!(+WW )) **""--Z--Z   8 $  " 0 9ioo9v#2"33%%B0>AU%':B  ?  @! <+  4 66)66>P66: T6 7 8d#==$=&iB iB iB %iB  iBX%:3%: *%:2D%:P 8  6 1  1  <' 8' 6'1 't~~l+ $  -./G l+ = 890#*"--"F,(= %@ -@L)D0,u>,u> |%@ArA