biUddlZddlmZddlmZmZddlZddlZ ddl m Z ddl m Z ddlmZdd lmZerddlZddlZe Zdaeeed <Gd d eed efZy)N)Mapping) TYPE_CHECKINGOptional)config) get_logger) map_nested)TensorFormatterDEVICE_MAPPINGceZdZdfd ZedeedffdZdZdZ dZ defd Z d e jdefd Zd e jdd fd Zd e jdefdZxZS) JaxFormatterc t|||ddl}ddlm}t ||rt d|dt|dt |tr|nt|jd|_ t|ja |jttjvrtj!d|jdttjd t|jdd t|jd|_ ||_y) N)featurestoken_per_repo_idr)Devicez Expected z to be a `str` not z, as `jaxlib.xla_extension.Device` is not serializable neither with `pickle` nor with `dill`. Instead you can surround the device with `str()` to get its string identifier that will be internally mapped to the actual `jaxlib.xla_extension.Device`.zDevice with string identifier z) not listed among the available devices: z), so falling back to the default device: .)super__init__jaxjaxlib.xla_clientr isinstance ValueErrortypestrdevicesdevicer _map_devices_to_strlistkeysloggerwarningjnp_array_kwargs)selfrrrr#rr __class__s \/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/formatting/jax_formatter.pyrzJaxFormatter.__init__'s (>OP, ff %F8#6tF|nE??  !+63 7fSqAQ=R   !!557N ;;d>#6#6#89 9 NN0 > !4!4!6789{s{{}Q/014  kckkmA./DK 0returnzjaxlib.xla_extension.Devicecfddl}|jDcic]}t||c}Scc}wNr)rrr)rrs r&rz JaxFormatter._map_devices_to_strBs*2=#++-@F V#@@@s.cddlddlm}ttr*r(t fdDr|j dSS)Nrc3K|]V}t|jxr:|jdjk(xr|jdjk(Xyw)rN)rArrayshapedtype).0xcolumnrs r& z,JaxFormatter._consolidate..MsUkl 1cii(fQWWq -GfAGGW]^_W`WfWfLffsAA)axis)r jax.numpynumpyrrallstack)r$r2jnprs ` @r& _consolidatezJaxFormatter._consolidateHsD fd #pvyyay00 r'cddl}ddlm}t|tt t dfr|St|tjtjfr>tj|jtjr|jSi}t|tjtjfrbtj|jtjr4|jj rd|j"i}nud|j$i}nft|tjtjfrTORCHCODEC_AVAILABLEtorchcodec.decodersr@rAr rdefault_devicerarrayr#) r$valuerr9 default_dtyper<r>r@rAs r& _tensorizezJaxFormatter._tensorizeSs ec5$t*5 6L  bjj9 :r}}U[[Z\ZfZf?g<<> ! ebii4 5"-- UWU_U_:`zz((!(#)) 4 !(#)) 4  2::6 7BMM%++WYWbWb  !!557N S  t{{ ; < R399UQ&P&P$:O:O&PQ R R Rs !J88Kcddl}tjrjdtjvrXddl}t ||jr>|j|jjjdSt|dr&t ||js|j}t |tj rA|j"t$k(rr|j'|Dcgc]}|j)|c}St |t*t,fr.|j'|Dcgc]}|j)|c}S|j|Scc}wcc}w)Nrtorch __array__)rrTORCH_AVAILABLErPrQr_rTensorr]detachcpur6hasattrr-rarCrEr/objectr:recursive_tensorizertuple)r$ data_structrr_ substructs r&_recursive_tensorizez!JaxFormatter._recursive_tensorizes  ! !g&< +u||4{'9'9';'?'?'A'G'G'I"'MNN ; ,Z SYY5W%//1K k2:: .  F*((_j)kR[$*B*B9*M)kll  dE] 3$$[f%gid&>&>y&I%gh h{++*l%gs -E!1E&rjc2t|j|dS)NF)map_list)r rl)r$rjs r&rhz JaxFormatter.recursive_tensorizes$33[5QQr'pa_tablec|jj|}|jj|}|j |SN)numpy_arrow_extractor extract_rowpython_features_decoder decode_rowrh)r$rorows r& format_rowzJaxFormatter.format_rowsB((*66x@**55c:'',,r' jax.Arrayc|jj|}|jj||jd}|j |}|j |}|Sr*)rrextract_columnrt decode_column column_namesrhr:)r$ror2s r& format_columnzJaxFormatter.format_columnsd++-<rs\  #*&'' !%%E?7K#@AEr'