biznddlZddlZddlZddlZddlZddlZddlZddlZddlZ ddl Z ddl Z ddl Z ddl Z ddlmZddlmZmZddlmZddlmZddlmZddlmZddlmZmZdd lmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$ddl%Z&ddl'Z(ddl)Z*ddl+Z,ddl-m.Z/dd l0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6dd l7m8Z8dd l9m:Z:m;Z;ddlm?Z?ddl@mAZAddlBmCZCmDZDmEZEddlFmGZGddlHmIZIddlJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSddlTmUZUmVZVmWZWmXZXmYZYmZZZddl[m\Z\m]Z]ddl^m_Z_ddl`maZambZbmcZcmdZdddlemfZfmgZgmhZhmiZiddljmkZlddlmmnZnddlompZpddlqmrZrmsZsmtZtmuZumvZvddlwmxZxmyZymzZzm{Z{dd l|m}Z}erddl~Z~ddlZddlZddlZeneZe$eefZd!Zd"ed#eeeffd$Zd"ed%ed&ed'eefd(Zdd)eeefd*e#eId+eIfd,Zd-eeee fd+eeeffd.Zd)eeefd+eeee ffd/Z dd0eeeefd1ed2ed+eeee,j$ffd3ZGd4d5ZGd6d7eZGd8d9eZGd:d;eZGd<d=eZGd>d?eZGd@dAeZGdBdCeZGdDdEeZGdFdGeZdHeefdIZGdJdKeZGdLdMeZd+e,j$fdNZGdOdPeZdQe$ee,j$fdRe$eee,jFe,jHe,jJfdSefdTZdUe"dQe$ee,j$fdSefdVZdUe"dQe$ee,j$fdSefdWZGdXdYeZGdZd[eZGd\d]eZGd^d_eZGd`daeZd"edbeIdceee$eedffd+efddZd)edbeIdceee$eedffd+efdeZeGdfdgZGdhdieZeGdjdkZeGdldmZdnZdoe$edpfd+e$edpffdqZGdrdsZGdtdueEZ ddveedwe#e\dxe#eadyed+ef dzZ dd{eed|e#eed}e#edwe#e\dxe#ead~erdd+efdZdededed+efdZdZy)N)Counter)IterableIterator)deepcopy) dataclass)partial)BytesIO)cycleislice)Path) TYPE_CHECKINGAnyBinaryIOCallableOptionalUnion) CommitInfoCommitOperationAddCommitOperationDelete DatasetCardDatasetCardDataHfApi)RepoFile)HfHubHTTPErrorRepositoryNotFoundError)Pool) HTTPError)config):PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDEDDatasetDatasetInfoMixin)sanitize_patterns)Features) FeatureTypeListValue_align_features!_check_if_features_can_be_aligned%_fix_for_backward_compatible_features_visitcast_to_python_objectsrequire_decoding)ArrowFormatterPythonFormatterTableFormatterTensorFormatterget_format_type_from_alias get_formatter) DatasetInfoDatasetInfosDict) _split_re) NamedSplitSplit SplitDict SplitInfo)cast_table_to_featuresembed_table_storageread_schema_from_file table_cast)tqdm) get_logger)MetadataConfigs)Literalasdictglob_pattern_to_regexiflatmap_unorderedstring_to_dict)_merge_gen_kwargs_number_of_shards_in_gen_kwargs_shuffle_gen_kwargs_split_gen_kwargs)PathLikec|SN)xs T/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/iterable_dataset.py identity_funcrQRs Hexamplecolumn_mappingc tfd|DrJtdt|dt|jdt |t z dtfd|jDrXtdt|dt|jdt t |jz d|j Dcic] \}}|| c}}Scc}}w)Nc3&K|]}|v ywrMrN.0colrSs rP z%_rename_columns_fn..Ws 8#3g  8zError when renaming z to z : columns z are not in the dataset.c3&K|]}|v ywrMrNrWs rPrZz%_rename_columns_fn..[s =c3'> =r[z are already in the dataset.)any ValueErrorlistvaluessetitems)rSrToriginal_column_namenew_column_names` rP_rename_columns_fnreVsI 8 88"4#7"8T.BWBWBY=Z<[[efijxfy|AH}IgIfJJb c   =^%:%:%< =="4#7"8T.BWBWBY=Z<[[efijqfruxzHzOzOzQvRgRfSSo p  6D5I5I5K  1 / !566  s-Didxnamecolumnc<||vrtd|d|d|||iS)NzError when adding z : column z is already in the dataset.)r^)rSrfrgrhs rP add_column_fnrjes4 w-dV9TFB]^__ &+ rRbatch try_featuresreturnc2tjj|}|* t|tj|j }tj|jS#t tjtjf$rYIwxYwrM) paTable from_pydictr>schematype TypeError ArrowInvalidArrowNotImplementedErrorr$from_arrow_schema)rkrlpa_tables rP_infer_features_from_batchryksxxx##E*H !(BIIl6G6G,HIH  % %hoo 662??B,G,GH   s)A++(BBexamplesc |Dcic] }|D]}|d}}}|Dcgc]!}|Dcgc]}|j|c}#}}}tt||Scc}}wcc}wcc}}wrM)getdictzip)rzrSrYcolsarrayss rP_examples_to_batchrusl&. A' A#CI AC AD ADH ISx8Gw{{38 IF I D&! "" B8 IsA A(A# A(#A(c#Kt|dk(rdnt|tt|}t|D]+}|j Dcic] \}}||| c}}-ycc}}ww)z3Convert a batch (dict of examples) to examples listrN)lennextiterrangerb)rk n_examplesirYarrays rP_batch_to_examplesr~sd%jAo3uT$u+5F/G+HJ : >/4{{}=esE!H}==>=sAA3A-$A3iterable batch_sizedrop_last_batchc #K||dkrBdtjjt|Dcgc]\}}| c}}dfyt |}|D]\}}t ||dz }||fgt |z}t||kr|ryt|\} } djd| D} | tjjt| dfycc}}ww) aConvert and group examples in Arrow tables of size `batch_size`. Args: iterable (`Iterable[Tuple[Key, dict]]`): An examples iterable containing tuples (example_key, example) of type (int/str, dict) batch_size (`Optional[int]`): Size of each sub-table to yield. If None or <= 0, yields the full table. drop_last_batch (`bool`, defaults to `False`): Drop the last batch if it is smaller than `batch_size`. NrallT)only_1d_for_numpyr_c32K|]}t|ywrMstrrXkeys rPrZz$_convert_to_arrow..s43s84) rorp from_pylistr,rr r_rr~join) rrrrrSiteratorriterator_batchkey_examples_listkeysrznew_keys rP_convert_to_arrowrsZ1_  HH !7S[8\ZQ8\pt!u v   H~H f W*q.9!7^,tN/CC  !J .? /0h((4t44rxx++,B8_c,deeef 9]s,C+ C%B1C+c XeZdZdZddZdeeeeffdZ e de e geeee jfffdZe defdZe de efdZd ej*j,ddfd Zdd ed eddfd Zdd ed edeefdZe defdZdefdZdedefdZdefdZy)_BaseExamplesIterablez?Base class for the examples iterable used by an IterableDatasetrmNcd|_yrM _state_dictselfs rP__init__z_BaseExamplesIterable.__init__s 8<rRc0tt|d)zWAn examples iterable should yield tuples (example_key, example) of type (int/str, dict)z doesn't implement __iter__ yetNotImplementedErrorrsrs rP__iter__z_BaseExamplesIterable.__iter__s!T$ZL0O"PQQrRcyrMrNrs rP iter_arrowz _BaseExamplesIterable.iter_arrowrRcyNFrNrs rPis_typedz_BaseExamplesIterable.is_typedsrRcyrMrNrs rPfeaturesz_BaseExamplesIterable.featuresrrR generatorc0tt|d)z Either shuffle the shards/sources of the dataset, or propagate the shuffling to the underlying iterable. If the order of the shards must stay fixed (when using .skip or .take for example), then this method returns self. z+ doesn't implement shuffle_data_sources yetrrrs rPshuffle_data_sourcesz*_BaseExamplesIterable.shuffle_data_sourcess "T$ZL0["\]]rR num_shardsindexc0tt|d)ZEither keep only the requested shard, or propagate the request to the underlying iterable.z) doesn't implement shard_data_sources yetrrrr contiguouss rPshard_data_sourcesz(_BaseExamplesIterable.shard_data_sourcess!T$ZL0Y"Z[[rRc|rT|j|z}|j|z}||zt||z}||z||krdndz}tt||Stt||j|S)Nrr)rminr_r)rrrrdivmodstartends rPsplit_shard_indices_by_workerz3_BaseExamplesIterable.split_shard_indices_by_workersu //Z/C//J.C%K#eS/1E#+eckq9CeS)* *eT__jAB BrRc0tt|d)Nz! doesn't implement num_shards yetrrs rPrz _BaseExamplesIterable.num_shardss!T$ZL0Q"RSSrRc0tt|d)Nz' doesn't implement _init_state_dict yetrrs rP_init_state_dictz&_BaseExamplesIterable._init_state_dicts!T$ZL0W"XYYrR state_dictc4fd|j|S)Nc|+t|tr|D]}||||||<|S|=t|tr-tt |D]}||||||<|S|SrM) isinstancer}r_rr)state new_staterr_inner_load_state_dicts rPrzE_BaseExamplesIterable.load_state_dict.._inner_load_state_dicts$E4)@$TC!7c IcN!SE#JT &:eT+Bs5z*NA5eAh ! ME!HN  rRr)rrrs @rPload_state_dictz%_BaseExamplesIterable.load_state_dicts &d&6&6 CCrRcn|jrtj|jStd)NzPState dict is not initialized, please call ex_iterable._init_state_dict() first.)rcopyr RuntimeErrorrs rPrz _BaseExamplesIterable.state_dicts,   ==!1!12 2mnnrR)rmNT) __name__ __module__ __qualname____doc__rrtupleKeyr}rpropertyrrrorprboolrr$rnprandom Generatorrintrr_rrrrrrNrRrPrrsRI=R(5d#34RHXb(5bhh;O2P.P%QR$(8,^bii.A.A^F]^\S\\Ri\CCCC]abe]fCTCTTZ$Z D$ D4 DoDorRrceZdZdedeeeffdeffd ZdefdZdZ de jjddfd Z dd ed eddfd Zedefd ZxZS)ExamplesIterablegenerate_examples_fn.kwargsc>t|||_||_yrM)superrrr)rrr __class__s rPrzExamplesIterable.__init__s $8! rRrmcXdd|jjd|_|jSNr) shard_idxshard_example_idxrsrrrrs rPrz!ExamplesIterable._init_state_dict')*DNNLcLcdrRc#K|jr|jdnd}tt|j|j|dD]}|jr|jdnd}t|j di||dD])}|jr|jdxxdz cc<|+|jsv|jdxxdz cc<d|jd<ywNrr max_num_jobsrrrN)rr rJrrr)rshard_idx_start gen_kwagsshard_example_idx_start key_examples rPrzExamplesIterable.__iter__s;?;K;K$**;7QR 1$++DOO \^most :IOSO_O_d&6&67J&Kef #%&?d&?&?&L)&LNegkl " ##$$%89Q>9!! "  -2-89  !45 :s B?C+)C+rcDt|j|j|SrM)#ShuffledDataSourcesExamplesIterablerrrs rPrz%ExamplesIterable.shuffle_data_sourcess243L3Ldkk[deerRrrct|j|j}|j|||}t |Dcgc]}|| c}}t |j |Scc}wKeep only the requested shard.rr)rJrrrrGrrrrrrgen_kwargs_list shard_indicesrrequested_gen_kwargss rPrz#ExamplesIterable.shard_data_sourcessc+DKKdooV:::uYc:d 0m1\/!2D1\] 9 9;OPP2] A(c,t|jSrMrHrrs rPrzExamplesIterable.num_shards .t{{;;rRr)rrrrrrr}rrrrrrrrrrr __classcell__rs@rPrrsXc5d;K6K-LVZ  $  :fbii.A.AfFXfQSQQRdQ9!! "  -2-89  !45 :s C D #)D rrrct|j}t||j}t |j |j |||Srr)rrrIrrrrrrrrr r s rPrz6ShuffledDataSourcesExamplesIterable.shard_data_sources,sOt~~&&9#t{{&K# 9 9;VWjj *k  rRr)rrrrrrr}rrrrrrrrrrs@rPrrsk-$,S%T 2B-B$C-MQ-^`^g^g^q^q-  $ :" S  Rd rRrceZdZdedeeejffdeffd Z e dZ defdZ dZ d Zd ej j"ddfd Zdd ed eddfdZe defdZxZS)ArrowExamplesIterablegenerate_tables_fn.rc>t|||_||_yrM)rrrr)rrrrs rPrzArrowExamplesIterable.__init__6s "4 rRc|jSrM _iter_arrowrs rPrz ArrowExamplesIterable.iter_arrow;rRrmcXdd|jjd|_|jSrrrs rPrz&ArrowExamplesIterable._init_state_dict?rrRc#Kt}|jr|jdnd}tt|j|j |dD]}|jr|jdnd}d}|j di|D]\}}|t|z|kr|t|z }&|jtjD]V}|j|} t| D]5} ||k\r)|jr|jdxxdz cc<|| f|dz }7X|js|jdxxdz cc<d|jd< yw)Nrrrr max_chunksizerrN) r/rr rJrrrr to_readerr'ARROW_READER_BATCH_SIZE_IN_DATASET_ITER format_batchr) r formatterrrrrrrx pa_subtableformatted_batchrSs rPrzArrowExamplesIterable.__iter__Cst#% ;?;K;K$**;7QR 1$++DOO \^most :IOSO_O_d&6&67J&Kef # ! !8!8!8!E9!E / X$s8}48OO%X6%#+#5#5FDrDr#5#s/K&/&<&<[&IO#5o#F/,0GG#// $ 0 01D E J E"%w,.)Q.) // /  -2-89  !45# :s D6E#9*E#c# K|jr|jdnd}tt|j|j|dD]}|jr|jdnd}d}|j di|D]K\}}|t |z }||kr|jr |jdxxt |z cc<||fM|js|jdxxdz cc<d|jd<ywr)rr rJrrrr)rrrrrrrxs rPrz!ArrowExamplesIterable._iter_arrowYs ;?;K;K$**;7QR 1$++DOO \^most :IOSO_O_d&6&67J&Kef # ! !8!8!8!E9!E $ X!S]2!$(??##$$%89S]J98m#  $  -2-89  !45 :s CD)DrcDt|j|j|SrM)(ShuffledDataSourcesArrowExamplesIterablerrrs rPrz*ArrowExamplesIterable.shuffle_data_sourcesis78O8OQUQ\Q\^ghhrRrrct|j|j}|j|||}t |Dcgc]}|| c}}t |j |Scc}wr)rJrrrrGrrrs rPrz(ArrowExamplesIterable.shard_data_sourceslsc+DKKdooV:::uYc:d 0m1\/!2D1\]$T%<%<>RSS2]rc,t|jSrMrrs rPrz ArrowExamplesIterable.num_shardssrrRr)rrrrrrrorpr}rrrrrrrrrrrrrrrs@rPrr5s8CsBHH}9M4M+NX\    $ :,: ibii.A.AiF]iTSTTRiTYZmm *n  rRr)rrrrrrrorpr}rrrrrrrrrrrs@rPr%r%xsv-$S%RXX *>%>?--99&& - $ :6:( S  Ri rRr%c eZdZddedeedeffd ZedZ edZ edZ de fd Z d Zdeeeej&ffd Zd ej,j.ddfd ZddededdfdZedefdZxZS)RebatchedArrowExamplesIterable ex_iterablerrcLt|||_||_||_yrM)rrr0rr)rr0rrrs rPrz'RebatchedArrowExamplesIterable.__init__s% &$.rRc|jSrMrrs rPrz)RebatchedArrowExamplesIterable.iter_arrowrrRc.|jjSrMr0rrs rPrz'RebatchedArrowExamplesIterable.is_typed(((rRc.|jjSrMr0rrs rPrz'RebatchedArrowExamplesIterable.featuresr5rRrmc|jjdddd|jjd|_|jS)Nr)examples_iterableprevious_state batch_idxnum_chunks_since_previous_statecropped_chunk_lengthrsr0rrrrrs rPrz/RebatchedArrowExamplesIterable._init_state_dictsE!%!1!1!B!B!D"/0$%NN++  rRc#8K|jEd{y7wrM)r0rs rPrz'RebatchedArrowExamplesIterable.__iter__s####s c# K|jr7|jdr(|jj|jd|jjr|jj}nt |jd}|j |j dkri|jr|jddkDryt j|Dcgc]\}}| c}}}|jrd|jd<d|fyg}g}d}|jr|jdnd}|jr|jd nd} |jr)|jj} | |jd<|D]6\} }t|j|j D]\} } |dkDr|dz}|dk(r | dk(r|dz}"|dk(r'| dkDr"| j| t| | z } d}d} t| dk(r]|t| z|j kr1|j| |j| |t| z }|t| z|j k(r|j| |j| d jd |D}|jrF|jdxxdz cc<|jdxxt|z cc<d|jd <|t jj!|fg}g}d}|js |jd<| dz|jd<|j |z }|j| d |d|j| jd|d jd|D}|jrF|jdxxdz cc<|jdxxt|z cc<||jd <|t jj!|f| d|dg}| j|t| |z g}t| |z }|js |jd<| |jd<|js|jj} 9|j"s|rd jd|D}|jrD |jd<|jdxxdz cc<d|jd<d|jd <|t jj!|fyyycc}}ww)z-Iterate over sub-tables of size `batch_size`.r:rrNrr;rr<r=rrc32K|]}t|ywrMrrX_keys rPrZz=RebatchedArrowExamplesIterable._iter_arrow..&ITs4y&Irz[:]c32K|]}t|ywrMrrCs rPrZz=RebatchedArrowExamplesIterable._iter_arrow..rEr[z:]c32K|]}t|ywrMrrCs rPrZz=RebatchedArrowExamplesIterable._iter_arrow..-sATs4yAr)rr0rrrrro concat_tablesr enumeraterslicerappendrrp from_batchesr)rrrrx all_pa_table keys_buffer chunks_bufferchunks_buffer_sizenum_chunks_to_skipchunk_length_to_cropr:rr<chunkrr=s rPrz*RebatchedArrowExamplesIterable._iter_arrowsA    0 01A B    , ,T-=-=>N-O P    & &''224H()9)9aHH ?? "doo&:D$4$4[$AA$E++,R+!XX,RSL01  -% %   TXTdTdT--.OPjkKOK[K[t//0FGab   !--88:N1?D  - .%4 ?MC:CHDVDVeietetDVDu:v1 n6/%)&!+&'1,1E1J&!+&'1,1E1I!KK(_`E)*&+,(u:?%E 2T__D&&s+!((/&#e*4&'#e*4G&&s+!((/!hh&I[&IIG''((5:5(()JKsS`OaaKCD(()?@!288#8#8#GGG"$K$&M)*&''=K(()9:NmpqNq(()JK+/??=O+O(&&#b1E0Fa'HI!((Q8L)MN!hh&I[&IIG''((5:5(()JKsS`OaaKCW(()?@!288#8#8#GGG&)U!,@+A#D"EK%*[[1Es5zThGh%i$jM),U6J)J&''=K(()9:Nm(()JKc1 nd!%!1!1!i4 ?j## hhA[AAG5C  !12  -2-FG  !BC;<  !7828800?? ?)6#E-Ss'CU% U (IU%t|||_||_yrM)rrr0r_)rr0r_rs rPrzSelectColumnsIterable.__init__Gs &(rRcH|jjr |jSyrM)r0rrrs rPrz SelectColumnsIterable.iter_arrowLs"    & &## # 'rRc.|jjSrMr4rs rPrzSelectColumnsIterable.is_typedQr5rRc.|jjSrMr7rs rPrzSelectColumnsIterable.featuresUr5rRrmcX|jj|_|jSrMr0rrrs rPrz&SelectColumnsIterable._init_state_dictY%++<<>rRc#K|jD]'\}}||jDcic]}||| c}f)ycc}wwrM)r0r_)rrfrowcs rPrzSelectColumnsIterable.__iter__]sF(( >HC4+<+<=a3q6 == = >=s "? :?c#K|jjD]3\}}t|dkDs||j|jf5ywNr)r0rrselectr_)rrfrxs rPrz!SelectColumnsIterable._iter_arrowasN!--88: >MC8}q 8??4+<+<=== >s /A"Arc`t|jj||jSrM)r^r0rr_rs rPrz*SelectColumnsIterable.shuffle_data_sourcesfs'$T%5%5%J%J9%UW[WhWhiirRrrcft|jj||||jSrX)r^r0rr_rs rPrz(SelectColumnsIterable.shard_data_sourcesis3$    / / Ej / Y[_[l[l  rRc.|jjSrMrZrs rPrz SelectColumnsIterable.num_shardsnr[rRr)rrrrr_rrrrrrr}rrrrrrorprrrrrrrrrrs@rPr^r^Fs)$9)c) $$)))) $ >>XeCM&:;> jbii.A.AjF]j S  Ri +C++rRr^ceZdZdededeffd ZedZedZde fdZ d Z d e jjddfd Zdd ed eddfdZedefdZxZS)StepExamplesIterabler0stepoffsetcLt|||_||_||_yrM)rrr0rrrs)rr0rrrsrs rPrzStepExamplesIterable.__init__ts$ &  rRc.|jjSrMr4rs rPrzStepExamplesIterable.is_typed{r5rRc.|jjSrMr7rs rPrzStepExamplesIterable.featuresr5rRrmcX|jj|_|jSrMrers rPrz%StepExamplesIterable._init_state_dictrfrRc#Kt|j} tt||j}t ||j kDr||j nyKwrM)rr0r_r rrrrs)r ex_iteratorrks rPrzStepExamplesIterable.__iter__sU4++,  TYY78E5zDKK'DKK(( sA"A$rcxt|jj||j|jS)Nrrrs)rqr0rrrrsrs rPrz)StepExamplesIterable.shuffle_data_sourcess2#    1 1) <499UYU`U`  rRrrc~t|jj||||j|jS)Nrr{)rqr0rrrrsrs rPrz'StepExamplesIterable.shard_data_sourcess8#    / / Ej / Y;;  rRc.|jjSrMrZrs rPrzStepExamplesIterable.num_shardsr[rRr)rrrrrrrrrr}rrrrrrrrrrs@rPrqrqss$9c)))) $  bii.A.A F\  S  Rh +C++rRrqceZdZ ddeededffd ZedZedZ dZ de fd Z d Z d ejj ddfd Zedefd Z ddededdfdZxZS)#CyclingMultiSourcesExamplesIterable ex_iterablesstopping_strategyfirst_exhausted all_exhaustedct|||_||_|dk(rtj |_ytj |_y)Nr)rrrrrrr]bool_strategy_func)rrrrs rPrz,CyclingMultiSourcesExamplesIterable.__init__sC (!2.?/-Q"&&XZX^X^rRc4|jdjSrkrrrs rPrz,CyclingMultiSourcesExamplesIterable.is_typed  #,,,rRc4|jdjSrkrrrs rPrz,CyclingMultiSourcesExamplesIterable.featuresrrRc #K|jr|jdnd}tttt |j |dzdD]#}|jr||jd<||}%ywNex_iterable_idxrr)rr r rrr)rrnext_ex_iterable_idxs rP_get_indices_iteratorz9CyclingMultiSourcesExamplesIterable._get_indices_iterators}AEAQAQ$**+<=WX$*5s4;L;L7M1N+OQ`cdQdfj$k 3 6J  !23! !2O  3sA:A<rmcd|jDcgc]}|jc}dgt|jzdgt|jz|jjd|_|j Scc}w)NrF)rrprevious_states is_exhaustedrs)rrrrrrrr0s rPrz4CyclingMultiSourcesExamplesIterable._init_state_dictsw OSO`O`a [99;a $vD,=,=(>>"Gc$*;*;&<<NN++   bsBc#zKdgt|jz}|jrdtt|jD]C}|jd||j|j |jd|E|jDcgc] }t |}}|j }|jr"tj|jdn(tjt|jd}|D]}|j|ry||t||d||<||}|jr+t|jd||jd|<t||d||<||durd||<|jrd|jd|<d||<|jr?|j|j|jd|<d|jd|<t |j|||<|dus|ycc}ww)NrrFrT)rrrrrrrrrfullrrrr)rnextsrr0 iteratorsindices_iteratorrresults rPrz,CyclingMultiSourcesExamplesIterable.__iter__sAT..//   3t0012 a##$56q9E%%a(889I9IJ[9\]^9_` a;?:K:KL;T+&L L557;?:J:JBHHT%%n5 6PRPWPWX[\`\m\mXnpuPv " A&&|4Qx ! e4a1XF9A$BRBRSaBbcdBe9f  !23A6IaL%0E!HQx5 "& Q##:>D$$^4Q7a##:>:K:KA:N:_:_:aD$$^4Q7=AD$$%67:#D$5$5a$89 ! U" 3 MsAH;>H;H6,FH;. H;rc|jDcgc]}|j|}}t||jScc}w)z*Shuffle each underlying examples iterable.)rrrrrrr0rs rPrz8CyclingMultiSourcesExamplesIterable.shuffle_data_sourcess?W[WhWhi 88Ci i2<AWAWXXjs?c:td|jDS)Nc34K|]}|jywrMrrXr0s rPrZzACyclingMultiSourcesExamplesIterable.num_shards..Ok;))Orrrs rPrz.CyclingMultiSourcesExamplesIterable.num_shardsOT=N=NOOOrRrrc t|jDcgc]}|j|||c}|jScc}w)rrr)rrrrrrrrrs rPrz6CyclingMultiSourcesExamplesIterable.shard_data_sourcessD3cgctct uW_X ( (Uz ( R u"44  usA)rr)rrrr_rrBrrrrrr}rrrrrrrrrrrs@rPrrsJ[ _01 _##EF _----3 $ (TYbii.A.AYFkY PCPP7;  &) . rRrceZdZdZdeeffd ZedZedZ edZ de fdZ d Z d Zd ej j"ddfd Zedefd Z ddededdfdZxZS)2VerticallyConcatenatedMultiSourcesExamplesIterablea VerticallyConcatenatedMultiSourcesExamplesIterable simply chains the input iterables. It doesn't require the examples iterables to always yield the same columns. Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`. For information, `IterableDataset` merges the features of all the datasets to concatenate into one. We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate. Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None. This is done with `_apply_feature_types_on_example`. rc0t|||_yrMrrrrrrs rPrz;VerticallyConcatenatedMultiSourcesExamplesIterable.__init__ (rRc4|jdjSrkrrs rPrz;VerticallyConcatenatedMultiSourcesExamplesIterable.is_typedrrRc4|jdjSrkrrs rPrz;VerticallyConcatenatedMultiSourcesExamplesIterable.featuresrrRcTtd|jDr |jSy)Nc38K|]}|jduywrM)rrs rPrZzPVerticallyConcatenatedMultiSourcesExamplesIterable.iter_arrow..#sWk{%%T1W)rrrrs rPrz=VerticallyConcatenatedMultiSourcesExamplesIterable.iter_arrow!s( WTEVEVW W## # XrRrmcd|jDcgc]}|jc}|jjd|_|jScc}w)Nr)rrrsrrrrrrs rPrzCVerticallyConcatenatedMultiSourcesExamplesIterable._init_state_dict&sM OSO`O`a [99;aNN++  bsAc#K|jr|jdnd}t|j|dD]0}|Ed{|js|jdxxdz cc<2y7+wr)rr rrex_iterable_idx_startr0s rPrz;VerticallyConcatenatedMultiSourcesExamplesIterable.__iter__.snGKGWGW 0 01B C]^!$"3"35JDQ 9K" " "  !23q83 9 "s=A-A+A-A-c#K|jr|jdnd}t|j|dD]>}|jEd{|js(|jdxxdz cc<@y7+wr)rr rrrs rPrz>VerticallyConcatenatedMultiSourcesExamplesIterable._iter_arrow5swGKGWGW 0 01B C]^!$"3"35JDQ 9K"--/ / /  !23q83 9 /sA A; A9A;A;rct|}t|j}|j||Dcgc]}|j |}}t |Scc}w)zTShuffle the list of examples iterable, as well as each underlying examples iterable.)rr_rshufflerr)rrr rr0s rPrzGVerticallyConcatenatedMultiSourcesExamplesIterable.shuffle_data_sources<sYy!D--.  L!Wcd 88Cd dA,OOesAc:td|jDS)Nc34K|]}|jywrMrrs rPrZzPVerticallyConcatenatedMultiSourcesExamplesIterable.num_shards..Hrrrrs rPrz=VerticallyConcatenatedMultiSourcesExamplesIterable.num_shardsFrrRrrc vt|jDcgc]}|j|||c}Scc}wrr)rrrrs rPrzEVerticallyConcatenatedMultiSourcesExamplesIterable.shard_data_sourcesJs<Bcgctct uW_X ( (Uz ( R u  u6r)rrrrr_rrrrrrr}rrrrrrrrrrrrs@rPrrs )T*?%@)----$$ $ 99P,,P =PPCPP7;  &) = rRrr_ct|}td|jDs'|Dcgc] }||dkDs |}}td|dycc}w)zBCheck the column names to make sure they don't contain duplicates.c3&K|] }|dk( yw)rNrN)rXcounts rPrZz&_check_column_names..Vs8euz8srzAThe examples iterables can't have duplicated columns but columns z are duplicated.N)rrr`r^)r_counterrYduplicated_columnss rP_check_column_namesrSsdl#G 8w~~'78 8-4Ic q8HcIIOPbOccs t   9Is AAceZdZdZdeeffd ZedZedZ de fdZ dZ d e jjddfd Zedefd Z dd ed eddfdZxZS)4HorizontallyConcatenatedMultiSourcesExamplesIterablea5 HorizontallyConcatenatedMultiSourcesExamplesIterable merges examples together for the input list of iterables. It also checks that there are no duplicate columns (otherwise we don't know which one to keep). This check is done once when yielding the first example. However it doesn't fill missing columns with None. Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`. For information, `IterableDataset` merges the features of all the datasets to concatenate into one. We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate. Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None. This is done with `_apply_feature_types_on_example`. rc0t|||_yrMrrs rPrz=HorizontallyConcatenatedMultiSourcesExamplesIterable.__init__mrrRc4|jdjSrkrrs rPrz=HorizontallyConcatenatedMultiSourcesExamplesIterable.is_typedrrrRc4|jdjSrkrrs rPrz=HorizontallyConcatenatedMultiSourcesExamplesIterable.featuresvrrRrmc|jDcgc]}|jc}|jjd|_|jScc}w)N)rrsrrs rPrzEHorizontallyConcatenatedMultiSourcesExamplesIterable._init_state_dictzsJOSO`O`a [99;aNN++ bsAc #(K|jDcgc] }t|}}tjD]}g}g}t |D]3} t |\}}|j ||j |5|r`|dk(r"t|D cgc] }|D]} |  c} }i} |D]}| j|djd|D} | | fyycc}w#t$r|j|YwxYwcc} }ww)Nrrc32K|]}t|ywrMrrs rPrZzPHorizontallyConcatenatedMultiSourcesExamplesIterable.__iter__..s"<3s8">#?#?#E#E*+OSO`O`a [99;a $vD,=,=(>>"Gc$*;*;&<<NN++   bsB!c|jDcgc]}|j|}}t|||j|jScc}w)z;Shuffle the data sources of each wrapped examples iterable.rrr)rrrrrrs rPrz@RandomlyCyclingMultiSourcesExamplesIterable.shuffle_data_sourcessQW[WhWhi 88Ci i: ,,"44   jsA rrc t|jDcgc]}|j|||c}|j|j|j Scc}wr)rrrrrrrs rPrz>RandomlyCyclingMultiSourcesExamplesIterable.shard_data_sourcessT;cgctct uW_X ( (Uz ( R u NN     " "   usA)Nrr)rrrr_rrrrrfloatrBrrrrrr}rrrrrrs@rPrrs 04IZ +01 +99&& + U , + ##EF +----!:  $   bii.A.A Fs 7;    &)  6  rRrct|tjr|St|tjtj frtjj |StjrHdtjvr6ddl }t||j|j fr|jS|S)Npolarsr) rrorppd DataFrameSeries from_pandasrPOLARS_AVAILABLEsysmodulesrto_arrow)outputpls rP_table_output_to_arrowrs&"((# &2<<34xx##F++ 8s{{#: fr||RYY7 8??$ $ MrRceZdZ ddedededeeededee dedeeed ee d ed d ee d ee ffd Z e dZe dZe dZde fdZdZdZddee deeeej0ffdZdej6j8ddfdZd de de ddfdZe de fdZxZ S)!MappedExamplesIterabler0function with_indices input_columnsbatchedrrremove_columns fn_kwargs formattingFormattingConfigr/max_num_running_async_map_functions_in_parallelc t |||_||_||_||_||_||_||_||_ | xsi|_ | |_ | |_ | xstj|_| r| j rt#|t$sSt'd| j(j+dt-|j.dt-|j.d|j |r|ndk7rQt'd| j(j+dt-|j.d|r|ndd|j d g|_y) NzThe z -formatted z" has underlying iterablethat is a z- instead of a RebatchedArrowExamplesIterable.rz has batch_size=z/ which isdifferent from ex_iterable.batch_size=z from its underlying iterable.)rrr0rrrrrrrrr _featuresr/MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLELris_tablerr/r^ format_type capitalizersr_owned_loops_and_tasks)rr0rrrrrrrrrrrrs rPrzMappedExamplesIterable.__init__ sv &   $.,(*"b$! ; uv?u?u < *--k+IJ :11<<>?{4PT:K^K^J_`!!%k!2!;!; <?{4PT:K^K^J__oFpzLMpNN>&1&<&<%>>\^ ce#rRcb|jr#|jjr |jSyyrM)rr rrs rPrz!MappedExamplesIterable.iter_arrow:s( ??t77## # 8?rRc|jduSrMrrs rPrzMappedExamplesIterable.is_typed?s}}D((rRc|jSrMr rs rPrzMappedExamplesIterable.featuresC ~~rRrmc|jjddd|jjd|_|jS)Nr)r9r:!num_examples_since_previous_stateprevious_state_example_idxrsr>rs rPrz'MappedExamplesIterable._init_state_dictGsB!%!1!1!B!B!D"12*+NN++  rRc#K|jrP|jjr:t}|jdD]\}}||j |fy|j Ed{y7w)Nrr)rr r/r format_row_iter)rr rrxs rPrzMappedExamplesIterable.__iter__Qsi ??t77')I!%!1!1!1!B : X9//999 :zz| # #sA0A:2A83A:c#R Kjrjdnd jrGjdr8jjjdjd}nd}tjjr>t jj }t|tr |jnd nd fd fd fdfdfd fd fd  gtjjr3 tjj"j%fnd fd } |}j&r d |D}|D]H\}}jr&jdjdxxdz cc<|dkDr|dz}C||fJy#t$rtj YwxYw#t(t*f$rrt,j/dt1dD]}|j3d j5tj6#tj8t:f$rt,j/dYwxYwwxYww)Nrrr:rc3(K D]\}} j jdkr nt jdz }||fgt|z}t|\}}dj d|D} j r5 j) jdkDrt | jkryt|} r |n|}tt |Dcgc]} |z }} t |z |||ffycc}ww)Nrrrc32K|]}t|ywrMrrs rPrZzLMappedExamplesIterable._iter..iter_batched_inputs..ws8Cs3x8r) rr r_r~rrrrr) rrSrrrrzrkrindices current_idx format_dictrrs rPiter_batched_inputsz9MappedExamplesIterable._iter..iter_batched_inputsks ( , W.$//Q2F$//A*=> '*7^$4tN7K$K!!$&7!8hhh8488((3!+H 7*84.9 E*u49#>O:P4QRq;?RRs7|+ U|++/ ,*SsC"D% D 1!Dc3XKD] \}}t|}dz dz ||ff"ywr)r})rrSr rs rP iter_inputsz1MappedExamplesIterable._iter..iter_inputssB ( 6 Ww-q !AoW~55  6s'*c <jr|rtt|}|Dcgc]"}t||t||k7s!|$}}|r>t d|d|Dcgc]}t||c}d|dt||d yyycc}wcc}w)Nz!Column lengths mismatch: columns z have length z while z has length .)rrrrr^)processed_inputs first_colrYbad_colsrs rPvalidate_function_outputz>MappedExamplesIterable._iter..validate_function_outputs|| 0 &6!78 #3s;KC;P7QUXYijsYtUu7uC$;H:]{CTDsvTWXhilXmTnTDSEE!!* <.prepare_inputsst&LC#'#5#5#=wi\`\n\nCoUXGCLCoG O  G:%']F7OT^^C C Dps A"c|jr(jD]}||vr||=||dus||vs||=i||}|Srr)rr.r'ritransformed_inputsrr*s rPprepare_outputsz5MappedExamplesIterable._iter..prepare_outputssn $%5 6"",,0AF{"1I';q>9aCS>S,Q/ 0 "@F!?.>!? % %rRcb||\}}}} jg||i|}|||S)z8Utility to apply the function on a selection of columns.r rrr.r,r-rr'r/r3rs rPapply_functionz4MappedExamplesIterable._iter..apply_functionsG:HV]:^ 7FG_i,t}}UgUU9U ";8HI IrRc~K||\}}}} jg||i|d{}|||S7w)zLUtility to apply the function on a selection of columns. Same code but asyncNr5r6s rPasync_apply_functionz:MappedExamplesIterable._iter..async_apply_functionsU:HV]:^ 7FG_i%2T]]%[G%[o%[QZ%[[ ";8HI I \s )=;=c 3Kjr n}tjjrjr:j j }|jd<d}jd}g}|D]\}}|j|jj ||tjk\rjtjtj\}}rjt|jk\rRjtjtj\}}rt|jk\rRtdjzk\rjdrdjr|j!dj!d} }|| j#fjr6| ur2jd<djd<jd<d\}}rdjrjsމsj j }d} }rA|djdf|j!dj!dfr@yyjrQjrEj j jd<djd< jd<|D]\}}jrjs jd<| ||fjsGjsTj j jd<djd< jd<yw) Nr:r) return_when rrNN)rinspectiscoroutinefunctionrrr0rrM create_taskrrrun_until_completeasynciowaitFIRST_COMPLETEDdonepopr)inputs_iteratorr:previous_state_taskrrrrrFpendingtaskr7r9r r"r$looprtaskss rP iter_outputsz2MappedExamplesIterable._iter..iter_outputsso7;||13O**4==9##%)%5%5%@%@%BN9GD$$%56*.'151A1AB^1_.=?&5ANA{NN1%LL!1!12F{TU2V!WX5zT%Y%YY(,(?(?#LLGTUD,,-PQMgD,,-IJBL?N,? E!HMMO'',?,GE)-)9)9)D)D)F.3Bi+5@27A8!!*d&=&=eAh&GGGKKNEIIaL0##||=A=M=M=X=X=Z(()9:PQ(()LMIT(()EF&5 YNA{''#||MXD,,-IJ^K;;;''<TUD,,-PQMXD,,-IJ Ys:E>O CO  O "O &O *A%O B'O 8 O AO c3JK|]\}}t|D]}||f ywrM)r)rXrtransformed_batchtransformed_examples rPrZz/MappedExamplesIterable._iter..s>../ABS/T,-..s!#rz Canceling z async tasks.KeyboardInterrupt)msgzTasks canceled.)rr0rrrr3r rr1recursive_tensorizer?r@rrCget_running_looprnew_event_looprrMr ExceptionrRloggerdebugrcancelrBgatherCancelledErrorr^)rnum_examples_to_skipr rNoutputsrrQrKr7r9r r!r"r$rrLr/r3rMr*s` @@@@@@@@@@@@rPrzMappedExamplesIterable._iterYsHLHXHXd&&'CD^_    0 01A B    , ,T-=-=>N-O P#'#3#34W#X #$ (() ??%doo&A&ABI;EiQ`;a)77gkKK ,6 6  D & J  J %'  & &t}} 5 0//1  ' ' . .e} =D8 Y8 Yt "nG||29 -4 /((##(8(89I(J(V$$%HIQNI'!+(A-(...  /Q  0--/ 0^,-  z#e*]CD!9DKK$7K894++GNNE,BC  .. ;4LL!23 4  shD!J'1G,J'2A)G?J'G<9J';G<<J'?A J$ !I.-J$..JJ$JJ$$J'rc #6K|jrt|jjn t}|jj r|jj }n:t |j|jr |jnd|j}|jrG|jdr8|jj|jd|jd}nd}|jr8|6|jj|jd<d|jd<|jr|jdnd}|D]\}}|jr2|j&t||jkr|jry|j|j|gn|jDcgc]}|| c}} |j rR|jr5| j#t%t|D cgc]} || z c} n| j#||j&| i|j(} t+| } t-| t.j0s2t3d|j4dt7| d |j4d |j8rJ|j8D];} | | j:vs| j=| j:j?| } =|B|t|z }|jr |jdxxt|z cc<|| ftA| jC| D]C\} }|dz }|jr|jdxxdz cc<|dkDr|dz}9|d | |fE|jsZ|jj|jd<d|jd<|jdxxt|z cc<ycc}wcc} ww) Nrrrr:rrrz(Provided `function` which is applied to z returns a variable of type z*. Make sure provided `function` returns a z to update the dataset.rr)"rr3r r.r0rrrrrrrrrrrrrMrrrrrrorprt table_typersrr_ remove_columnrrKr)rrr rr]r rrxrY function_argsrr output_tablerhr!s rPrz"MappedExamplesIterable._iter_arrowsRVRaRaM$//2M2M$Ngugw    & &''224H(  .2ll4?? $ 4 4H    0 01A B    , ,T-=-=>N-O P#'#3#34W#X #$    9151A1A1L1L1ND  - .DED  @ AHLHXHXd&&'CD^_ %3 TMC OO/MDOO3((%%-''12/3/A/ABhsmB    <<!((5XCW)Xa+/)XY!((5"T]]MDT^^DF1&9LlBHH5>y?S?S>TTpF|n$NyOcOcNdd{} """11kF!:!::'3'A'A,B[B[BaBabhBi'j k$s8}, ##$$%ABc(mSB<''&/ 0F0FUb0F0c&d4NA{1$K''(()LMQRRM+a/,1,  E1#, 334##9=9I9I9T9T9VD$$%56LMD$$%HI$$%ABc(mSBg3 TC*Ys-G P Pt|jj||j|j|j |j |j|j|j|j|j|j|j S)&Shuffle the wrapped examples iterable. rrrrrrrrrrr)rr0rrrrrrrrrrrrrs rPrz+MappedExamplesIterable.shuffle_data_sourcesdsz%    1 1) <]]**,,LL 00..nn]]<@,e/0,e8$,e:B#,e\$$)) $ $AFFT#FT(5QTVXV^V^Q^K_B`FTP bii.A.A F^ " S  Rj "+C++rRrinputmaskmask_column_namect|tjrkt|ttjtj fs*tj |gtj}|j||S||iS)N)rs) rrorpr_Array ChunkedArrayrbool_ append_column)rjrkrls rP _add_maskrrsb %"$rxx AB88TF4D""#3T:: $''rR mask_functionc4||g|i|}t|||SrMrrrsrjrlargsrrks rPadd_maskrxs&  0 0 0D UD"2 33rRcPK||g|i|d{}t|||S7wrMrurvs rPasync_add_maskrzs4u6t6v6 6D UD"2 33 7s &$&ceZdZdZ ddedededeee dedee dee d ed ffd Z fd Z dd ee ffd Zdee ddfdZdde de ddfdZede fdZxZS)FilteredExamplesIterablez ===MASK===r0rrrrrrrrc (||_|jr/ti|j|jt di} nd} t ||ttj|rtnt||j|||||||  y)Nr)rl) r0rrrrrrrr) rsrr$rrlr'rrrr?r@rzrx) rr0rrrrrrrrrs rPrz!FilteredExamplesIterable.__init__s&    ^;#7#7 ^9N9NPUV\P] ^_HH #")"="=h"GX!%!6!6 &'!!  rRc#Kt|D]2\}}t|}|j|js-||f4ywrM)rrr}rGrl)rrrSrs rPrzFilteredExamplesIterable._itersE!GMO #LC7mG{{40017l" #s ;A Arc#Kt||D]B\}}||j}||j|jj |fDyw)Nr)rrrldropfilter)rrrrxrkrs rPrz$FilteredExamplesIterable._iter_arrows^"W0}0M IMCD112Dx}}T%:%:;BB4HH H IsAAseedrmc t|jj||j|j|j |j |j|j|jS)rfrrrrrrr) r|r0rrsrrrrrr)rrs rPrz-FilteredExamplesIterable.shuffle_data_sourcessZ'    1 1$ 7''**,,LLnn  rRrrc t|jj||||j|j|j |j |j|j|jS)rrr) r|r0rrsrrrrrrrs rPrz+FilteredExamplesIterable.shard_data_sourcessa'    / / Ej / Y''**,,LLnn  rRc.|jjSrMrZrs rPrz#FilteredExamplesIterable.num_shardsr[rR)FNFrNNrMr)rrrrlrrrrr_rrr}rrrrrrrrrs@rPr|r|s# #-1$($(37 *     S *    SM D> /0 @# I#I  #  ;U    S    Rl  +C++rRr|c LeZdZdededej jffd Ze dZ e dZ de fdZ d e de ffd Zedd ej jdedeefd Zd Zdej jddfdZddededdfdZe defdZxZS)BufferShuffledExamplesIterabler0 buffer_sizercLt|||_||_||_yrM)rrr0rr)rr0rrrs rPrz'BufferShuffledExamplesIterable.__init__s% &&"rRc.|jjSrMr4rs rPrz'BufferShuffledExamplesIterable.is_typedr5rRc.|jjSrMr7rs rPrz'BufferShuffledExamplesIterable.featuresr5rRrmc|jj|_|j|_|jSrM)r0rrr_original_state_dictrs rPrz/BufferShuffledExamplesIterable._init_state_dicts4++<<>$(OO$5!rRrc|jr$||jk7rtjdt||S)NzLoading a state dict of a shuffle buffer of a dataset without the buffer content.The shuffle buffer will be refilled before starting to yield new examples.)rrrXwarningrr)rrrs rPrz.BufferShuffledExamplesIterable.load_state_dicts>   T666aw&z22rRr c#XK d|jd||DEd{$7w)Nc32K|]}t|ywrM)r)rXrs rPrZzFBufferShuffledExamplesIterable._iter_random_indices..s]1A]rrr)r)r rrs rP_iter_random_indicesz3BufferShuffledExamplesIterable._iter_random_indicess/] Q J[ (\] ] ] ]s *(*c#@K|j}t|j}|j||}g}|jD]9}t ||k(rt |}|||||<)|j|;|j||Ed{y7wrM) rrrrr0rrrMr)rrr r mem_bufferrOrs rPrz'BufferShuffledExamplesIterable.__iter__s&& t~~&44S+F !! %A:+-)* m# ! 1 !!!$  % JsBBBBcdt|jj||j|S)zFShuffle the wrapped examples iterable as well as the shuffling buffer.rr)rr0rrrs rPrz3BufferShuffledExamplesIterable.shuffle_data_sources%s.-    1 1) <$JZJZfo  rRrrc~t|jj||||j|jS)rrr)rr0rrrrs rPrz1BufferShuffledExamplesIterable.shard_data_sources+s:-    / / Ej / Y((nn  rRc.|jjSrMrZrs rPrz)BufferShuffledExamplesIterable.num_shards3r[rR)rr)rrrrrrrrrrrrr}rr staticmethodrrrrrrrrs@rPrrs #$9##XZXaXaXkXk#)))) $ 3$343^"))"5"5^C^dlmpdq^^" bii.A.A Ff  S  Rr +C++rRrc eZdZ ddedededeffd ZedZedZ de fd Z d Z e d Zd ej j"ddfd ZddededdfdZedefdZxZS)SkipExamplesIterabler0n"block_sources_order_when_shufflingsplit_when_shardingcZt|||_||_||_||_yrMrrr0rrrrr0rrrrs rPrzSkipExamplesIterable.__init__9/ &2T/#6 rRc.|jjSrMr4rs rPrzSkipExamplesIterable.is_typedGr5rRc.|jjSrMr7rs rPrzSkipExamplesIterable.featuresKr5rRrmcd|jj|jjd|_|jS)NF)skippedr9rsr>rs rPrz%SkipExamplesIterable._init_state_dictOs<!%!1!1!B!B!DNN++  rRc#K|jr|jdrdn |j}|jrd|jd<t|j|dEd{y7w)NrrT)rrr r0)rrs rPrzSkipExamplesIterable.__iter__Ws\%)%5%5$:J:J9:U[_[a[a   *.D  Y '$**,A4HHHsAA)!A'"A)c`||z}||z}|g|z}t|D]}||xxdz cc<|Srrnumrquotient remainderrrs rP split_numberz!SkipExamplesIterable.split_number]E!8!G ay! A 1INI  rRrc|jr|St|jj||j|j|j S)zeMay not shuffle the wrapped examples iterable since it would skip examples from other shards instead.rrr)rrr0rrrrs rPrz)SkipExamplesIterable.shuffle_data_sourcesfJ  2 2K'  55i@&&373Z3Z$($<$<  rRrrc|jr\t|jj||||j |j |||j |jS|Srrr)rrr0rrrrrs rPrz'SkipExamplesIterable.shard_data_sourcesrsf  # #'  33JR\3]##DFFJ7>373Z3Z$($<$<  KrRc.|jjSrMrZrs rPrzSkipExamplesIterable.num_shards~r[rRTTrrrrrrrrrrrr}rrrrrrrrrrrrs@rPrr8s 48$( 7* 7  7-1 7 " 7)))) $ I  bii.A.A F\  S  Rh +C++rRrceZdZdZdedeeffd ZdefdZ dZ de jjddfd Zdd ed eddfd Zedefd ZxZS)RepeatExamplesIterablezP Iterable that repeats the underlying iterable a given number of times. r0 num_timesc>t|||_||_yrM)rrr0r)rr0rrs rPrzRepeatExamplesIterable.__init__s &"rRrmcd|jj|jjd|_|jS)Nr) repeat_indexr9rsr>rs rPrz'RepeatExamplesIterable._init_state_dicts<!%!1!1!B!B!DNN++  rRc#NK|jr|jdnd} |j|t|jdk\ry|jEd{|dz }|jr6||jd<|jj |jd<7Lw)Nrrrr9)rrmaxr0r)rrs rPrzRepeatExamplesIterable.__iter__s;?;K;Kt''7QR ~~)lc$..RS>T.T'' ' ' A L3?  08<8H8H8Y8Y8[  !45 (sAB%B#A B%rcbt|jj||jS)z-Shuffle the underlying iterable, then repeat.r)rr0rrrs rPrz+RepeatExamplesIterable.shuffle_data_sourcess'%d&6&6&K&KI&VbfbpbpqqrRrrcht|jj||||jS)zShard, then repeat shards.rr)rr0rrrs rPrz)RepeatExamplesIterable.shard_data_sourcess1%    / / Ej / Ynn  rRc.|jjSrMrZrs rPrz!RepeatExamplesIterable.num_shardsr[rRr)rrrrrrrrr}rrrrrrrrrrrs@rPrrs#*#C=# $  \rbii.A.ArF^r S  Rj +C++rRrc eZdZ ddedededeffd ZedZedZ de fd Z d Z e d Zd ej j"ddfd ZddededdfdZedefdZxZS)TakeExamplesIterabler0rrrcZt|||_||_||_||_yrMrrs rPrzTakeExamplesIterable.__init__rrRc.|jjSrMr4rs rPrzTakeExamplesIterable.is_typedr5rRc.|jjSrMr7rs rPrzTakeExamplesIterable.featuresr5rRrmcd|jj|jjd|_|jS)Nr) num_takenr9rsr>rs rPrz%TakeExamplesIterable._init_state_dicts<!%!1!1!B!B!DNN++  rRc#K|jr|jdnd}t|j|j|z D])}|jr|jdxxdz cc<|+yw)Nrrr)rr r0r)rex_iterable_num_takenrs rPrzTakeExamplesIterable.__iter__smAEAQAQ 0 0 =WX!$"2"2DFF=R4RS K  -2-  sA.A0c`||z}||z}|g|z}t|D]}||xxdz cc<|Srrrs rPrz!TakeExamplesIterable.split_numberrrRrc|jr|St|jj||j|j|j S)zeMay not shuffle the wrapped examples iterable since it would take examples from other shards instead.r)rrr0rrrrs rPrz)TakeExamplesIterable.shuffle_data_sourcesrrRrrcd|jr\t|jj||||j |j |||j |jSt|jj||||j |j |jSr)rrr0rrrrrs rPrz'TakeExamplesIterable.shard_data_sourcess  # #'  33JR\3]##DFFJ7>373Z3Z$($<$<  (  33JR\3]&&373Z3Z$($<$<  rRc.|jjSrMrZrs rPrzTakeExamplesIterable.num_shardsr[rRrrrrs@rPrrs 48$( 7* 7  7-1 7 " 7)))) $  bii.A.A F\ SRh"+C++rRrrtoken_per_repo_idct|}|D] }||vsd||<|j|}|j||}|SNr)r}encode_exampledecode_example)rSrrrencoded_exampledecoded_examples rP_apply_feature_types_on_examplers\7mG( g %#'GK (--g6O--oQb-cO rRct|}t|tt|}|D]}||vsdg|z||<|j |}|j ||}|Sr)r}rrr encode_batch decode_batch)rkrrrr encoded_batch decoded_batchs rP_apply_feature_types_on_batchrs{ KEU4U ,-.J5 e #"&*!4E+ 5))%0M))-K\)]M rRcJeZdZUeeed<edefdZedefdZ y)rr rmcHtt|jtSrM)rr3r r0rs rPr zFormattingConfig.is_table(s-(8(89>JJrRcHtt|jtSrM)rr3r r1rs rP is_tensorzFormattingConfig.is_tensor,s-(8(89?KKrRN) rrrrr__annotations__rrr rrNrRrPrr$sF# K$KKL4LLrRrc (eZdZdedeedeedeee ee dffffd Z e dZ e dZe d Zd efd Zd Zd eeeej,ffd Zdej2j4d dfdZddeded dfdZe d efdZxZS)FormattedExamplesIterabler0rrrNcZt|||_||_||_||_yrM)rrr0r rr)rr0rrrrs rPrz"FormattedExamplesIterable.__init__2s. &!$!2rRc|jjr/|jr|jjr |jSyyrM)r0rrr rrs rPrz$FormattedExamplesIterable.iter_arrow?s9    & &4??C[C[## #D\ &rRcN|jjxs|jduSrM)r0rr rs rPrz"FormattedExamplesIterable.is_typedDs"((FDNN$,FFrRc|jSrMrrs rPrz"FormattedExamplesIterable.featuresHrrRrmcX|jj|_|jSrMrers rPrz*FormattedExamplesIterable._init_state_dictLrfrRc#K|jr|jjr:t|jjs |j nd|j }nNt|jj|jjs |j nd|j }|jjr@|jD],\}}|j|}t|D]}||f .yt|tr |jnd}|jD]Y\}}|j r8|jjs"t#||j |j }|r||}||f[yw)N)rrr)rr r/r0rr rr3r rrrrrr1rTrr)rr rrxrkrSr!s rPrz"FormattedExamplesIterable.__iter__PsZ$//":":'/3/?/?/H/Hd"&"8"8I &++/3/?/?/H/Hd"&"8"8I    & &!%!1!1!3 ' X!..x81%8'Gw,&' 'i9--  !% 0 0 # W==)9)9)B)B=$BXBXG)'2G7l" #sFF c#&K|js"|jjEd{|jjD]\}}t|j}|jj }|jD]V}||vst jjt jt|dg}|j||}X|j|k7rt||j}||fy7wrM)rr0rrar_ arrow_schemaro NullArray from_buffersnullrrqrrr;)rrrxcolumnsrrrrYs rPrz%FormattedExamplesIterable._iter_arrowrs}}''335 5 5!--99; MC(//0G]]//F#}} H g-,,33BGGIs8}tfUC'55k3GH H&(1(DMMJx-   6s*DDA"DBDrct|jj||j|j|j S)rfrrr)rr0rrrrrs rPrz.FormattedExamplesIterable.shuffle_data_sourcess:(    1 1) <]]"44   rRrrct|jj||||j|j|j S)rrr)rr0rrrrrs rPrz,FormattedExamplesIterable.shard_data_sourcessA(    / / Ej / Y]]"44   rRc.|jjSrMrZrs rPrz$FormattedExamplesIterable.num_shardsr[rRr) rrrrrrr$r}rrrrrrrrrrrrrrorprrrrrrrrrrs@rPrr1s 3* 3-. 38$ 3  U3d?%; ;< 3$$GG $  #D  XeCM&:;   bii.A.A Fa  S  Rm +C++rRrcTeZdZUejj ed<dZee ed<y)ShufflingConfigrN_original_seed) rrrrrrrrrrrNrRrPrrs yy"""$(NHSM(rRrc"eZdZUeed<eed<y)DistributedConfigrank world_sizeN)rrrrrrNrRrPrrs IOrRrctjrfddl}|jjj |j vr5|xj |jjj fz c_yyy)zNAdd torch.utils.data.IterableDataset as a parent class if 'torch' is availablerN)rTORCH_AVAILABLEtorch.utils.datautilsdataIterableDataset __bases__)clstorchs rP._maybe_add_torch_iterable_dataset_parent_classrsV  ;;   + +3== @ MMekk..>>@ @M ArRvaluez torch.TensorctjrJddl}t||jr|j S|j |j S|Srk)rrrrTensor share_memory_tensor)r rs rP*_maybe_share_with_torch_persistent_workersrsJ  eU\\ *&&( (5<<&446 6 rRcHeZdZdZdeddefdZdeefdZ deddfdZ y ) IterableColumnaQ An iterable for a specific column of an [`IterableDataset`]. Example: Iterate on the texts of the "text" column of a dataset: ```python for text in dataset["text"]: ... ``` It also works with nested columns: ```python for source in dataset["metadata"]["source"]: ... ``` source)rrrc ||_||_yrMrr)rrrs rPrzIterableColumn.__init__s &rRrmc#PK|jD]}||jywrMr)rrSs rPrzIterableColumn.__iter__s*{{ ,G$**+ + ,s$&ct||SrMrrrs rP __getitem__zIterableColumn.__getitem__dK00rRN) rrrrrrrrrrrrNrRrPrrsE('u%HI'X[',(3-,1s1/?1rRrceZdZdZ dzdedeedeedeedee dee d ee e e e edfffd Zed eefd Zed eee fd Zd e fdZde d dfdZdZdZdZd{dZed efdZdZed efdZed efdZdZdZ d|deded efdZ!dZ"d}dedefdZ#d e d e$fd!Z%e&dde'jPfd"e)d#ee*d$ee ded df d%Z+e& d~d&d'deed#ee*d dfd(Z,e&d)e d dfd*Z- dd+ee d dfd,Z. dd-ee)d.ed/ee e ee fd0edeeded1ee e ee fd#ee*d2ee d dfd3Z/ dd-ee)d/ee e ee fd0edeed2ee d df d4Z0 dd"ee1jdjfd5ed dfd6Z4d7efd8Z5d9ed dfd:Z6d;eed dfd<Z7d9ed dfd=Z8 dd>ed?ed@ed dfdAZ9dBe dCe ee1jtfd dfdDZ;dEe dFe d dfdGZdJe e ee fd dfdLZ?dCe dMe@d dfdNZAd#e*d dfdOZBddPedQed dfdRZCdSedTed dfdUZDdVZEd}deded dfdWZFddeed0ed e e eGe ffdXZHd efdYZI ddeed0ed e eJjeGeJjffdZZL ddeed0ed[ee d\ed e d]eGd]ff d^ZM d~d_e eNeOfdeed`ee d efdaZP d~d_e eNeOfdeed`ee d efdbZQ ddBe dce e dddedffdeed efdgZR d~d_e eNeOfdeed`ee d efdhZSdiedjedke dle de dmee dnee doeed>edped eTeUeeVeeffdqZWdke dle de dmee dnee doeed>eedpedreed eUeeVeeeffdsZX ddke dte dueedee dlee dvee dwee dxeedmee dnee doeed>eedpedreed eYfdyZZy)rz A Dataset backed by an iterable.Nr0infosplitr shuffling distributedrc|r(|jdkDr|r|j td||jn t }t j |||tj||_||_||_ ||_ |xsi|_ td|_ d|_|jt!|j"y)NrzThe dataset doesn't have a fixed random seed across nodes to shuffle and split the list of dataset shards by node. Please pass e.g. `seed=42` in `.shuffle()` to make all the nodes use the same seed. )rrr)rrrrr4r"r _ex_iterable _formatting _shuffling _distributed_token_per_repo_idr_epoch_starting_state_dict"_prepare_ex_iterable_for_iterationrr)rr0rrrrrrs rPrzIterableDataset.__init__s ;11A5) H`H`Hhg  #.tyy{KM!!$T? IIk2%#'EVE\Z\2\]^2_ 48! //16t~~FrRrmcH|jdSt|jS)afNumber of columns in the dataset. This can be None if the dataset has unknown features (e.g. after a map() operation). Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") >>> ds.num_columns 2 ``` N)rrrs rP num_columnszIterableDataset.num_columnss!}},tD#dmm2DDrRcH|jdSt|jS)aNames of the columns in the dataset. This can be None if the dataset has unknown features (e.g. after a map() operation). Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) >>> ds.column_names ['text', 'label'] ``` N)rr_rs rPr_zIterableDataset.column_names s!}},tE$t}}2EErRc@tj|jS)aGet the current state_dict of the dataset. It corresponds to the state at the latest example it yielded. Resuming returns exactly where the checkpoint was saved except in two cases: 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch. Returns: `dict` Example: ```py >>> from datasets import Dataset, concatenate_datasets >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3) >>> for idx, example in enumerate(ds): ... print(example) ... if idx == 2: ... state_dict = ds.state_dict() ... print("checkpoint") ... break >>> ds.load_state_dict(state_dict) >>> print(f"restart from checkpoint") >>> for example in ds: ... print(example) ``` which returns: ``` {'a': 0} {'a': 1} {'a': 2} checkpoint restart from checkpoint {'a': 3} {'a': 4} {'a': 5} ``` ```py >>> from torchdata.stateful_dataloader import StatefulDataLoader >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train") >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4) >>> # checkpoint >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood >>> # resume from checkpoint >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood ``` )rrrrs rPrzIterableDataset.state_dictsf}}T--..rRrc||_y)aLoad the state_dict of the dataset. The iteration will restart at the next example from when the state was saved. Resuming returns exactly where the checkpoint was saved except in two cases: 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch. Example: ```py >>> from datasets import Dataset, concatenate_datasets >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3) >>> for idx, example in enumerate(ds): ... print(example) ... if idx == 2: ... state_dict = ds.state_dict() ... print("checkpoint") ... break >>> ds.load_state_dict(state_dict) >>> print(f"restart from checkpoint") >>> for example in ds: ... print(example) ``` which returns: ``` {'a': 0} {'a': 1} {'a': 2} checkpoint restart from checkpoint {'a': 3} {'a': 4} {'a': 5} ``` ```py >>> from torchdata.stateful_dataloader import StatefulDataLoader >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train") >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4) >>> # checkpoint >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood >>> # resume from checkpoint >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood ``` N)r&)rrs rPrzIterableDataset.load_state_dictOs `%/!rRcd|jj-t|jjjndd|jdS)Nz IterableDataset({ features: Unknownz, num_shards: z }))_inforr_rrrs rP__repr__zIterableDataset.__repr__st3X\XbXbXkXkXwD9L9L9Q9Q9S4T~G4HH[\`\k\k[llqr rrRc|jSrM)__dict__rs rP __getstate__zIterableDataset.__getstate__s }}rRcp||_t|j|_t|jyrM)r2rr%rr)rds rP __setstate__zIterableDataset.__setstate__s% @M 6t~~FrRcJtt|j|S)NrA)rr)rrs rP_headzIterableDataset._headsDa0122rRc,t|jSrM)rr%rs rPepochzIterableDataset.epochs4;;rRc`|jr%|jdk(r|jjS|jrgt|jjj dd|jz }|dkrd|zn|}t j j|Std)NrlzThis dataset is not shuffled) r"r:rrrrr default_rngr^)reffective_seeds rP_effective_generatorz$IterableDataset._effective_generators ??tzzQ??,, , __%doo&?&?@II!WUX\XbXbbN;IA;Mg7SaN99((8 8;< }|j6r|j7}n tA|d}|D]\}} |jC| y|D] \}} |  tj%|d|j"dt'|d|jdytj%|d|j"d|jd|jdyw)NrzToo many dataloader workers:  (max is dataset.num_shards= ). Stopping z dataloader workers.zTo parallelize data loading, we give each process some shards (or data sources) to process. Therefore it's unnecessary to have a number of workers greater than dataset.num_shards=J. To enable more parallelism, please split the dataset in more files than r&znode# Frrrzdataloader worker#z, ': Starting to iterate over /z shards.r9r:r:r9rrrAz, ': Finished iterating over z9, ': Stopping... Number of dataset shards < num_workers (>A+=rRrrc |j}|jr"|js;|jjs%|jr'|j|jk7rt |||}|j r |j|j}n|}|jr|jj}|jj}|j|zdk(r\|jr7|j|z}|dkDrdnd}tjd|d|d|d |j!||d }n_|jrAtjd |d tjd|d|jd|t#|||}|js%|jrF|j|jk7r-t%||j|j|j&}|j)|j*d|_|j.r:|j*|j.dk(r|j1|j.d|S)Nr`rrsrGz Assigning z shardz (or data sourcez) of the dataset to each node.FrHzAssigning 1 out of zS examples of the dataset to each node. The others are skipped during the iteration.zIt is more optimized to distribute the dataset shards (or data sources) across nodes. You can do that by using a dataset with number of shards that is a factor of world_size=z. The current dataset has z which is not a factor of r{rrrrJr:r9)r r!rr rr/r"rr>r#rrrrPrXrrrqrr$rr:rr&r)rrrr0rrnum_shards_per_nodeplurals rPr'z2IterableDataset._prepare_ex_iterable_for_iterationsK''   ''4+;+;+D+D +"6"6$--"G8 OK ??%::4;T;T;VWK%K   $$))D**55J%% 2a7((**5*@*@J*N'$7!$;SFKK$%8$9xGWX^W__}~*<< Z^kp<q ((*KK-j\:MNKKss}r~33>3I3I2JJdeodpr 3;ZX\]    +2F2F$--2W3++"&"9"9 K"-!=!=!?ZZ   $ $t7P7PQX7Y)Y  ' '(A(ABU(V WrRc#Kdtjvrmddl}|jjj }t ||jjjr||jEd{y|j}|jr|js|jjrst|jj|j}|jr|j}n t!|d}|D]\}}|j#|y|D] \}}| y7ŭw)NrrrrrA)rrrrrrOrrrVr'r!rr r3r rrr) rrrSr0r rrrxrSs rPrzIterableDataset.__iter__( s ckk ! #++**:: ?#asD E E :Erct||SrMrrs rPrzIterableDataset.__getitem__d rrRrr gen_kwargscDddlm}||||d|jS)aCreate an Iterable Dataset from a generator. Args: generator (`Callable`): A generator function that `yields` examples. features (`Features`, *optional*): Dataset features. gen_kwargs(`dict`, *optional*): Keyword arguments to be passed to the `generator` callable. You can define a sharded iterable dataset by passing the list of shards in `gen_kwargs`. This can be used to improve shuffling and when iterating over the dataset with multiple workers. split ([`NamedSplit`], defaults to `Split.TRAIN`): Split name to be assigned to the dataset. Returns: `IterableDataset` Example: ```py >>> def gen(): ... yield {"text": "Good", "label": 0} ... yield {"text": "Bad", "label": 1} ... >>> ds = IterableDataset.from_generator(gen) ``` ```py >>> def gen(shards): ... for shard in shards: ... with open(shard) as f: ... for line in f: ... yield {"line": line} ... >>> shards = [f"data{i}.txt" for i in range(32)] >>> ds = IterableDataset.from_generator(gen, gen_kwargs={"shards": shards}) >>> ds = ds.shuffle(seed=42, buffer_size=10_000) # shuffles the shards order + uses a shuffle buffer >>> from torch.utils.data import DataLoader >>> dataloader = DataLoader(ds.with_format("torch"), num_workers=4) # give each worker a subset of 32/4=8 shards ``` r)GeneratorDatasetInputStreamT)rrr` streamingr) io.generatorrbread)rrr`rrbs rPfrom_generatorzIterableDataset.from_generatorg s)b >*(zUYaf $& rRdfzpyspark.sql.DataFramec ddlm}tjdk(r t d||f||dd|j S)aCreate an IterableDataset from Spark DataFrame. The dataset is streamed to the driver in batches. Args: df (`pyspark.sql.DataFrame`): The DataFrame containing the desired data. split (`NamedSplit`, *optional*): Split name to be assigned to the dataset. features (`Features`, *optional*): Dataset features. Returns: [`IterableDataset`] Example: ```py >>> df = spark.createDataFrame( >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]], >>> columns=["id", "name"], >>> ) >>> ds = IterableDataset.from_spark(df) ``` r)SparkDatasetReaderwin32z@IterableDataset.from_spark is not currently supported on WindowsT)rrrc)io.sparkrirplatformOSErrorre)rgrrrris rP from_sparkzIterableDataset.from_spark sP< 1 <<7 "\] ]!      $&  rRfilenamect|}tj|}ttj d|i}t |t|S)zInstantiate a IterableDataset from Arrow table at filename. Args: filename (`str`): File name of the dataset. Returns: [`IterableDataset`] ro)rr)r0r)r=r$rwrr! _generate_tables_from_cache_filerr4)ropa_table_schemainferred_featuresr0s rP from_filezIterableDataset.from_file sK09$66G+G,T,T^hjr]st ;[Rc=deerRrsc 0t|}t|j|jj |j t |t j|jt j|j|jS)at Return a dataset with the specified format. Args: type (`str`, *optional*): Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. `None` means it returns python objects (default). Example: ```py >>> from datasets import load_dataset >>> from transformers import AutoTokenizer >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) >>> ds = ds.with_format("torch") >>> next(iter(ds)) {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', 'label': tensor(1), 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} ``` r r0rrrrrr) r2rr r/r_splitrrr"r#r$)rrss rP with_formatzIterableDataset.with_format soX*$/))"++'D9mmDOO4 d&7&78"55  rRrrrrrrc t|tr|g}t|tr|g}|t}| i} | t|}|j} | j r;|j j#|j j| jk(rdn|j j} |jr_|jjrIt| tj|j| |j} t| |r|nd|} n|jr2|jjrt|j|r|nd|} |js| r6t| tj|j| |j} t!| |||||||| |j| } |j"j} || _t%| | |j&|jtj|j(tj|j*|jS)a Apply a function to all the examples in the iterable dataset (individually or in batches) and update them. If your function returns a column that already exists, then it overwrites it. The function is applied on-the-fly on the examples when iterating over the dataset. You can specify whether the function should be batched or not with the `batched` parameter: - If batched is `False`, then the function takes 1 example in and should return 1 example. An example is a dictionary, e.g. `{"text": "Hello there !"}`. - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. A batch is a dictionary, e.g. a batch of 1 example is {"text": ["Hello there !"]}. - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. Note that the last batch may have less than `n` examples. A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simulatenous calls. It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. Args: function (`Callable`, *optional*, defaults to `None`): Function applied on-the-fly on the examples when you iterate on the dataset. It must have one of the following signatures: - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True` - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` - `function(batch: Dict[str, List], indices: List[int]) -> Dict[str, List]` if `batched=True` and `with_indices=True` For advanced usage, the function can also return a `pyarrow.Table`. If the function is asynchronous, then `map` will run your function in parallel. Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. If no function is provided, default to identity function: `lambda x: x`. with_indices (`bool`, defaults to `False`): Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`. input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`): The columns to be passed into `function` as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. batched (`bool`, defaults to `False`): Provide batch of examples to `function`. batch_size (`int`, *optional*, defaults to `1000`): Number of examples per batch provided to `function` if `batched=True`. `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`. drop_last_batch (`bool`, defaults to `False`): Whether a last batch smaller than the batch_size should be dropped instead of being processed by the function. remove_columns (`[List[str]]`, *optional*, defaults to `None`): Remove a selection of columns while doing the mapping. Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding columns with names in `remove_columns`, these columns will be kept. features (`[Features]`, *optional*, defaults to `None`): Feature types of the resulting dataset. fn_kwargs (`Dict`, *optional*, default `None`): Keyword arguments to be passed to `function`. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> def add_prefix(example): ... example["text"] = "Review: " + example["text"] ... return example >>> ds = ds.map(add_prefix) >>> list(ds.take(3)) [{'label': 1, 'text': 'Review: the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, {'label': 1, 'text': 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth .'}, {'label': 1, 'text': 'Review: effective but too-tepid biopic'}] ``` NrZrr`) rrrrrrrrrrrw)rrrQr*r rr/rr!r rrrr$r/rrrrrxr"r#) rrrrrrrrrrr0input_featuresrs rPmapzIterableDataset.map sf mS )*OM nc *,-N  $H  I  7#}}T-=-=>+&*&=&=  - %'!+)''  yy~~  #++''mmDOO4 d&7&78"55  rRc 0t|tr|g}|j}|jjs |j rEt ||j |jrdn|jj|j}t||||||||j }t||j|j|j tj|jtj|j|jS)a Apply a filter function to all the elements so that the dataset only includes examples according to the filter function. The filtering is done on-the-fly when iterating over the dataset. If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simulatenous calls (configurable). It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. Args: function (`Callable`): Callable with one of the following signatures: - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False` - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False` - `function(example: Dict[str, List]) -> List[bool]` if `with_indices=False, batched=True` - `function(example: Dict[str, List], indices: List[int]) -> List[bool]` if `with_indices=True, batched=True` If the function is asynchronous, then `filter` will run your function in parallel. If no function is provided, defaults to an always True function: `lambda x: True`. with_indices (`bool`, defaults to `False`): Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`. input_columns (`str` or `List[str]`, *optional*): The columns to be passed into `function` as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. batched (`bool`, defaults to `False`): Provide batch of examples to `function`. batch_size (`int`, *optional*, default `1000`): Number of examples per batch provided to `function` if `batched=True`. fn_kwargs (`Dict`, *optional*, default `None`): Keyword arguments to be passed to `function`. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> ds = ds.filter(lambda x: x["label"] == 0) >>> list(ds.take(3)) [{'label': 0, 'movie_review': 'simplistic , silly and tedious .'}, {'label': 0, 'movie_review': "it's so laddish and juvenile , only teenage boys could possibly find it funny ."}, {'label': 0, 'movie_review': 'exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}] ``` NrZrrw)rrr r/rr!rrr$r|rrxrrr"r#)rrrrrrrr0s rPrzIterableDataset.filter sh mS )*OM'' ::  $"2"23++!,!5!54::;N;N"&"9"9 K/ %'!''  #++''mmDOO4 d&7&78"55  rRrc l| tjj|}n t|}t ||}t t |j|||jj|j|j|tj|j|jS)a^ Randomly shuffles the elements of this dataset. This dataset fills a buffer with `buffer_size` elements, then randomly samples elements from this buffer, replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or equal to the full size of the dataset is required. For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will initially select a random element from only the first 1000 elements in the buffer. Once an element is selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element, maintaining the 1000 element buffer. If the dataset is made of several shards, it also does shuffle the order of the shards. However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`] then the order of the shards is kept unchanged. Args: seed (`int`, *optional*, defaults to `None`): Random seed that will be used to shuffle the dataset. It is used to sample from the shuffle buffer and also to shuffle the data shards. generator (`numpy.random.Generator`, *optional*): Numpy random Generator to use to compute the permutation of the dataset rows. If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). buffer_size (`int`, defaults to `1000`): Size of the buffer. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> list(ds.take(3)) [{'label': 1, 'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, {'label': 1, 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth .'}, {'label': 1, 'text': 'effective but too-tepid biopic'}] >>> shuffled_ds = ds.shuffle(seed=42) >>> list(shuffled_ds.take(3)) [{'label': 1, 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."}, {'label': 1, 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'}, {'label': 1, 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}] ``` )rrrrw)rrr<rrrrr r/rrxr!r#r$)rrrrrs rPrzIterableDataset.shuffle sd   --d3I +I#iM 6!!{i"++'' d&7&78"55  rRr:cH|xj||jz z c_yrM)r%)rr:s rP set_epochzIterableDataset.set_epochD s ut{{** rRrc ht|j||jdu|jdu}t ||j j |j|jt j|jt j|j|jS)a Create a new [`IterableDataset`] that skips the first `n` elements. Args: n (`int`): Number of elements to skip. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> list(ds.take(3)) [{'label': 1, 'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, {'label': 1, 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth .'}, {'label': 1, 'text': 'effective but too-tepid biopic'}] >>> ds = ds.skip(1) >>> list(ds.take(3)) [{'label': 1, 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth .'}, {'label': 1, 'text': 'effective but too-tepid biopic'}, {'label': 1, 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}] ``` Nrrrw) rr r"r#rr/rrxr!rr$rrr0s rPskipzIterableDataset.skipG s8+    /3$/F $ 1 1T 9  #"++''mmDOO4 d&7&78"55  rRrc tt|j||j|j|j t j|jt j|j|jS)a~ Create a new [`IterableDataset`] that repeats the underlying dataset `num_times` times. N.B. The effect of calling shuffle after repeat depends significantly on buffer size. With buffer_size 1, duplicate data is never seen in the same iteration, even after shuffling: ds.repeat(n).shuffle(seed=42, buffer_size=1) is equivalent to ds.shuffle(seed=42, buffer_size=1).repeat(n), and only shuffles shard orders within each iteration. With buffer size >= (num samples in the dataset * num_times), we get full shuffling of the repeated data, i.e. we can observe duplicates in the same iteration. Args: num_times (`int`) or (`None`): Number of times to repeat the dataset. If `None`, the dataset will be repeated indefinitely. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") >>> ds = ds.take(2).repeat(2) >>> list(ds) [{'label': 1, 'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, {'label': 1, 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth .'}, {'label': 1, 'text': 'effective but too-tepid biopic'}, {'label': 1, 'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, {'label': 1, 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth .'}, {'label': 1, 'text': 'effective but too-tepid biopic'}] ``` rrw) rrr r/rxr!rrr"r#r$)rrs rPrepeatzIterableDataset.repeats sdB.t/@/@IV++''mmDOO4 d&7&78"55  rRc ht|j||jdu|jdu}t ||j j |j|jt j|jt j|j|jS)a Create a new [`IterableDataset`] with only the first `n` elements. Args: n (`int`): Number of elements to take. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> small_ds = ds.take(2) >>> list(small_ds) [{'label': 1, 'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, {'label': 1, 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson's expanded vision of j . r . r . tolkien's middle-earth .'}] ``` Nrrw) rr r"r#rr/rrxr!rr$rs rPtakezIterableDataset.take s*+    /3$/F $ 1 1T 9  #"++''mmDOO4 d&7&78"55  rRrrrc B|jj|||}t||jj |j |j t j|jt j|j|jS)a(Return the `index`-nth shard from dataset split into `num_shards` pieces. This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks, so it can be easily concatenated back together after processing. If `dataset.num_shards % n == l`, then the first `l` datasets each have `(dataset.num_shards // n) + 1` shards, and the remaining datasets have `(dataset.num_shards // n)` shards. `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original. In particular, `dataset.shard(dataset.num_shards, i)` returns a dataset with 1 shard. Note: n should be less or equal to the number of shards in the dataset `dataset.num_shards`. On the other hand, `dataset.shard(n, i, contiguous=False)` contains all the shards of the dataset whose index mod `n = i`. Be sure to shard before using any randomizing operator (such as `shuffle`). It is best if the shard operator is used early in the dataset pipeline. Args: num_shards (`int`): How many shards to split the dataset into. index (`int`): Which shard to select and return. contiguous: (`bool`, defaults to `True`): Whether to select contiguous blocks of indices for shards. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("amazon_polarity", split="train", streaming=True) >>> ds Dataset({ features: ['label', 'title', 'content'], num_shards: 4 }) >>> ds.shard(num_shards=2, index=0) Dataset({ features: ['label', 'title', 'content'], num_shards: 2 }) ``` rHrw) r rrr/rrxr!rr"r#r$)rrrrr0s rPshardzIterableDataset.shard s}\''::jX]jt:u #"++''mmDOO4 d&7&78"55  rRrgrhcH|jtt||dS)zAdd column to Dataset. Args: name (str): Column name. column (list or np.array): Column data to be added. Returns: `IterableDataset` )rgrhT)r)r|rrj)rrgrhs rP add_columnzIterableDataset.add_column s!xx DHW[x\\rRrcrdc(|j||iS)a Rename a column in the dataset, and move the features associated to the original column under the new column name. Args: original_column_name (`str`): Name of the column to rename. new_column_name (`str`): New name for the column. Returns: `IterableDataset`: A copy of the dataset with a renamed column. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> next(iter(ds)) {'label': 1, 'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} >>> ds = ds.rename_column("text", "movie_review") >>> next(iter(ds)) {'label': 1, 'movie_review': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} ``` )rename_columns)rrcrds rP rename_columnzIterableDataset.rename_column s8""$8/#JKKrRrTc |jjr$|jjjnd}|jt t |t |}|St|jDcic]\}}||jvr||n||!c}}|j_|Scc}}w)aa Rename several columns in the dataset, and move the features associated to the original columns under the new column names. Args: column_mapping (`Dict[str, str]`): A mapping of columns to rename to their new names Returns: `IterableDataset`: A copy of the dataset with renamed columns N)rTr1) r/rrr|rrer_r$rbr)rrToriginal_features ds_iterablerYfeatures rPrzIterableDataset.rename_columns& s;?**:M:MDJJ//446SWhh &~ FW[\jWk   ()1):(?(?(A$W,/.2E2E2G+GN3'SRYY*K   &  s$B? r_cL|jjr$|jjjnd}|j|}|S|j|j_|j D]!\}}||vs |jj|=#|S)a^ Remove one or several column(s) in the dataset and the features associated to them. The removal is done on-the-fly on the examples when iterating over the dataset. Args: column_names (`Union[str, List[str]]`): Name of the column(s) to remove. Returns: `IterableDataset`: A copy of the dataset object without the columns to remove. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> next(iter(ds)) {'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1} >>> ds = ds.remove_columns("label") >>> next(iter(ds)) {'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} ``` Nr1)r/rrr|rb)rr_rrrYrs rPrzIterableDataset.remove_columns? s2;?**:M:MDJJ//446SWhhlh;  ():)?)?)AK   &+113 8Q,&#))2237 8rRc t|tr|g}|jrtj|j}|jj t |t |jj jz }|rFtdt|dt|jj jdt|Dcic]}||j |c}|_t|j|}t||j|j|j |j"|j$Scc}w)aVSelect one or several column(s) in the dataset and the features associated to them. The selection is done on-the-fly on the examples when iterating over the dataset. Args: column_names (`Union[str, List[str]]`): Name of the column(s) to select. Returns: `IterableDataset`: A copy of the dataset object with selected columns. Example: ```py >>> from datasets import load_dataset >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> next(iter(ds)) {'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1} >>> ds = ds.select_columns("text") >>> next(iter(ds)) {'text': 'the rock is destined to be the 21st century's new " conan " and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} ``` z Column name z- not in the dataset. Columns in the dataset: r&rw)rrr/rrrrarr^r_r$r^r rrxr!r"r#r$)rr_rmissing_columnsrir0s rPselect_columnszIterableDataset.select_columnsb s'2 lC ((>L ::==,Dzz""."%l"3c$**:M:M:R:R:T6U"U"$&tO'<&=>< 3 3 8 8 :;>> from datasets import load_dataset, Audio >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train", streaming=True) >>> ds.features {'audio': Audio(sampling_rate=8000, mono=True, decode=True, id=None), 'english_transcription': Value('string'), 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']), 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']), 'path': Value('string'), 'transcription': Value('string')} >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000)) >>> ds.features {'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None), 'english_transcription': Value('string'), 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']), 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']), 'path': Value('string'), 'transcription': Value('string')} ``` rw r*r/rrrr rxr!rr"r#r$)rrhrrs rP cast_columnzIterableDataset.cast_column sD8@zz  ' f))++''mmDOO4 d&7&78"55  rRc Bt|}|jj}||_t |j ||j |jtj|jtj|j|jS)a Cast the dataset to a new set of features. Args: features ([`Features`]): New features to cast the dataset to. The name of the fields in the features must match the current column names. The type of the data must also be convertible from one type to the other. For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~Dataset.map`] to update the Dataset. Returns: `IterableDataset`: A copy of the dataset with casted features. Example: ```py >>> from datasets import load_dataset, ClassLabel, Value >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) >>> ds.features {'label': ClassLabel(names=['neg', 'pos']), 'text': Value('string')} >>> new_features = ds.features.copy() >>> new_features["label"] = ClassLabel(names=["bad", "good"]) >>> new_features["text"] = Value("large_string") >>> ds = ds.cast(new_features) >>> ds.features {'label': ClassLabel(names=['bad', 'good']), 'text': Value('large_string')} ``` rwrrrrs rPcastzIterableDataset.cast szD9Bzz   ))++''mmDOO4 d&7&78"55  rRenable num_threadsc|js td|}dtfd}|r|dkDr|jj}|jj}t |t |dt |t |d|j |}tjj|}t t||j}|j||}t|jtsJd|z|j_|S|jj} t | t |||j | }|S) a Enable or disable the dataset features decoding for audio, image, video. When enabled (default), media types are decoded: * audio -> dict of "array" and "sampling_rate" and "path" * image -> PIL.Image * video -> torchvision.io.VideoReader You can enable multithreading using `num_threads`. This is especially useful to speed up remote data streaming. However it can be slower than `num_threads=0` for local data on fast disks. Disabling decoding is useful if you want to iterate on the paths or bytes of the media files without actually decoding their content. To disable decoding you can use `.decode(False)`, which is equivalent to calling `.cast()` or `.cast_column()` with all the Audio, Image and Video types set to `decode=False`. Args: enable (`bool`, defaults to `True`): Enable or disable features decoding. num_threads (`int`, defaults to `0`): Enable multithreading for features decoding. Returns: `IterableDataset`: A copy of the dataset with casted features. Examples: Disable decoding: ```py >>> from datasets import load_dataset >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True) >>> next(iter(ds)) {'image': , 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'} >>> ds = ds.decode(False) >>> ds.features {'image': Image(mode=None, decode=False, id=None), 'text': Value('string')} >>> next(iter(ds)) { 'image': { 'path': 'hf://datasets/sshh12/planet-textures@69dc4cef7a5c4b2cfe387727ec8ea73d4bff7302/train/textures/0000.png', 'bytes': None }, 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.' } ``` Speed up streaming with multithreading: ```py >>> import os >>> from datasets import load_dataset >>> from tqdm import tqdm >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True) >>> num_threads = min(32, (os.cpu_count() or 1) + 4) >>> ds = ds.decode(num_threads=num_threads) >>> for _ in tqdm(ds): # 20 times faster ! ... ... ``` zFeatures decoding is only available for datasets with known features, but features are Unknown. Please set the datasets features with `ds = ds.cast(features)`.decodec,t|dr||_yy)Nr)hasattrr)rrs rP set_decodingz,IterableDataset.decode..set_decoding: sw)!'*rRrFTr)rr^rrr+rrmultiprocessingpool ThreadPool _apply_asyncrr|rr rr) rrrdsrdisabled_decoding_featuresenabled_decoding_featuresrfuncrs rPrzIterableDataset.decode s9@}}R  ( ( kAo)-););)= &(, (:(:(< % -w|U/K L ,glD.I J34B"''22;?D</H/W/WXD'@ABboo/EF FFNOR]oBOO K  {{'')H 8W\6: ;"B rRrrrsc 6t|j||}t||jj |j |j t j|jt j|j|jS)Nr{rw) rqr rr/rrxr!rr"r#r$)rrrrsr0s rP_stepzIterableDataset._stepP sp*4+<+<4PVW #"++''mmDOO4 d&7&78"55  rRc |j|S|jjr|jj}n(t|j dj }|j j}||_t|j||j|jtj|jtj|j|jS)Nrw)rr rryryr8rrrrxr!rr"r#r$rs rP_resolve_featuresz!IterableDataset._resolve_features\ s == $K    ' '((11H1$2B2B42H2N2N2PQHyy~~  ))++''mmDOO4 d&7&78"55  rRc d}|jrAt|jjDcic]\}}|t|c}}}nd}|j |d|||Scc}}w)a Group samples from the dataset into batches. Args: batch_size (`int`): The number of samples in each batch. drop_last_batch (`bool`, defaults to `False`): Whether to drop the last incomplete batch. Example: ```py >>> ds = load_dataset("some_dataset", streaming=True) >>> batched_ds = ds.batch(batch_size=32) ``` cX|jDcic] \}}||g c}}Scc}}wrM)rb) unbatchedkvs rPbatch_fnz'IterableDataset.batch..batch_fn~ s''0'89tq!AsF9 99s&NT)rrrr)rr$rbr&r|)rrrrrYrrs rPrkzIterableDataset.batcho sl : == H[H[H] ^ Wd7m!3 ^_HHxx dz?em  !_sA) c#FK|rD|jdj|D]}t|dj!yt j t |jdjd}t|djSw)aReturns the dataset as a Python dict. Can also return a generator for large datasets. Args: batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`. Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. Returns: `dict` or `Iterator[dict]` Example: ```py >>> ds.to_dict() ``` arrowrAunset fingerprintrN)ryrr!to_dictrorJr_rrrtables rPrzIterableDataset.to_dict s ))'277:7N De9AACC D$$T$*:*:7*C*H*HTX*H*Y%Z[E5g6>>@ @BB!ctjt|jdj d}t |dj S)zReturns the dataset as a Python list. Returns: `list` Example: ```py >>> ds.to_list() ``` rrrArr)rorJr_ryrr!to_list)rrs rPrzIterableDataset.to_list sI  d&6&6w&?&D&DPT&D&U!VWu'2::<>> ds.to_pandas() ``` rrArrrN)ryrr! to_pandasrorJr_rs rPrzIterableDataset.to_pandas s, ))'277:7N Fe9CCEE F$$T$*:*:7*C*H*HTX*H*Y%Z[E5g6@@B Brschema_overridesrechunkz pl.DataFramec#RK|rG|jdj|D]"}t|dj||$yt j t |jdjd}t|dj||Sw)aReturns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets. Args: batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`. Defaults to `genomicsml.datasets.config.DEFAULT_MAX_BATCH_SIZE`. batched (`bool`): Set to `True` to return a generator that yields the dataset as batches of `batch_size` rows. Defaults to `False` (returns the whole datasets once). schema_overrides (`dict`, *optional*): Support type specification or override of one or more columns; note that any dtypes inferred from the schema param will be overridden. rechunk (`bool`): Make sure that all data is in contiguous memory. Defaults to `True`. Returns: `polars.DataFrame` or `Iterator[polars.DataFrame]` Example: ```py >>> ds.to_polars() ``` rrArr)rrrN)ryrr! to_polarsrorJr_)rrrrrrs rPrzIterableDataset.to_polars s< ))'277:7N xe9CCUeovCww x$$T$*:*:7*C*H*HTX*H*Y%Z[E5g6@@Rbls@t tsB%B' path_or_bufstorage_optionsc tjt|jdj d}t |dj |f||d|S)aLExports the dataset to csv. This iterates on the dataset and loads it completely in memory before writing it. Args: path_or_buf (`PathLike` or `FileOrBuffer`): Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`), or a BinaryIO, where the dataset will be saved to in the specified format. batch_size (`int`, *optional*): Size of the batch to load in memory and write at once. Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. storage_options (`dict`, *optional*): Key/value pairs to be passed on to the file-system backend, if any. **to_csv_kwargs (additional keyword arguments): Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html). The parameter `index` defaults to `False` if not specified. If you would like to write the index, pass `index=True` and also set a name for the index column by passing `index_label`. Returns: `int`: The number of characters or bytes written. Example: ```py >>> ds.to_csv("path/to/dataset/directory") ``` rrrArrrr)rorJr_ryrr!to_csv)rrrr to_csv_kwargsrs rPrzIterableDataset.to_csv shF  d&6&6w&?&D&DPT&D&U!VW9wu'299  !+    rRc tjt|jdj d}t |dj |f||d|S)a*Export the dataset to JSON Lines or JSON. This iterates on the dataset and loads it completely in memory before writing it. The default output format is [JSON Lines](https://jsonlines.org/). To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`. Args: path_or_buf (`PathLike` or `FileOrBuffer`): Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`), or a BinaryIO, where the dataset will be saved to in the specified format. batch_size (`int`, *optional*): Size of the batch to load in memory and write at once. Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. storage_options (`dict`, *optional*): Key/value pairs to be passed on to the file-system backend, if any. **to_json_kwargs (additional keyword arguments): Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html). Default arguments are `lines=True` and `orient="records". The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`. If you would like to write the index, pass `index=True`. Returns: `int`: The number of characters or bytes written. Example: ```py >>> ds.to_json("path/to/dataset/directory/filename.jsonl") ``` ```py >>> num_shards = dataset.num_shards >>> for index in range(num_shards): ... shard = dataset.shard(index, num_shards) ... shard.to_json(f"path/of/my/dataset/data-{index:05d}.jsonl") ``` rrrArrr)rorJr_ryrr!to_json)rrrrto_json_kwargsrs rPrzIterableDataset.to_jsonsh\  d&6&6w&?&D&DPT&D&U!VW:wu'2::  !+    rRconzsqlalchemy.engine.Connectionzsqlalchemy.engine.Enginezsqlite3.Connectionc tjt|jdj d}t |dj ||fd|i|S)aExports the dataset to a SQL database. Args: name (`str`): Name of SQL table. con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`): A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database. batch_size (`int`, *optional*): Size of the batch to load in memory and write at once. Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. **sql_writer_kwargs (additional keyword arguments): Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html). The parameter `index` defaults to `False` if not specified. If you would like to write the index, pass `index=True` and also set a name for the index column by passing `index_label`. Returns: `int`: The number of records written. Example: ```py >>> # con provided as a connection URI string >>> ds.to_sql("data", "sqlite:///my_own_db.sql") >>> # con provided as a sqlite3 connection object >>> import sqlite3 >>> con = sqlite3.connect("my_own_db.sql") >>> with con: ... ds.to_sql("data", con) ``` rrrArrr)rorJr_ryrr!to_sql)rrgrrsql_writer_kwargsrs rPrzIterableDataset.to_sqlRs\N  d&6&6w&?&D&DPT&D&U!VW9wu'299$pPZp^opprRc ddlm}||jxstj}t j t|jdj|}t|dj|fd|i|S)aExports the dataset to parquet Args: path_or_buf (`PathLike` or `FileOrBuffer`): Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`), or a BinaryIO, where the dataset will be saved to in the specified format. batch_size (`int`, *optional*): Size of the batch to load in memory and write at once. Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. storage_options (`dict`, *optional*): Key/value pairs to be passed on to the file-system backend, if any. **parquet_writer_kwargs (additional keyword arguments): Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`. Returns: `int`: The number of characters or bytes written. Example: ```py >>> ds.to_parquet("path/to/dataset/directory") ``` ```py >>> num_shards = dataset.num_shards >>> for index in range(num_shards): ... shard = dataset.shard(index, num_shards) ... shard.to_parquet(f"path/of/my/dataset/data-{index:05d}.parquet") ``` r)get_arrow_writer_batch_size_from_featuresrrArrr) arrow_writerrrrDEFAULT_MAX_BATCH_SIZErorJr_ryrr! to_parquet)rrrrparquet_writer_kwargsrrs rPrzIterableDataset.to_parquet|sP L>t}}MnQWQnQn   d&6&6w&?&D&DPZ&D&[!\]=wu'2==  )8 .s2 Z[UQY cEkt T U s&)endpointrrrrT)rrrI-05d-of-z.parquetc3TK|]}j|j!ywrM) row_grouptotal_byte_size)rXrparquet_metadatas rPrZzEIterableDataset._push_parquet_shards_to_hub_single..s("BC **1-=="s%( path_in_repopath_or_fileobjdataset)r additions repo_typerrFN)rrrr HF_ENDPOINTrrryr|rr<r$rr rpq read_metadatanum_rowssumnum_row_groupsgetvaluerrpreupload_lfs_filesrM)rrrrrrrrrrrrr index_shardsapi uploaded_sizedataset_nbytes num_examplesrrrrshard_path_in_repobufferparquet_contentshard_additionrrrs` @@@rP"_push_parquet_shards_to_hub_singlez2IterableDataset._push_parquet_shards_to_hub_singles,H$8#f s63//ck&3,QA6 _dehkpep_q V//u=  .0 ( #LE5#S))'2 /4CZCZ[ HX" %-:QugQuSkjQTEUU]!^ YF   V $!//7  ,55 5L c"GLM]MlMlGm" N%oo/O S1 1M/=OapqN  # #)*#!# $    ^ ,%" "= #@dY EEEsF)F/num_procc p|rE|jjjD cgc]\} } t| ds| c} } ng} |xr t | }| |j }g} dx}}| xsd}t |Dcgc]"}|j||d||||||||||d $}}d}|| | dk\rd | d nd z }td || }| | dkrtjn t| 5}|tjdi|dnt|tj|}|D]2\}}}|s|j|| |dz } ||dz }||dz }4| |j!|j#dddt%d| D}| |||fScc} } wcc}w#1swY,xYw)aPushes the dataset shards as Parquet files to the hub. Returns: additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards uploaded_size (`int`): number of uploaded bytes to the repository dataset_nbytes (`int`): approximate size in bytes of the uploaded dataset after uncompression num_examples (`int`): number of examples of the uploaded dataset T)ignore_decode_attributeNrrrH) rrrrrrrrrrrzUploading the dataset shardsz (num_proc=)rGz shards)unittotaldesc)kwargs_iterablerc3HK|]}|jjywrM) upload_infor)rXadditions rPrZz>IterableDataset._push_parquet_shards_to_hub..FsP(H0055Ps "rN)r/rrbr-rrrrhf_tqdm contextlib nullcontextrrrrErcloserr)rrrrrrrrrrrrdecodable_columnsrrrrrrrpbarr update_streamrFcontentrs rP_push_parquet_shards_to_hubz+IterableDataset._push_parquet_shards_to_hubs4$ ::..446 l41a:J1fj:kQ l  4O=N8O  J.0 ())=q /  hfQU V $"$$&((<   . X-AhRSm+hZq)Y[[  *2)9X\Z # # %tT\~ ae< BBX_UVEWX'#FF$3 *7 /%gKK(+I"gaj0N GAJ.L  /  ' *PiPP -EEw m .  sF!F!'F'1BF,,F5 config_name set_defaultcommit_messagecommit_descriptionprivatec   !"dtjvr td||jkDrrtj d|djd|jz dtj djd jd j}d k(r td #jtjnd tjtstdtddttj  jdj  ' j)dsj+ dds dk7rnd j- | | | | \" !dt.tt0t2tt4tff !  "f d }||nd}t7tj8kDrOtj dtj8dt;j<t7tj8z }t?d|D]}|tj8z|d ztj8z}tAtCjDt?d!tCjFd"d #D]E\}}|d tIjHzz} jK||d$|d%d&|d%d'z|d | (}ntj d.|d zd/||z d z r d0||z d z d1nd2zd zg}n}tAtCjDt?d!tCjFd"d #D]\}}|d tIjHzz}|\}}}}g}|r9|j]t_tj`|jcd34|j]t_tjdt|jc4 jK||z|z||d | |5}|SS#t"$r%j%d|d}|j&Y>wxYw#tL$r}|jNrtQ|jNtRr~|jNjTjVd)k(r[tYjZ|tj d*d+d,|d-|jNjTjVd' Yd}~pd}~wwxYw#tL$r}|jNrtQ|jNtRr}|jNjTjVd6vr[tYjZ|tj d7d+d,|d-|jNjTjVd' Yd}~cd}~wwxYw)8aGPushes the dataset to the hub as a Parquet dataset. The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed. The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`] data, the Parquet files will store the bytes of your images or audio files. You can disable this by setting `embed_external_files` to `False`. Args: repo_id (`str`): The ID of the repository to push to in the following format: `/` or `/`. Also accepts ``, which will default to the namespace of the logged-in user. config_name (`str`, defaults to "default"): The configuration name (or subset) of a dataset. Defaults to "default". set_default (`bool`, *optional*): Whether to set this configuration as the default one. Otherwise, the default configuration is the one named "default". split (`str`, *optional*): The name of the split that will be given to that dataset. Defaults to `self.split`. data_dir (`str`, *optional*): Directory name that will contain the uploaded data files. Defaults to the `config_name` if different from "default", else "data". commit_message (`str`, *optional*): Message to commit while pushing. Will default to `"Upload dataset"`. commit_description (`str`, *optional*): Description of the commit that will be created. Additionally, description of the PR if a PR is created (`create_pr` is True). private (`bool`, *optional*): Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. token (`str`, *optional*): An optional authentication token for the Hugging Face Hub. If no token is passed, will default to the token saved locally when logging in with `huggingface-cli login`. Will raise an error if no token is passed and the user is not logged-in. revision (`str`, *optional*): Branch to push the uploaded files to. Defaults to the `"main"` branch. create_pr (`bool`, *optional*, defaults to `False`): Whether to create a PR with the uploaded files or directly commit. num_shards (`int`, *optional*): Number of shards to write. Equals to this dataset's `.num_shards` by default. embed_external_files (`bool`, defaults to `True`): Whether to embed file bytes in the shards. In particular, this will do the following before the push for the fields of type: - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files. num_proc (`int`, *optional*, defaults to `None`): Number of processes when preparing and uploading the dataset. This is helpful if the dataset is made of many samples and transformations. Multiprocessing is disabled by default. Return: huggingface_hub.CommitInfo Example: ```python >>> dataset.push_to_hub("/") >>> dataset_dict.push_to_hub("/", private=True) >>> dataset.push_to_hub("/", num_shards=1024) ``` If your dataset has multiple splits (e.g. train/validation/test): ```python >>> train_dataset.push_to_hub("/", split="train") >>> val_dataset.push_to_hub("/", split="validation") >>> # later >>> dataset = load_dataset("/") >>> train_dataset = dataset["train"] >>> val_dataset = dataset["validation"] ``` If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): ```python >>> english_dataset.push_to_hub("/", "en") >>> french_dataset.push_to_hub("/", "fr") >>> # later >>> english_dataset = load_dataset("/", "en") >>> french_dataset = load_dataset("/", "fr") ``` zVideo(a]push_to_hub is not implemented for video datasets, instead you should upload the video files using e.g. the huggingface_hub library and optionally upload a metadata.csv or metadata.jsonl file containing other information like video captions, features or labels. More information at https://huggingface.co/docs/datasets/main/en/video_load#videofolderNzToo many num_proc: rCrDz processes.zTo parallelize data loading, we give each process some shards (or data sources) to process. Therefore it's unnecessary to have a number of processes greater than dataset.num_shards=rEr&rzN`config_name` cannot be 'data'. Please, choose another name for configuration.trainzSplit name should match 'z ' but got 'z'.rr)rT)rr exist_okzrefs/pr/)branchrrr#default) rrrrrrrrrrmc ~ %j*d+j}d\}}g}d}g}$Dcgc]}|j}}%j*|d/dD]2}t |t s|j tjk(rd}5|j tjk(rd}U|j j'd.drC|j |vr5|jt|j ||jz }tj|j tj d d st#t} t%|j | } | J| d } | |vs"|j| 5d*vr*j'dnd*f\} } ,j(j+}d|_0|_(|_0(z|_&|_t7.t9.()| i|_|r%j=*tjd|}t?j@tC|}|jD}tGjH|}tKjH|}|r &|vr|&}nd}n|rd}tM}tG}%j=*tjd|}tO|d5}tQj@|}|r|jS&dnd}|rtUjV|nd}dddnd}tM}tG}d}tXj)d|j:rlt[|j:.gk7rR,j\j^|j^k7r/tad,j\j^d|j^.|j:vrZ|xj.|zc_|xj0|j:jS.t9jbxsdzc_d|_|j.xsd0z|_|j0xsd(z|_|j.|j0z|_|j:je.dt9.(tg,| |j:.<|}|s7|r5d|Dcgc] }|d|dd c}i}tGd|iji|&|vrb|&}d|vrtk|d}ni}'d.dg|.<d|jmDcgc]\}}|tg|dk(r|dn|dc}}i}nd.'d.ddgi}&|i}-rJ&dk7rE|r>|jo} | dk(r tad| r|| jed}!|| || <d|d<|rt%j=*tjd|}tO|d5}tQj@|}dddtq|&<tQjr|d}"nd}"tK&|iji|tG|ji||t?d|dn|}#|||#|"fScc}w#1swYdxYwcc}wcc}}w#1swYxYw)Nr)rr)FFrT)rrrr recursiverIr)rz{split}*r) num_bytesr dataset_nameutf-8)encodingz0Updating downloaded metadata with the new split.zVFeatures of the new split don't match the features of the existing splits on the hub: z != data_fileszdata/z-*)rpathr%rzzThere exists a configuration named 'default'. To set a different configuration as default, rename the 'default' one first.)indentz--- z --- ): repo_infosharlist_repo_treerr rfilenamerREPOCARD_FILENAMEDATASETDICT_INFOS_FILENAME startswithrMrrfnmatchr replacerDrFrrrdownload_checksums download_size dataset_size size_in_bytesrr9r:splitshf_hub_downloadrloadr rrAfrom_dataset_card_datar5ropenjsonr|r4 from_dictrXr_r/rr^r)rGrto_dataset_card_datar#rbget_default_config_namerCdumps)1 parent_commitrepo_with_dataset_cardrepo_with_dataset_infos deletions deleted_size repo_splitsrrepo_files_to_add repo_filepatternsplit_pattern_fields repo_split organizationr* info_to_dumpdataset_card_path dataset_carddataset_card_datametadata_configs dataset_infosr1dataset_infos_pathf dataset_infor default_metadata_configs_to_dumpmetadata_configdata_files_to_dumprx_patternmetadata_config_to_dumpconfigs_to_dumpcurrent_default_config_namernew_dataset_infosnew_dataset_cardrrrrrrrrrrrrrs1 rPget_deletions_and_dataset_cardzCIterableDataset.push_to_hub..get_deletions_and_dataset_cards`MM'YQYMZ^^M?K ; "$;57IL%'KGP Q8!6!6 Q  Q //-9TYei0 7 ")X6&&&*B*BB-1*((F,M,MM.2+''22hZqq3IJ!++3DD$$%: H[H[%\] INN2L__''NVVW`bef44noG+9):M:Mw+W(/;;;!5g!>J!4#**:61 74@Cg~s);TXZaSb &L,99>>+L.2L +)6L &(6L %)6)GL &'2L $"+9lam#L &$'$7$7V55Ub%8%! +//5F0GH $0$5$5!#2#I#IJ[#\ 2B2Y2YZk2l  [M%A -k :I $I(# $3$5!#2#4 %(%8%8V>>)^k&9&",w?^1*.))A,MKX=#4#4[$#G^bLGS 5 5l CY]I^^ $ $3$5!#2#4  $ NO##Y-=-=(>5'(Izz**i.@.@@(tuyuuvIvIuJJNOXOaOaNbc 0 00!//<?/!..)2B2B2F2Fuik2Z2d2d2ihii.37I0/8/F/F/K!}.\I+.7.D.D.I^-[I*.7.E.E H^H^.^I+$$((5.7c$i^j/I$$U+$-L#  \g"hSXUeE7"3,-HIMMiXGWXsGt(CD59' 2&%(%8%8V>>)^k&9&",w?71*.))A,M7-3L-A k*$(JJ}Q$G!$(! k<8 9 N NO` a O , A ABS TCOCW e$5#6g>?]i !)-=?PP Pk!R|^^D#i#877s*ZAZZ( "Z-Z3Z%3Z<zUpload datasetz)Number of files to upload is larger than z+. Splitting the push into multiple commits.rrr<)rz (part rrr ) operationsrrrrrz!Retrying intermediate commit for z, z (z/n with status_code zCommit #z completedz (still z to go)rGr+r)rhrrrrrrH)irizRetrying commit for )3rrrrrXrrr^rrematchr6rrrr1rRr create_reporr7 create_branchrrr_rrrUPLOADS_MAX_NUMBER_PER_COMMITmathceilrrKrchainrr create_commitr __context__rrresponse status_codetimesleeprMrr6encoder5)#rrrrrrrrr rrrrrrrepo_urlrf num_commitsrrhretry sleep_time commit_infoerrlast_commit_additionsrHrKrVrYdataset_card_additionsrrrrrs#`````` `` @@@@@rP push_to_hubzIterableDataset.push_to_hubIsaH s4==) )%Y   Ht$> NN%hZ/KDOOK\]$t67{D  KKllpl{l{k|}[[_[j[jZkkln  H & mn n ='+zz'=C O7Exx 5)8 ;ugUWXY YV//u= 'mmGymADDG  (;(;J(G   gheycg  h&1Y&>{FHAEAaAa!!5Bb B > =.,^ Qc4@U;VX[]efi]j6j0k^ Q^ Q^ Q@,:+EK[ y>F@@ @ KK;F<`<`;abMN ))C NV5Y5Y$YZK1k*$ &<<<AImIm?m *39??59iN^N^_aNb3ckl)m%E:!fmmo"55J"&)&7&7#'1+9gaWDQ\]`Paab[]_A$ J%' !$- !!*9??59iFVFVWYFZ+[cd!e)  E: !fmmo- -JDbDd AM9lM%' "&--&%+%F%F(5(<(00<< JJJz*KK.wir+bOcdgdsdsd|d|eIeIdJJKL sP4Q"1$R=U "*RR UBUUU W;BW65W66W;)NNNNNN))rFr\r=rM) NFNFrFNNN)NFNFrN)NNrr)Trr)NFNT) r%NNNNNNNNFNTN)[rrrrrrr4r7rrrr}rrrrrrr)r_r_rrr0r3r6r8r:r>rrArVrPr'rrrrrr8TRAINrr$rfrnrtryr|rrrrrrrrrrrrrrrrr%rrrrrrkrrrrrrrrKrrrrrrrrrrrrrNrRrPrrsC * '+&*15/337IMG*G{#G # G -. G O, G/0G$DeCtO.D)D$EFG: EXc] E E FhtCy1 F F3/D3/j0/$0/40/drG3 s   =,C,, #9v '+$(V 8$V V  c49n 56 V  V SM V V !sDI~!67V 8$V D>V  V t(,9=$($(T 8$T  c49n 56 T  T SM T D>T  T n^bA $,RYY-@-@$AA WZA A F+s+* c* /* X)  ) 2C) V# c# /# R 7 7 7  7  7 r ]s ]E$.,A ]FW ]L#LLPaL<T#s(^@Q2!5d3i+@!EV!F1 5d3i+@1 EV1 f- #- - @Q- ^- -  - ^[T[s[CT[z  #  s  /@   &  d GX 4A(3-AARWX\^fgk^lXlRmA. = = AFC"3-C9=C r||Xbll33 4C>%)+/ #uSM#u#u#4. #u  #u ~x77 8 #uP%)*. ) 8X-.) SM) "$ ) ) \%)*. 4 8X-.4 SM4 "$ 4 4 t%) (q(q368RThh i(qSM (q (qZ%)*. . 8X-.. SM. "$ . . `EFEFEF EF  EF  EF}EF3-EFD>EFEF#EF %/0#s:; <EFNTFTFTF TF } TF 3- TFD>TFSMTF#TF3-TF t&'c36 7TFr%&*#"&(,,0"&#"&$)$(%)"&!```d^ ` } ` 3- `! `%SM`$`}`3-`D>`SM`#` 3-!`" #`rRrdsetsrraxisc |Dcgc]}|j}}|dk(r$t|Dcgc]}|jc}n,t|Dcgc]}|jD]}|c}}t d|Drd}nt d|Drd}t jdne|Dchc]}|jj}}t|dk(r|j} t| }nd}t jdtt|Dcgc]}|jc}D cic]} | jD]\} } | |  c} } } } |Dcgc]!}t!j"|j$#} }|dk(r t'| }n t)| }|.t+j,|Dcgc]}|jc}}n|j!}| |_|Dcic]'}|j.jD]\}}|| )}}}}t1|||||Scc}wcc}wcc}}wcc}wcc}wcc} } } wcc}wcc}wcc}}}w) a Converts a list of `IterableDataset` with the same schema into a single `IterableDataset`. Missing data are filled with None values. Args: dsets (`List[datasets.IterableDataset]`): List of Datasets to concatenate. info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. split (`NamedSplit`, optional): Name of the dataset split. axis (``{0, 1}``, default ``0``, meaning over rows): Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns (horizontally). *New in version 1.6.0* Example: ```py >>> ds3 = _concatenate_iterable_datasets([ds1, ds2]) ``` rc38K|]}|jduywrMr!rXdsets rPrZz1_concatenate_iterable_datasets..s 64  t # 6rNc38K|]}|jduywrMrrs rPrZz1_concatenate_iterable_datasets..s 8$T   % 8rzoSome of the datasets have disparate format or format not set. Resetting the format of the concatenated dataset.rrv)r0rrrr)rr)rrrr]rXrr!r rrGrr$r(rbrrr rrr4 from_merger$r)rrrrr5rcol_namerformat_type_setr rrrrr0rrrrs rP_concatenate_iterable_datasetsrsG8-2 2qQ " 2E 2 qy)U*KT4==*KL%V$ VHXVXVW 6 66 8% 8 8  } EJJD4++77JJ  1 $)--/K)kBJJ KKB  -.Ot}}.OPnn(]e]k]k]mnUYUVXYAnnHyy{DM7<vvGSZSmSmSsSsSuv%%vvv  +  a 3+LVK/PnB'> ws4III 6I%I "I 2&I$ I),I.datasetsrrrrc j|Dcgc]}|j}}t|Dcgc]}|jc}tt |Dcgc]}|jc}D cic]}|j D]\} } | |  c} } }}|Dcgc]!}t j|j#} }|t| |} n.tjj|} t| | ||} |.tj|Dcgc]}|j c}}n|j }||_|Dcic]'}|j"j D]\}}|| )}}}}t%| |||Scc}wcc}wcc}wcc} } }wcc}wcc}wcc}}}w)a Interleave several iterable datasets (sources) into a single iterable dataset. The new iterable dataset alternates between the sources to yield examples. If `probabilities = None` (default) the iterable dataset will cycles through the sources in order for each next example in the iteration. If `probabilities` is not `None, the iterable dataset will sample a random source according to the provided probabilities for each next examples in the iteration. Args: datasets (`List[IterableDataset]`): list of datasets to interleave probabilities (`List[float]`, optional, default None): If specified, the new iterable dataset samples examples from one source at a time according to these probabilities. seed (`int`, optional, default None): The random seed used to choose a source for each example. stopping_strategy (`str`, defaults to `first_exhausted`): Two strategies are proposed right now. By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples. If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once. Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous: - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples. - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting. Output: `datasets.IterableDataset` rr)r0rrr)rr)rr$r(rbrrr rrrr<rr4rrr$r)rrrrrrr5rrrrrr0rrrrrs rP_interleave_iterable_datasetsrAs@088!##%8H8&&Jt}}&JK-.Rt}}.RSqq(`h`n`n`pqX\XY[\AqqH7wrk) apply_asyncreadyr|rCrw)rrrOfutures rPrrsN   dQD )F  <<>::< --" " "  #sA AAArMr\)NNr)NNNNr)rCrrr8r?rrCromultiprocessing.poolrrrjrrv collectionsrcollections.abcrrr dataclassesr functoolsrior r r pathlibr typingr rrrrr fsspec.asynrLnumpyrpandasrpyarrowropyarrow.parquetparquetrhuggingface_hubrrrrrrhuggingface_hub.hf_apirhuggingface_hub.utilsrr multiprocessrrequestsrrGr arrow_datasetr r!r"r-r#rr$features.featuresr%r&r'r(r)r*r+r,r-rr.r/r0r1r2r3rr4r5namingr6r>r7r8r9r:rr;r<r=r>rr?r utils.loggingr@utils.metadatarAutils.py_utilsrBrCrDrErFutils.shardingrGrHrIrJ utils.typingrKsqlite3rr sqlalchemyrrrXrrrrQr}rer_rjryrrrrrprrrrrr%r/r^rqrrrrrrrrnro BooleanScalarrrrxrzr|rrrrrrrrrrrrrrrrrrrrNrRrPrs   .!#JJvv+Ipp)   0;;aa"%+ffvv" H  CHo   d38n 4cd4j 7d39o7XhEW7ck7#d38n!5#$sDy/#>d39o>(4S>2J>"fuS$Y'(fffeCM"# f@CoCoL"<,"-B4]`4 44$)$.$94TW4J+5J+ZD+%:D+NH+0H+V.+2.+bN+0N+b   % :>sE#tUY/DZ?Z:[      # 8