bi)dZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZddlmZddlmZmZmZddlZddlZddlZddlmZdd lmZmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%d d l&m'Z'm(Z(d d l)m*Z*d dl+m,Z,m-Z-d dl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4d dl5m6Z6m7Z7d dl8m9Z9d dl:m;Z;d dlZ>m?Z?m@Z@d dlAmBZBmCZCd dlDmEZEd dlFmGZGd dlHmIZId dlJmKZKmLZLd dlMmNZNd dlOmPZPmQZQd dlRmSZSmTZTmUZUmVZVd dlWmXZXd dlYmZZZd dl[m\Z\d dl]m^Z^m_Z_m`Z`maZambZbd d lcmdZdd d!lemfZfmgZgd d"lhmiZid d#ljmkZkd d$llmmZmd d%lnmoZoeiepZqereSjd&gzZtGd'd(Zud)eve-d*ere,d+eewd,ewd-eve-f d.Zxd-eeve-fd/Zy dbd0d1d,eewd-eve-fd2Zzd3ewfd4Z{ dbd5e0d6ee9d-e|eewe}ffd7Z~ dbd5e0d6ee9d-e|eewe}ffd8Z dcd9e/d:eewd6ee9d-e|eewe}ewefffd;Z dddeewd?e}ewefd6ee9d-e|ere,ewff d@Ze GdAdBZe GdCd1ZGdDdEZGdFdGeZGdHdIeZGdJdKeZGdLdMeZGdNdOeZ ded:ewdPeeeweofd6ee9dQeee;ewfdReewd9eee}erewe/fdSeewd-efdTZ dfd:ewd3eewdReewd9eeewe ewe eweewe ewfffdSeewdUeeEd6ee9dQeee;ewfdPeeeweofdVeeeewfdWee}d-e-fdXZ dgd:ewd3eewdReewd9eeewe ewe eweewe ewfffdYeeeweZerewereZfdSeewdUeeEd6ee9dQeee;ewfdZeeefewfd[eed\edPeeeweofdVeeeewfd]ed^eedWee}d-ee6e*e7eNff$d_Z dcd`emd[eedWee}d-ee*e6ffdaZy)hzAccess datasets.N)Counter)MappingSequence) dataclassfield)Path)AnyOptionalUnion) url_to_fs) DatasetCardDatasetCardDataHfApi)EntryNotFoundErrorGatedRepoErrorLocalEntryNotFoundErrorOfflineModeIsEnabledRepositoryNotFoundErrorRevisionNotFoundError get_session) __version__config)Dataset) BuilderConfigDatasetBuilder) DataFilesDict DataFilesListDataFilesPatternsDictEmptyDatasetErrorget_data_patternssanitize_patterns) DatasetDictIterableDatasetDict)DownloadConfig) DownloadMode)StreamingDownloadManager xbasenamexglobxjoin)DataFilesNotFoundErrorDatasetNotFoundError)Features)%_fix_for_backward_compatible_features)Hasher) DatasetInfoDatasetInfosDict)IterableDataset)camelcase_to_snakecasesnakecase_to_camelcase)_EXTENSION_TO_MODULE_MODULE_TO_EXTENSIONS_MODULE_TO_METADATA_FILE_NAMES_PACKAGED_DATASETS_MODULES)FolderBasedBuilder)Split)_dataset_viewer)!_raise_if_offline_mode_is_enabled cached_pathget_datasets_user_agentis_relative_pathrelative_to_absolute_path)hf_dataset_url)VerificationModeis_small_dataset) get_logger)MetadataConfigs)PathLike)Version.zipceZdZdZdZy)#_InitializeConfiguredDatasetBuilderaL From https://stackoverflow.com/questions/4647566/pickle-a-dynamically-parameterized-sub-class See also ConfiguredDatasetBuilder.__reduce__ When called with the param value as the only argument, returns an un-initialized instance of the parameterized class. Subsequent __setstate__ will be called by pickle. cBt}t|||||_|S)N)default_config_name dataset_name)rJconfigure_builder_class __class__)self builder_clsmetadata_configsrLnameobjs H/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/load.py__call__z,_InitializeConfiguredDatasetBuilder.__call__is)13/ )?Rae   N)__name__ __module__ __qualname____doc__rVrWrUrJrJ`s rWrJrQbuilder_configsrLrMreturncGfdd}jjjt||_jjjt||_|S)z Dynamically create a builder class with custom builder configs parsed from README.md file, i.e. set BUILDER_CONFIGS class variable of a builder class to custom configs list. c:eZdZWZWZWjZdZy)9configure_builder_class..ConfiguredDatasetBuilderc|jjd}t||j|j|j f|j jfS)Nr)rO__mro__rJBUILDER_CONFIGSDEFAULT_CONFIG_NAMErM__dict__copy)rPparent_builder_clss rU __reduce__zDconfigure_builder_class..ConfiguredDatasetBuilder.__reduce__s\!%!7!7!: 35&((,,%%   ""$  rWN)rXrYrZrdreri)rQr]rLsrUConfiguredDatasetBuilderra}s)1 ++  rWrj)rXlower capitalizer4rZ)rQr]rLrMrjs``` rUrNrNrs;(    % % ' 2 2 4 56L\6Z5[\%    % % ' 2 2 4 56L\6Z5[\) $#rWc6tj|}d}|jjD]d\}}t j |st |ts-t j|rC|}t j|}|]||k(sc|S|S)zJImport a module at module_path and return its main class: a DatasetBuilderN) importlib import_modulerfitemsinspectisclass issubclassr isabstract getmodule) module_pathmodulemodule_main_clsrSrT obj_modules rUimport_main_classrzs  $ $[ 1FO__**, c ??3 JsN$C!!#&!O **3/J%&J*>  rWdataset_module DatasetModulec"t|j}|jjrc|xs|jj d}| t dt||jj|jj|}|S)NrMz-dataset_name should be specified but got None)r]rLrM) rzrvbuilder_configs_parametersr]builder_kwargsget ValueErrorrNrL)r{rMrQs rUget_dataset_builder_classrs$N$>$>?K00@@#X~'D'D'H'H'X  LM M- *EEUU . I I ] ]%  rWrSctjs\tjrK tj dj tj ||dzfdtidyyy#t$rYywxYw)z'Update the download count of a dataset./.pyz User-Agent)headerstimeoutN) rHF_HUB_OFFLINEHF_UPDATE_DOWNLOAD_COUNTSrheadjoinS3_DATASETS_BUCKET_PREFIXr> Exception)rSs rUincrease_load_countrsv  V%E%E  M  &::D$,OP%'>'@A  &F    sA A.. A:9A:data_files_listdownload_configc^td|dtjD}|rdtttt ft fdtt t ffd}t|j|dD]/\\}}}|tvr t|cS|dk(s"t|| cSdifS) a%Infer module (and builder kwargs) from list of data files. It picks the module based on the most common file extension. In case of a draw ".parquet" is the favorite, and then alphabetical order. Args: data_files_list (DataFilesList): List of data files. download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types. Returns: tuple[str, dict[str, Any]]: Tuple with - inferred module name - dict of builder kwargs c3K|]T}t|jdddD]2}d|jzt|tjvf4Vyw.rN)r(splitrkr9METADATA_FILENAMES.0filepathsuffixs rU z3infer_module_for_data_files_list..se! )//4QR8!  v||~ y26H6[6[[\!\!sAAN ext_countr^c>|\\}}}| ||dk(|dk(|dk(|dk(|fS)z]Sort by count and set ".parquet" as the favorite in case of a draw, and ignore metadata filesz.parquetz.jsonlz.jsonz.csvr\)rext is_metadatacounts rUsort_keyz2infer_module_for_data_files_list..sort_keys@(1 % S+#OUC:,=shPSW^P^`cgm`mors srWT)keyreverserHr) rr*DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCEtuplestrboolintsortedrpr5,infer_module_for_data_files_list_in_archives)rrextensions_counterrr_s rU infer_module_for_data_files_listrs"!!'([&*[*[\!  teCI&6&; < tsDyAQ t ""4":":"<(TXY vKHS!a**+C00COetuu  v 8OrWcg}d}|D]}t|jds|dz }|tjkDrngt t j |d}|t|d|dtjDcgc]}|jddc}z }td |D}|r(|jddd}|tvr t|SdifScc}w) aInfer module (and builder kwargs) from list of archive data files. Args: data_files_list (DataFilesList): List of data files. download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types. Returns: tuple[str, dict[str, Any]]: Tuple with - inferred module name - dict of builder kwargs rrHrz**T) recursiverNz::c3K|]9}t|jdddD]}d|jz;ywr)r(rrkrs rUrz?infer_module_for_data_files_list_in_archives.. sM!!))T\J]JcJcdgJhijikJl!@Fflln!!s?A) rendswithr2GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCEr*r'extractr)3ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCErr most_commonr5) rrarchived_filesarchive_files_counterr extractedfrrs rUrrsN#  x= ! !& ) !Q & !$v'`'``68@@JDQI yD/ZPfPP a  N !!-;!(44Q7:1= . .' 4 4 8OsC& data_filespathc \|jDcic]\}}|t||}}}tt|j \t fd|j Drt d|std|r d|zdzfScc}}w)a$Infer module (and builder kwargs) from data files. Raise if module names for different splits don't match. Args: data_files ([`DataFilesDict`]): Dict of list of data files. path (str, *optional*): Dataset name or path. download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters to authenticate on the Hugging Face Hub for private remote files. Returns: tuple[str, dict[str, Any]]: Tuple with - inferred module name - builder kwargs rc3,K|] }f|k7 ywNr\)r split_moduledefault_builder_kwargs module_names rUrz.infer_module_for_data_files..)s l\K/ 0L @ lsz=Couldn't infer the same data file format for all splits. Got zNo (supported) data files foundz in )rprnextitervaluesanyrr+)rrrrr split_modulesrrs @@rUinfer_module_for_data_filesrs$'1&6&6&8 "E? /Q`aaM+/tM4H4H4J/K*L'K' lUbUiUiUk llXYfXghii $%F[_DQUPV-%hiieg%hii . ..sB(rvrR base_pathrct|}|j}|j}g}|in|}||nd}|jD]\} } | j d} | j d} | r t || n|} | t | n t| |}tj|t}| Dcgc]}t||r|d k7s|}}|rtjd |d |j|d| || d i|| jDcic]\}}t||r|d vr||c}}||fS#t$r}td|d| d|d}~wwxYwcc}wcc}}w)Nrrdata_dirr)allowed_extensionsz Dataset at 'z?' doesn't contain data files matching the patterns for config 'zZ', check `data_files` and `data_fir` parameters in the `configs` YAML field in README.md. defaultz#Some datasets params were ignored: zx. Make sure to use only valid params for the dataset builder and to have a up-to-date version of the `datasets` library.)rSrr)rrrr\)rzBUILDER_CONFIG_CLASSget_default_config_namerprr*r"r!r from_patternsALL_ALLOWED_EXTENSIONSr hasattrloggerwarningappend)rvrRrrrrQbuilder_config_clsrLr] config_name config_paramsconfig_data_filesconfig_data_dirconfig_base_pathconfig_patternsconfig_data_files_dicteparamignored_paramsvalues rU,create_builder_configs_from_metadata_configsr0s$K0K$99*BBDO#9#ARG]&2 I&6&<&<&>' " ])--l;'++J7@O5Op5L  9' P / //5! #yk)hithuvkl    s0?5E8E, E,E,E1 E)E$$E)cTeZdZUdZdZeeed<dZee e ed<dZ ee ed<y)BuilderConfigsParametersaDataclass containing objects related to creation of builder configurations from yaml's metadata content. Attributes: metadata_configs (`MetadataConfigs`, *optional*): Configs parsed from yaml's metadata. builder_configs (`list[BuilderConfig]`, *optional*): List of BuilderConfig objects created from metadata_configs above. default_config_name (`str`): Name of default config taken from yaml's metadata. NrRr]rL) rXrYrZr[rRr rE__annotations__r]listrrLrr\rWrUrris< 37h/659OXd=129)-#-rWrc\eZdZUeed<eed<eed<eeZeed<dZ e e ed<y)r|rvhashr)default_factoryr~N dataset_infos) rXrYrZrrdictrrr~rr r1r\rWrUr|r|{s5 I;@Qi;j 8j04M8,-4rWceZdZdefdZy)_DatasetModuleFactoryr^ctr)NotImplementedError)rPs rU get_modulez _DatasetModuleFactory.get_modules!!rWN)rXrYrZr|rr\rWrUrrs"M"rWrc deZdZdZ d dedeedeeeeefdeee effdZ de fd Z y) LocalDatasetModuleFactoryzGet the module of a dataset loaded from the user's data files. The dataset builder module to use is inferred from the data files extensions.Nrrr download_modec|r-tjj|rtd|t |j |_t |j |_||_||_ ||_ y)Nz;`data_dir` must be relative to a dataset directory's root: ) osrisabsrras_posixstemrSrrr)rPrrrrs rU__init__z"LocalDatasetModuleFactory.__init__sd  h/Z[_Z`ab bJ'') JOO $  *rWr^c  tjj|jtj}tjj|jtj }tjj |rtj|jn t}tjj|rgt|d5}tj|j}|r,|j!}|j#|tdi|}dddt%j&|}t)j&|}t+|j|j,xsdj/j1j3} |j4t7|j4} nl|r_|j,sSdt9t;|j=vr/t7t9t;|j=d} n t?| } tAjB| | tD} tG| |j\} } | jItJ| tL| } tN| \}}|rtQ||| | \}}n tS|jTdd| i| g}d}|jtWt+|jjXd }|j,r| |d<tjj tjj|jtjZrttjj|jtjZd5}t)t]j|j_Dcic]\}}|tajb|c}}}te|d k(r(t9t;|}|jg||d <dddj#||}|"te|d k(rt9t;|}tijj||d }tm||||to||| S#1swYxYwcc}}w#1swYxYw)Nutf-8encodingrr)rr)rr extensions file_names)rr)rrMrr)rr]rRr]rLrr~r\)8rrrrREPOCARD_FILENAMEREPOYAML_FILENAMEisfiler loaddatarexistsopenyaml safe_loadreadto_dictupdaterEfrom_dataset_card_datar1rr expanduserresolverrr"rrrr!rrrrfilterr6r7r8rrzrr3rSDATASETDICT_INFOS_FILENAMEjsonrpr0 from_dictlenpopr/rr|r)rP readme_pathstandalone_yaml_pathdataset_card_datarstandalone_yaml_data_dataset_card_data_dictrRrrpatternsrrrrvrr]rLrrdataset_info_dictlegacy_dataset_infoslegacy_config_namers rUrz$LocalDatasetModuleFactory.get_modulesggll499f.F.FG !ww||DIIv7O7OPBD''..Q\B]K,,[9>>crct 77>>. /*W= S'+~~affh'?$'.?.G.G.I++223GH(7(R:Q(R%  S +AABST(??@QR DMM$7R8CCEMMOXXZ ?? &(9H dmm TRbRiRiRkMlHm8m(d3C3J3J3L.M)N|)\]H(3H"00 5 /J!/ + + &&,[9FdepFq' 4K@ Q 3_ #'= 4 0O0D!+.CC),4O #' 24 ?3G3GH  ==+5N< ( 77>>"'',,tyy&2S2ST Ubggll499f.O.OP[bc cgh'7?Ciil>P>P>R:K):$[%:%:;L%MM($ +,1)-d3G.H)I&6J6N6NOa6b(3 c ! ' ' 60M  &3}+=+B"&tM':"; {{]_`   ''?!1 /$7(  M S Sn c cs+%AR8;,S ' S >S 8SS  SNNN) rXrYrZr[rr r rrr&rr|rr\rWrUrrsn' #'7;<@ ++3-+U3d?34 +  lC&7 89 + U MU rWrcpeZdZdZ d dedeedeeeeefdee deee eff dZ d e fd Z y) PackagedDatasetModuleFactoryz`Get the dataset builder module from the ones that are packaged with the library: csv, json, etc.NrSrrrrc`||_||_||_||_||_t |yr)rSrrrrr)rPrSrrrrs rUrz%PackagedDatasetModuleFactory.__init__s1 $  .*D!rWr^ct|jxsdjjj }|j t |j nt||j}tj||j|}t|j\}}||jd}t|||S)Nrr)rr)rrM)rrrrrrr"r!rrrr8rSr|)rPrrrrvrrs rUrz'PackagedDatasetModuleFactory.get_modules,"-88:BBDMMO * doo ."9d>R>RS  #00  00 7tyyA T% II  [$??rW)NNNN)rXrYrZr[rr r rrr%r&rr|rr\rWrUr%r%s~j #'7;48<@ " "3- "U3d?34 " ".1 "  lC&7 89 "@M@rWr%czeZdZdZ d dededeedeeeeefdee deee efd e fd Z d e fd Zy)HubDatasetModuleFactoryz Get the module of a dataset loaded from data files of a dataset repository. The dataset builder module to use is inferred from the data files extensions. NrS commit_hashrrrruse_exported_dataset_infosc||_||_||_||_|xs t |_||_||_t|yr) rSr*rrr%rrr+r)rPrSr*rrrrr+s rUrz HubDatasetModuleFactory.__init__$sI &$  .B.2B**D'D!rWr^c  ttj|jjdt t |jj} |j|jtjd|j|jj}tj|j}|jj%}|j&d|_ t)t+|jtj,|j|}t/|d 5}t1j2|j5}|r,|j7}|j9|t#di|}dddd |jd |jd |j<xsd j?d } tAjB|} tEjB|} tjFr~|jHrr tKjL|j|j|jj} tE| D cic]} | tOjP| | c} } nd} | r| j9| | } |jTtW|jT}nx| r_|j<sSdtYt[| j]vr/tWtYt[| j]d}nt_| |j}tajb|| td|j}tg||j|j\}}|jitj|tl|}tn|\}}| rtq|| | ||j\}}n ts|jtdd|i|g}d}t+|jd |jj?d |jtwty|jjd}|j<r||d<|jj%}|j&d|_ t)t+|jtjz|j|}t/|d 5}tEt}j|jD cic]\} }| tOjP|c}} }t|dk(r(tYt[|}|j||d<dddj9| |} |"t| dk(rtYt[| }t||j|| t| ||S#t $rt#}YwxYw#1swY]xYw#t:$rYlwxYwcc} w#tJjR$rd} YwxYwcc}} w#1swYxYw#t:$rYwxYw)Ndatasetsendpointtoken library_namelibrary_version user_agentdatasetrepo_idfilename repo_typerevisionproxieszDownloading standalone yaml)r:rrrzhf://datasets/@rrr5r*r1r)rrr)rrrr)rrr)rr7rMzDownloading metadatarrrrr\)Drr HF_ENDPOINTrr1rr>r4hf_hub_downloadrSrr*r;r rr rrrg download_descr=rArr r r rrrFileNotFoundErrorrrstriprErr1USE_PARQUET_EXPORTr+r;get_exported_dataset_infosr0rDatasetViewerErrorrr"rrrr!rrrrrr6r7r8rrzrr3rrrrprrr|r)rPapidataset_readme_pathrrrrrrrrRrexported_dataset_infosrrrrrrvrr]rLrdataset_infos_pathr r!r"s rUrz"HubDatasetModuleFactory.get_module7s''&&,,#'.t/C/C/N/NO   2"%"5"5 11#)),,44 #6# !, 0 01D E J J ..335  ( ( 0,IO ) #.tyy&*B*BTM]M]^ /$ *W= S'+~~affh'?$'.?.G.G.I++223GH(7(R:Q(R%  S%TYYKq1A1A0B!DMMDWUWCXY``ade *AABST(??@QR  $ $)H)H .)8)S)S II43C3C4K_K_KeKe*&*:,B'$[%:%:;QR];^%__*&&* " ! " ) )- 82M ?? &(9H dmm TRbRiRiRkMlHm8m(d3C3J3J3L.M)N|)\]H(DDXDXYH"00 5 00  /J! 00/ + +  &&,[9FdepFq' 4K@ Q 3_ #'= $ 4 4 4 0O0D!+.CC),4O #' ' 2@P@PQXXY\]yy24 ?3G3GH  ==+5N< (..335  ( ( 0,BO ) !,tyy&*K*KVZVfVfg /" (7; cq'7?Ciil>P>P>R:K):$[%:%:;L%MM($ +,1)-d3G.H)I&6J6N6NOa6b(3 c ! ' ' 60M  &3}+=+B"&tM':";      ''?!1 /$7(  a" 2 / 1  2 S S !    #55 .)-& .@ c c!   sA+W .AW06AW#W0A X X<XAX3,X' X! &>X'$X3 W W #W-(W00 W=<W=XXX!X''X0,X33 X?>X?)NNNNF)rXrYrZr[rr r rrr%r&rrr|rr\rWrUr)r)s#'7;48<@+0"""3- " U3d?34 " ".1 " lC&7 89"%)"&L ML rWr)c:eZdZdZ d dededeefdZdefdZ y) (HubDatasetModuleFactoryWithParquetExportzg Get the module of a dataset loaded from parquet files of a dataset repository parquet export. NrSr*rc\||_||_|xs t|_t |yr)rSr*r%rr)rPrSr*rs rUrz1HubDatasetModuleFactoryWithParquetExport.__init__s+  &.B.2BD!rWr^c tj|j|j|jj }tj |j|j|jj }t|Dcic]}|tj||c}}ttj|jj dtt|jjj!|jd|jj dj"}t%j&|||}t(d\}}t+|||j \} } |jt-t/|jjd } t1||j| |t3|| |  Scc}w) Nr=r.r/zrefs/convert/parquetY@)r:r1r)parquet_commit_hashexported_parquet_filesrparquetrr7rMrr)r;get_exported_parquet_filesrSr*rr1rDr1r0rrrr>rr>r4 dataset_infosharE._from_exported_parquet_files_and_dataset_infosr8rr3rr|r) rPrPrHrrrOrRrvrr]rLrs rUrz3HubDatasetModuleFactoryWithParquetExport.get_modules!0!K!KII4+;+;4CWCWC]C]" "1!K!KII4+;+;4CWCWC]C]" )$: [223I+3VWW   ++**00' +243G3G3R3RS  \ /**00  S  +YY 3#9'  4I> Q/[   000 ,, yy24 ?3G3GH       ''?!1 /$7(  K s Gr) rXrYrZr[rr r%rr|rr\rWrUrKrKs>59 " " "".1 "7 M7 rWrKc6eZdZdZ ddedeefdZdefdZy) CachedDatasetModuleFactoryzS Get the module of a dataset that has been loaded once already and cached. NrS cache_dirc`||_||_|jjddksJy)Nrr)rSrYr)rPrSrYs rUrz#CachedDatasetModuleFactory.__init__s-  "yys#q(((rWr^cNtjjt|jxst j }|jjd}t|d|d<dj|}tjj||}tjtjj|dddDcgc]#}tjj|r|%}}|rx|j|jjddd}d|jd}t jr|dz }tj|t!d d i|d d iSt#d |jd |jcc}w)Nr___*rRz5Using the latest cached version of the dataset since z* couldn't be found on the Hugging Face Hubz (offline mode is enabled).z%datasets.packaged_modules.cache.cacheautoversionzDataset z is not cached in )rrrrrYrHF_DATASETS_CACHErSrr3rglobisdirrrrr|rA) rPrYnamespace_and_dataset_namecached_relative_path#cached_datasets_directory_path_rootcached_directory_pathcached_directory_pathsr warning_msgs rUrz%CachedDatasetModuleFactory.get_modulesGG&&s4>>+UV=U=U'VW %)YY__S%9")?@Z[]@^)_"2&$zz*DE.0ggll9FZ.[+*.277<<@cehjmor3s)t" %ww}}23 "" " "99 $  4R 8NRRVR[R[Q\]GHK$$<< NN; ' 75>59f5   (499+5GGW XYY'" s(F"r) rXrYrZr[rr rr|rr\rWrUrXrXs5$())C=)ZMZrWrXr:rrrYc | td*i|}t|xstj}d|_d|_|tj k(|_ttd|jtjdjdd}|jds|dz}tjj||} |t vrt#|||||j%S|j|rt'd|tjj)| rt'd|tjj+|rt-|||| j%St/|r|j1dd kr t3t4j6|j8d t:t=|j> } tA| jC|t4jDd ||jF} tjjItjjK| } | jC||d | |jFt'd|t}d%t|d)#tL$rz} tO| jPtRtTjVjXtTjVjZfr't[d|d| j\j^d| d} ~ wt`$r!| jc||djd} YtRtTjVjXtTjVjZf$r,} t[d|d| j\j^d| d} ~ wtf$rX} d|d}| jhjjdk(r|dz }n"| jhjjdk(r |d|dz }tm|| d} ~ wtn$r} tmd|d|d| d} ~ wtp$r} tmd|d| d} ~ wwxYw#t`$r3|s |s|r|dk7rd }nd}ts|| |||||!j%cYStf$rX} d|d}| jhjjdk(r|dz }n"| jhjjdk(r |d|dz }tm|| d} ~ wtn$r} tmd|d|d| d} ~ wwxYw#tt$r} tw||"j%cYd}~S#tt$rtO|tRrt[d#|d$|dtO|txtltzfr|dtO|t|r4t}d%t|d&|d't|j^d(|d|dwxYwd}~wwxYw)+aX Download/extract/cache a dataset module. Dataset codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks). Args: path (str): Path or name of the dataset. Depending on ``path``, the dataset builder that is used comes from one of the generic dataset builders (JSON, CSV, Parquet, text etc.). For local datasets: - if ``path`` is a local directory (containing data files only) -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory e.g. ``'./path/to/directory/with/my/csv/data'``. For datasets on the Hugging Face Hub (list all available datasets with ``huggingface_hub.list_datasets()``) - if ``path`` is a dataset repository on the HF hub (containing data files only) -> load a generic dataset builder (csv, text etc.) based on the content of the repository e.g. ``'username/dataset_name'``, a dataset repository on the HF hub containing your data files. revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset to load. As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. data_dir (:obj:`str`, optional): Directory with the data files. Used only if `data_files` is not specified, in which case it's equal to pass `os.path.join(data_dir, "**")` as `data_files`. data_files (:obj:`Union[Dict, List, str]`, optional): Defining the data_files of the dataset configuration. cache_dir (`str`, *optional*): Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. Returns: DatasetModule NTc|Srr\)xs rUz(dataset_module_factory..tsQrWrr\r)rrrrz3Dataset scripts are no longer supported, but found )rrrrr.r/r5r6zCouldn't reach 'z' on the Hub ()rN)r:rz Dataset 'z ' is a gated dataset on the Hub.iz( You must be authenticated to access it.iz; Visit the dataset page at https://huggingface.co/datasets/z to ask for access.z Revision 'z' doesn't exist for dataset 'z ' on the Hub.z1' doesn't exist on the Hub or cannot be accessed.mainF)r*rrrrr+)rYz1Couldn't reach the Hugging Face Hub for dataset 'z': zCouldn't find any data file at z. Couldn't find 'z"' on the Hugging Face Hub either: z: rr\)Ar%r&REUSE_DATASET_IF_EXISTSextract_compressed_file force_extractFORCE_REDOWNLOADforce_downloadrrreplacerseprrrrr8r%r RuntimeErrorrrcrr?rrrr>r1rr>r4r<r?rr;basenamedirnamer isinstance __cause__rrequests exceptionsTimeoutConnectionErrorrOrXrrTrUrresponse status_coder,rrr)rrXr+r rAr@type)rr:rrrrrYdownload_kwargsr8 combined_pathrFrGr*rmessager+e1s rUdataset_module_factoryr:sf(;?; !V,2V2VWM.2O+$(O!%2l6S6S%SO"F; RVVS(A(G(G(LMNrRH   U #e#GGLLx0M$ ))+ !+'  *,   x PQYPZ[\\  &PQYPZ[\\ t ( 8 - *,  $ DJJsOq$8k #++%++' +2?3M3MN C. w13&)&9&9 #55'%+33 ':'#!gg..rww?R/ST L" ## %'(+33 $#%XYaXb#cddX "AB[\`BaAbbc dees+ KK, ++33 ++;;*,T;Q;'N"" Q.AP Q P  Q,P<<QT9S>=T?S>AS S>&S99S>>T W T+%W+BV<<V??Wfeaturesr1storage_optionsc  t|xstj}| #|r|jn t}| |_| 7|r|jn t}|j j | | t|}t|||||||} | j} | jd|}| jd|}| jd|xs| jj}| jdd}| jr| jj|nd}|tvrd|b| jj dj"?d|d }t$Dcgc]}t$||k(s|}}|r |d |dd z }t'|t)| | }|d|||||| j*||| | d | | }|j-| |Scc}w)aLoad a dataset builder which can be used to: - Inspect general information that is required to build a dataset (cache directory, config, dataset info, features, data files, etc.) - Download and prepare the dataset as Arrow files in the cache - Get a streaming dataset without downloading or caching anything You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`]. A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.) Args: path (`str`): Path or name of the dataset. - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`]) -> load the dataset builder from supported files in the repository (csv, json, parquet, etc.) e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files. - if `path` is a local directory -> load the dataset builder from supported files in the directory (csv, json, parquet, etc.) e.g. `'./path/to/directory/with/my/csv/data'`. - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder") -> load the dataset builder from the files in `data_files` or `data_dir` e.g. `'parquet'`. name (`str`, *optional*): Defining the name of the dataset configuration. data_dir (`str`, *optional*): Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`, the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory. data_files (`str` or `Sequence` or `Mapping`, *optional*): Path(s) to source data file(s). cache_dir (`str`, *optional*): Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. features ([`Features`], *optional*): Set the features type to use for this dataset. download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): Download/generate mode. revision ([`Version`] or `str`, *optional*): Version of the dataset to load. As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. token (`str` or `bool`, *optional*): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. If `True`, or not specified, will get token from `"~/.huggingface"`. storage_options (`dict`, *optional*, defaults to `None`): **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any. **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the [`BuilderConfig`] and used in the [`DatasetBuilder`]. Returns: [`DatasetBuilder`] Example: ```py >>> from datasets import load_dataset_builder >>> ds_builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes') >>> ds_builder.info.features {'label': ClassLabel(names=['neg', 'pos']), 'text': Value('string')} ``` N)r:rrrrrYrrrrMrz@Please specify the data files or data directory to load for the z dataset builder.z9 For example `data_files={"train": "path/to/data/train/*.z"}`)rM) rYrMrrrrinforr1rr\)r&rprgr%r1rrr.rrrr~rLrrr8r]rr5rrr!_use_legacy_cache_dir_if_possible)rrSrrrYrrrr:r1r config_kwargsr{rrrMr error_msg extensionexample_extensionsrQbuilder_instances rUload_dataset_builderr s,n!!V,2V2VWM 4C/..0IY %"4C/..0IY''..?8B+ '#N$22N!!*h7H##L*=J $$t\~HH\\K"%%nd;L ' ' + +K 8^bD **    5 5 E Ea H S S [VW[V\\mn '; #?ST]?^bf?fI    VWijkWlVmmqr rI##+NVK'2 (!  ' (  (  (66~F 3 s +G=Grverification_modekeep_in_memory save_infos streamingnum_procc d|vr*|jdrtjd|d||std|dt |t j jr td|r | tdt|xstj}t| s| xs tjntj} td ||||||||| | |d |}|r|j| S|j!||| || | | nt#|j$j&} |j)|| | }|S) a Load a dataset from the Hugging Face Hub, or a local dataset. You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`]. A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.) This function does the following under the hood: 1. Load a dataset builder: * Find the most common data format in the dataset and pick its associated builder (JSON, CSV, Parquet, Webdataset, ImageFolder, AudioFolder, etc.) * Find which file goes into which split (e.g. train/test) based on file and directory names or on the YAML configuration * It is also possible to specify `data_files` manually, and which dataset builder to use (e.g. "parquet"). 2. Run the dataset builder: In the general case: * Download the data files from the dataset if they are not already available locally or cached. * Process and cache the dataset in typed Arrow tables for caching. Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types. They can be directly accessed from disk, loaded in RAM or even streamed over the web. In the streaming case: * Don't download or cache anything. Instead, the dataset is lazily loaded and will be streamed on-the-fly when iterating on it. 3. Return a dataset built from the requested splits in `split` (default: all). Args: path (`str`): Path or name of the dataset. - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`]) -> load the dataset from supported files in the repository (csv, json, parquet, etc.) e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files. - if `path` is a local directory -> load the dataset from supported files in the directory (csv, json, parquet, etc.) e.g. `'./path/to/directory/with/my/csv/data'`. - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder") -> load the dataset from the files in `data_files` or `data_dir` e.g. `'parquet'`. name (`str`, *optional*): Defining the name of the dataset configuration. data_dir (`str`, *optional*): Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`, the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory. data_files (`str` or `Sequence` or `Mapping`, *optional*): Path(s) to source data file(s). split (`Split` or `str`): Which split of the data to load. If `None`, will return a `dict` with all splits (typically `datasets.Split.TRAIN` and `datasets.Split.TEST`). If given, will return a single Dataset. Splits can be combined and specified like in tensorflow-datasets. cache_dir (`str`, *optional*): Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. features (`Features`, *optional*): Set the features type to use for this dataset. download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): Download/generate mode. verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...). keep_in_memory (`bool`, defaults to `None`): Whether to copy the dataset in-memory. If `None`, the dataset will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the [improve performance](../cache#improve-performance) section. revision ([`Version`] or `str`, *optional*): Version of the dataset to load. As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. token (`str` or `bool`, *optional*): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. If `True`, or not specified, will get token from `"~/.huggingface"`. streaming (`bool`, defaults to `False`): If set to `True`, don't download the data files. Instead, it streams the data progressively while iterating on the dataset. An [`IterableDataset`] or [`IterableDatasetDict`] is returned instead in this case. Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example. Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats like rar and xz are not yet supported. The tgz format doesn't allow streaming. num_proc (`int`, *optional*, defaults to `None`): Number of processes when downloading and generating the dataset locally. Multiprocessing is disabled by default. storage_options (`dict`, *optional*, defaults to `None`): **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any. **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the `BuilderConfig` and used in the [`DatasetBuilder`]. Returns: [`Dataset`] or [`DatasetDict`]: - if `split` is not `None`: the dataset requested, - if `split` is `None`, a [`~datasets.DatasetDict`] with each split. or [`IterableDataset`] or [`IterableDatasetDict`]: if `streaming=True` - if `split` is not `None`, the dataset is requested - if `split` is `None`, a [`~datasets.streaming.IterableDatasetDict`] with each split. Example: Load a dataset from the Hugging Face Hub: ```py >>> from datasets import load_dataset >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train') # Load a subset or dataset configuration (here 'sst2') >>> from datasets import load_dataset >>> ds = load_dataset('nyu-mll/glue', 'sst2', split='train') # Manual mapping of data files to splits >>> data_files = {'train': 'train.csv', 'test': 'test.csv'} >>> ds = load_dataset('namespace/your_dataset_name', data_files=data_files) # Manual selection of a directory to load >>> ds = load_dataset('namespace/your_dataset_name', data_dir='folder_name') ``` Load a local dataset: ```py # Load a CSV file >>> from datasets import load_dataset >>> ds = load_dataset('csv', data_files='path/to/local/my_dataset.csv') # Load a JSON file >>> from datasets import load_dataset >>> ds = load_dataset('json', data_files='path/to/local/my_dataset.json') ``` Load an [`~datasets.IterableDataset`]: ```py >>> from datasets import load_dataset >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train', streaming=True) ``` Load an image dataset with the `ImageFolder` dataset builder: ```py >>> from datasets import load_dataset >>> ds = load_dataset('imagefolder', data_dir='/path/to/images', split='train') ``` trust_remote_codezZ`trust_remote_code` is not supported anymore. Please check that the Hugging Face dataset 'z' isn't based on a loading script and remove `trust_remote_code`. If the dataset is based on a loading script, please ask the dataset author to remove it and convert it to a standard format like Parquet.zEmpty 'data_files': 'z3'. It should be either non-empty or None (default).zjYou are trying to load a dataset that was saved using `save_to_disk`. Please use `load_from_disk` instead.zLoading a streaming dataset in parallel with `num_proc` is not implemented. To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead.) rrSrrrYrrrr:r1r)r)rrrrr)rr in_memoryr\)rrerrorrrrDATASET_STATE_JSON_FILENAMEr rr&rprB BASIC_CHECKS ALL_CHECKSras_streaming_datasetdownload_and_preparerCr dataset_size as_dataset)rrSrrrrYrrrrrrr:r1rrrrrdss rU load_datasetrshm+   0 1 LL??CfE\\  j0  3  X)! |  !!V,2V2VWM(DN  ;.;;TdToTo ,   '#'    4454AA))'#+' *)4:JK[K`K`KmKm:n  $ $5DUao $ pB IrW dataset_pathct|fi|xsi^}}|j|std|d|jt j |t jrK|jt j |t jrtj|||S|jt j |t jrtj|||Std|d)aJ Loads a dataset that was previously saved using [`~Dataset.save_to_disk`] from a dataset directory, or from a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`. Args: dataset_path (`path-like`): Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3://my-bucket/dataset/train"`) of the [`Dataset`] or [`DatasetDict`] directory where the dataset/dataset-dict will be loaded from. keep_in_memory (`bool`, defaults to `None`): Whether to copy the dataset in-memory. If `None`, the dataset will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the [improve performance](../cache#improve-performance) section. storage_options (`dict`, *optional*): Key/value pairs to be passed on to the file-system backend, if any. Returns: [`Dataset`] or [`DatasetDict`]: - If `dataset_path` is a path of a dataset directory: the dataset requested. - If `dataset_path` is a path of a dataset dict directory, a [`DatasetDict`] with each split. Example: ```py >>> from datasets import load_from_disk >>> ds = load_from_disk('path/to/dataset/directory') ``` z Directory z not found)rrz@ is neither a `Dataset` directory nor a `DatasetDict` directory.) r r rAr posixpathrrDATASET_INFO_FILENAMErrload_from_diskDATASETDICT_JSON_FILENAMEr#)rrrfsrs rUrrsF| ?(=2 ?FB 99\ "*\N* EFF yy f.J.JKLQSQZQZ|V%G%GHR%%l>crss 9>>,0P0PQ R)),~gvww&f g  rWr)NNr#)NNNNNN) NNNNNNNNNN)NNNNNNNNNNFNNFNN)r[rbrnrqrrr collectionsrcollections.abcrr dataclassesrrpathlibrtypingr r r fsspecr|r  fsspec.corer huggingface_hubr rrhuggingface_hub.utilsrrrrrrrrrr arrow_datasetrbuilderrrrrrrr r!r" dataset_dictr#r$download.download_configr%download.download_managerr&#download.streaming_download_managerr'r(r)r*r}r+r,rr-features.featuresr. fingerprintr/rr0r1iterable_datasetr2namingr3r4packaged_modulesr5r6r7r8:packaged_modules.folder_based_builder.folder_based_builderr9splitsr:utilsr;utils.file_utilsr<r=r>r?r@ utils.hubrAutils.info_utilsrBrC utils.loggingrDutils.metadatarE utils.typingrF utils.versionrGrXrrkeysrrJrrrNrzrrrrrrrrrr|rrr%r)rKrXrrrrrrr\rWrUrs   -(''  !??""2;43bbDD/-B ["&@%+"" H 72779:fXE$%$n%%$-(%$"#%$ %$  . %$PhtN/C&D$DH#3;C= ." c QU"""5=n5M" 8C=$ "LQU#"#5=n5M# 8C=$ #Nhl//%-c]/LTUcLd/ 8C=$sCx. ()/> $-104 6060%60}60!cN 60 n- 60  4  #$ 60r .. ." 55 5"" i 5i X&@#8&@Re 3e PG /DG T'Z!6'ZX/3048<"BF#Of OfuS'\*+Ofn-OfE,"345 Of sm Of tT3 =>? Of}OfOfh"_c##'048<.2(,&*R R 3-RsmRsHSM73c8TW=FX@Y;Y3ZZ[\ R } R x Rn-RE,"345RuS'\*+R E$)$ %Rd^RRn"_cAE##'048<@D%).2(,"&*#q q 3-qsmqsHSM73c8TW=FX@Y;Y3ZZ[\ q E#ud3ie<= > q } qx qn-qE,"345q &6&; <=qTNqquS'\*+q E$)$ %qq sm!q"d^#q& ;!4o EF'qjfj/ / ,4TN/ T\]aTb/  7K  / rW