
    bi'                     >   U d dl Z d dlZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ  ej                  ej                  dd      d          ZdZd	Zd
Zej(                  j+                  dd      Zedz   ZdZdZ ej4                   ej6                               Zh dZh dZej?                  dh      Z ej?                  dh      Z! ej4                  e jD                  j                  d            Z# ej4                  e jD                  j                  d            Z$ ej4                  e jD                  j                  d            Z% ej4                  e jD                  j                  d            Z& ej4                  e jD                  j                  d            Z'ej(                  j+                  dd      jQ                         Z)ej(                  j+                  dd      jQ                         Z*ej(                  j+                  dd      jQ                         Z+dZ,dZ-e*e v ree)evrae j\                  j_                  d      duZ-e-rS	  ej4                  e jD                  j                  d            Z,eja                  de, d       neje                  d        dZ3e j\                  j_                  d!      duZ4e4rA	  ej4                  e jD                  j                  d!            Z3eja                  d"e3 d       dZ5e j\                  j_                  d#      duZ6e6rA	  ej4                  e jD                  j                  d#            Z5eja                  d$e5 d       dZ7dZ8e)e v re*evre j\                  j_                  d%      duZ8e8r6d&D ]/  Z9	  ej4                  e jD                  j                  e9            Z7 n dZ8e8rNe7jt                  d'k  reje                  d(e7 d)       dZ8n'eje                  d*e7 d       neje                  d+       dZ;dZ<e+e v re j\                  j_                  d,      duxr e j\                  j_                  d-      duZ<e<rS	  ej4                  e jD                  j                  d,            Z;eje                  d.e; d       neje                  d/       e j\                  j_                  d0      duZ=e j\                  j_                  d1      duZ>d2Z?d2Z@e j\                  j_                  d3      duZAe j\                  j_                  d4      duZBe j\                  j_                  d5      duZCe j\                  j_                  d6      duZDe j\                  j_                  d7      duZEe j\                  j_                  d8      duZFe j\                  j_                  d9      duZGd:ZH ej                  d;eH      ZJej                  j                  eJd<      ZMej                  j                   ej                  d=eM            ZOej                  j                  eOd>      ZP e ej                  d?eP            ZQej                  j                  eOd@      ZR e ej                  dAeR            ZSdBZTej                  j                  eQeT      ZU e ej                  dCeU            ZVdDZWej                  j                  eUeW      ZX e ej                  dEeX            ZYej(                  j+                  dFd      jQ                         e v ZZdGZ[d2Z\dHZ]dIdJd dKZ^dLZ_dMZ`dNZadZbdZcdZddZedOZfdOZgdOZhdLZiej(                  j+                  dP      Zjeje
j                  nejjQ                         ev ZkekZlej(                  j+                  dQ      ZmememjQ                         ev ndZneeo   epdQ<   d Zq erej(                  j+                  dReq            ZsdSZtdTZudUZvdVZwdWZxdXZydYZzdZZ{d[Z|d\Z}d]Z~d^Zd_Zd`ZdaZdbZdLZdbZdHZdcZddZdeZy# e jD                  jb                  $ r Y }w xY w# e jD                  jb                  $ r Y ;w xY w# e jD                  jb                  $ r Y w xY w# e jD                  jb                  $ r Y w xY w# e jD                  jb                  $ r Y w xY w)f    N)Path)Optional)	constants)version.   zBhttps://s3.amazonaws.com/datasets.huggingface.co/datasets/datasetsz5https://cdn-datasets.huggingface.co/datasets/datasetszXhttps://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}HF_ENDPOINTzhttps://huggingface.coz-/datasets/{repo_id}/resolve/{revision}/{path}z)hf://datasets/{repo_id}@{revision}/{path}main>   1ONYESTRUE>   0NOOFFFALSEAUTOdillfsspecpandaspyarrowhuggingface_hubUSE_TF	USE_TORCHUSE_JAXzN/AFtorchzPyTorch version z available.z'Disabling PyTorch because USE_TF is setpolarszPolars version duckdbzDuckdb version 
tensorflow)	r   ztensorflow-cpuztensorflow-gpuz
tf-nightlyztf-nightly-cpuztf-nightly-gpuzintel-tensorflowztensorflow-rocmztensorflow-macos   z"TensorFlow found but with version z(. `datasets` requires version 2 minimum.zTensorFlow version z-Disabling Tensorflow because USE_TORCH is setjaxjaxlibzJAX version z-Disabling JAX because USE_JAX is set to False
sqlalchemyPILT
torchcodectorchvision
pdfplumberrarfile	zstandardlz4py7zrz~/.cacheXDG_CACHE_HOMEhuggingfaceHF_HOMEdatasetsHF_DATASETS_CACHEmodulesHF_MODULES_CACHE	downloads$HF_DATASETS_DOWNLOADED_DATASETS_PATH	extracted#HF_DATASETS_EXTRACTED_DATASETS_PATHHF_UPDATE_DOWNLOAD_COUNTS   i  i   i   )min_chunk_sizemax_chunk_size
norm_level
   500MB100MBd   HF_DATASETS_OFFLINE!HF_DATASETS_DISABLE_PROGRESS_BARSHF_DATASETS_IN_MEMORY_MAX_SIZEzdataset.arrowzindices.arrowz
state.jsonzdataset_info.jsonzdataset_infos.jsonLICENSEzdataset_dict.jsonconfigsz	README.mdz.huggingface.yamldatasets_modules   zhf_datasets-         g?2   l        )	importlibimportlib.metadataloggingosplatformpathlibr   typingr   r   r   	packagingr   	getLogger__name__splitloggerS3_DATASETS_BUCKET_PREFIX"CLOUDFRONT_DATASETS_DISTRIB_PREFIXREPO_DATASETS_URLenvirongetr	   HUB_DATASETS_URLHUB_DATASETS_HFFS_URLHUB_DEFAULT_VERSIONparsepython_version
PY_VERSIONENV_VARS_TRUE_VALUESENV_VARS_FALSE_VALUESunionENV_VARS_TRUE_AND_AUTO_VALUESENV_VARS_FALSE_AND_AUTO_VALUESmetadataDILL_VERSIONFSSPEC_VERSIONPANDAS_VERSIONPYARROW_VERSIONHF_HUB_VERSIONupperr   r   r   TORCH_VERSIONTORCH_AVAILABLEutil	find_specdebugPackageNotFoundErrorinfoPOLARS_VERSIONPOLARS_AVAILABLEDUCKDB_VERSIONDUCKDB_AVAILABLE
TF_VERSIONTF_AVAILABLEpackagemajorJAX_VERSIONJAX_AVAILABLESQLALCHEMY_AVAILABLEPIL_AVAILABLEIS_OPUS_SUPPORTEDIS_MP3_SUPPORTEDTORCHCODEC_AVAILABLETORCHVISION_AVAILABLEPDFPLUMBER_AVAILABLERARFILE_AVAILABLEZSTANDARD_AVAILABLELZ4_AVAILABLEPY7ZR_AVAILABLEDEFAULT_XDG_CACHE_HOMEgetenvr,   pathjoinDEFAULT_HF_CACHE_HOME
expanduserHF_CACHE_HOMEDEFAULT_HF_DATASETS_CACHEr0   DEFAULT_HF_MODULES_CACHEr2   DOWNLOADED_DATASETS_DIR DEFAULT_DOWNLOADED_DATASETS_PATHDOWNLOADED_DATASETS_PATHEXTRACTED_DATASETS_DIRDEFAULT_EXTRACTED_DATASETS_PATHEXTRACTED_DATASETS_PATHr7   &HF_DATASETS_MULTITHREADING_MAX_WORKERSUSE_PARQUET_EXPORTDEFAULT_MAX_BATCH_SIZEDEFAULT_CDC_OPTIONS'ARROW_READER_BATCH_SIZE_IN_DATASET_ITERMAX_SHARD_SIZEMAX_ROW_GROUP_SIZE)PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS)PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS*PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS)PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS*ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS*ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS+ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS*ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS_offlineHF_HUB_OFFLINEr@   #__HF_DATASETS_DISABLE_PROGRESS_BARSrA   bool__annotations__DEFAULT_IN_MEMORY_MAX_SIZEfloatIN_MEMORY_MAX_SIZEDATASET_ARROW_FILENAMEDATASET_INDICES_FILENAMEDATASET_STATE_JSON_FILENAMEDATASET_INFO_FILENAMEDATASETDICT_INFOS_FILENAMELICENSE_FILENAMEDATASETDICT_JSON_FILENAMEMETADATA_CONFIGS_FIELDREPOCARD_FILENAMEREPOYAML_FILENAMEMODULE_NAME_FOR_DYNAMIC_MODULES%MAX_DATASET_CONFIG_ID_READABLE_LENGTHTEMP_CACHE_DIR_PREFIXSTREAMING_READ_MAX_RETRIESSTREAMING_READ_RETRY_INTERVAL*DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE2GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE3ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE/MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLELPBAR_REFRESH_TIME_INTERVALUPLOADS_MAX_NUMBER_PER_COMMITMAX_TABLE_NBYTES_FOR_PICKLING     J/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/config.py<module>r      s      	    %  
		8>>#q1!4	5 a %\ "n  jjnn],DE!PP C  W]]282245
 2 3  4 : :F8 D !6!<!<fX!F  w}}Y//77?@y1199(CDy1199(CD'--	 2 2 : :9 EFy1199:KLM	&	)	/	/	1JJNN;/557	
**..F
+
1
1
3--&@T2Tnn..w7tCO	)GMM)*<*<*D*DW*MNMLL+M?+FG KK9:>>++H5TA &y'9'9'A'A('KL~&6kBC
 >>++H5TA &y'9'9'A'A('KL~&6kBC 
	**y@T/T>>++L9EL

 	!G*W]]9+=+=+E+Eg+NO
 !	!$ !LaKK<ZLHpqr LKK-j\EF
KK?@ 
++NN,,U34?rINND\D\]eDfnrDrM	''--	(:(:(B(B5(IJKKK,{m;?@ KK?@ !~~//=TI  ((/t;   ~~//=TI !00?tK  ~~//=TI  NN,,Y7tC nn..{;4G ((/t;..**734? $ +-CD^]C ""9299Y8M#NOGGLL
C #68QRS 77<<yA 			"46NOP % #%77<<0ACZ#[  			*PRr st $ "$'',,/OQg"h yryy)NPopq  JJNN.7==?C`` 
 *, &    )3{bcd  +- '    -1 ),0 )-1 *,0 ) .1 *-0 *.1 +-/ * ::>>/0-5-=))8>>CSWkCk$  ')jjnn5X&Y # +6 (--/3GG	 "8D>   2::>>*JLfgh  ) * * + 1  / "  ' "4 (+ % '     !  .1 *57 269 3 37 / "  !#  !( g !!66 		 22  22 . %%:: 2 !!66 		s^   A _?  A ` "A `? +aA a? ?```<;`<?aaa<;a<?bb