bi+: dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZddlmZmZddlmZmZmZmZddlmZdd lmZdd l m!Z"ddl#Z#ddl$Z$ddl%Z$ddl&Z&dd l'm(Z(m)Z)dd l*m+Z+dd l,m-Z-m.Z.m/Z/ddl0m1Z1ddl2m3Z3m4Z4ddl5m6Z6ddl7m8Z8ddl2m9Z9m:Z:ddl;mZ>ddl?m@Z@ ddlAmBZCe:jeGZHdZIedeJeZKdeJdeLfdZMdeJdeLfd ZNdeJdeLfd!ZOd"eKdeKfd#ZPd$eJd%eJdeJfd&ZQd'eJdeJfd(ZRd|d)ZS d|deJfd*ZTd|d+eeeJeUfdeJfd,ZVd|d-eJd.eeeJeLfdeUfd/ZWd|d0eeJfd1ZXd|d2ZYd3ZZGd4d5e#jjZ\d}d7Z] d~deJfd8Z^d9Z_d:Z`d;ZadZdie8Dcic]/}|jD]}|jd?|j 1c}}d@d@iZhe8Dcic]/}|jD]}|jd?|j 1c}}Zie8Dchc]}|jc}ZjejdAZlemjdBd@emjdCd@emjdDd@emjdEdFemjdGdHemjdIdJemjdKdLemjdMdNiZodOdPiZpeqdQeeoepDZrGdRdSeEZsd"eJdeJfdTZtdeeJfdUZud|dVeJdWee6deeJfdXZvdYZwdZZxd|dVeJdWee6fd[Zyd\Zzd]Z{d^Z|d|dWee6deLfd_Z}d|dWee6de~fd`Zd|dWee6deLfdaZd|dbZdcZ d|dVeJdWee6deeJeUeJeUeJeffffddZ d|dVeJdWee6deeJeUeJeUeJeffffdeZdddfdgeJdWee6fdhZd|d"eJdWee6deeJfdiZd6ddjdWee6fdkZd|dWee6fdlZGdmdneeZd"eeJeeffdoZddfdWee6fdpZddfdWee6fdqZd|dWee6fdrZd|dWee6fdsZd|dWee6fdtZd|dWee6fduZddWee6fdvZd|dWee6fdwZGdxdye@ZGdzd{e@Zy#eD$rGddeEZCYwxYwcc}}wcc}}wcc}w)z Utilities for working with the local dataset cache. This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp Copyright by the AllenNLP authors. N) Generator)BytesIO)chain)Path PurePosixPath)AnyOptionalTypeVarUnion)patch)urlparse) ElementTree)strip_protocol url_to_fs) can_be_local)EntryNotFoundError get_sessioninsecure_hashlib)version) __version__config)DownloadConfig)COMPRESSION_FILESYSTEMS)_tqdmlogging)FileLock)ExtractManager)TrackedIterableFromGenerator) ClientErrorc eZdZy)_AiohttpClientErrorN__name__ __module__ __qualname__T/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/utils/file_utils.pyr#r#5s r)r# .incompleteTurl_or_filenamereturnct|jdk7xr6tjj t|jdz SNz:/r schemeospathismountr-s r* is_remote_urlr8@s? O $ + +r 1 r"''//(SbJcJjJjmqJq:r6rrr)ct|jdk(xs5tjj t|jdzSr0r2r7s r* is_local_pathr:Ds= O $ + +r 1 mRWW__XoE^EeEehlEl5mmr)cvt|jdk(xr tjj | S)Nr1)r r3r4r5isabsr7s r*is_relative_pathr=Ks- O $ + +r 1 X"''--:X6XXr)r5c tjjtjjtjj t |}t |tr t|S|S)z'Convert relative path to absolute path.)r4r5abspath expanduser expandvarsstr isinstancer)r5 abs_path_strs r*relative_to_absolute_pathrEOsO77??277#5#5bgg6H6HT6S#TUL!+D$!74 I\Ir) base_name pathnamesct|rtj|gd|DSt|g|j S)Nc3K|]<}t|jtjdj d>yw/N)rBreplacer4seplstrip).0pathnames r* z#url_or_path_join..Ws2*t^f3x=+@+@+M+T+TUX+Y*tsAA)r8 posixpathjoinras_posix)rFrGs r*url_or_path_joinrUUs>Y~~iu*tjs*tuuI* *3355r) url_or_pathc~t|r|d|jdStjj |S)NrK)r8rindexr4r5dirname)rVs r*url_or_path_parentrZ\s6[!4[//455ww{++r)c|jd}tj|}|j}|r<|jd}tj|}|d|jzz }|j dr|dz }|S)a Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) utf-8.z.py)encodersha256 hexdigestendswith)urletag url_bytesurl_hashfilename etag_bytes etag_hashs r*hash_url_to_filenamerics 7#I&&y1H!!#H [[) $++J7 C)--/// ||EE Or)c Z| tdi|}|jxstj}t |t r t |}t |t r t |}t|r t|}t|rLt||\}}|jdrtjtj|jj!|} tj"tj|jdt$t'|j(j+|j,|j.|j0|j2|j4|j6}ntE|||j4|j(|jF|j||jH|jJ }nKtLjNjQ|r|}n)tS|rtCd |d tUd |d ||S|jVr|jXrt[||}t]|j_d d} |r| dvr|j_d djadsatc|}|tdvrEtLjNjg|} d| vr| d| jidn| } |d| d |}|S|d|}|Stk|jjm||jn}tc|S#tj8j:tj8j<tj8j>tj8j@f$r}tCt ||d}~wwxYw)a Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and return the path to the cached file. If it's already a local path, make sure the file exists and then return the path. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk ValueError: if it couldn't parse the url or filename correctly requests.exceptions.ConnectionError: in case of internet connection issue Ndownload_confighf://)endpointtokendatasets)rnro library_namelibrary_version user_agent)repo_id repo_typerevisionrfforce_downloadproxies) cache_dirrwrsuse_etagrostorage_options download_desc disable_tqdmz Local file z doesn't existzunable to parse z as a URL or as a local path::rtgztarz.tar.gzz.tar.bz2z.tar.xzr]://z://::)ry) force_extractr()8rryrDOWNLOADED_DATASETS_PATHrCrrBrrr8!_prepare_path_and_storage_options startswithhuggingface_hub HfFileSystem HF_ENDPOINTro resolve_pathHfApirget_datasets_user_agentrshf_hub_downloadrtrurv path_in_reporwrxutilsRepositoryNotFoundErrorrRevisionNotFoundErrorGatedRepoErrorFileNotFoundErrorget_from_cacherzr|r}r4r5existsr: ValueErrorextract_compressed_fileextract_on_the_fly_get_extraction_protocol_get_path_extensionsplitrarE!SINGLE_FILE_COMPRESSION_PROTOCOLSbasenamerXrextractr) r-rldownload_kwargsryr{ resolved_path output_patheprotocol extension inner_files r* cached_pathr{sn,(;?;))LV-L-LI)T" N /4(o.O$(9_%+L _, (  % %g .+88++?3H3Hl?+  7-33#//)//!+$/67Q7QR  "/)11+55*33*77#2#A#A+33 " .)#.==*55(11%++ /-;;,99 K  (%  '+o->n MNN+O+<JILPZIZ,Dj.?.?.D!E`jJ%-Jc*R }"MK#"&.JeK="AK""%/H/HIQQ 'D'DR  %[ 11s %%==%%88%%;;%%44  7 (A/Q6  7s$BL''A)N*N%%N*rsc,dt}|dtjz }|dtjz }|dtjz }tj r|dtj z }tjr|dtjz }tjr|dtjz }t|tr.|ddjd |jDz }|St|tr|d|zz }|S) Nz datasets/z ; python/z; huggingface_hub/z ; pyarrow/z; torch/z ; tensorflow/z; jax/z; c30K|]\}}|d|ywrJr()rOkvs r*rQz*get_datasets_user_agent..sGDAqs!A3ZGs)rr PY_VERSIONrPYARROW_VERSIONTORCH_AVAILABLE TORCH_VERSION TF_AVAILABLE TF_VERSION JAX_AVAILABLE JAX_VERSIONrCdictrSitemsrB)rsuas r*rrs [M "BIf''( ))B ::; <8!6B8HB 774=r)cNtjtjddiS)N8HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS1)r rr4environr(r)r*,stack_multiprocessing_download_progress_barsr s  ::bjj#]_b"c ddr)c eZdZdfd ZxZS) TqdmCallbackctjtjdkrt ||g|i|t |_yt j|d<t ||g|i|y)Nz2024.2.0tqdm_cls)rFSSPEC_VERSIONrparsesuper__init__rtqdm)self tqdm_kwargsargskwargs __class__s r*rzTqdmCallback.__init__'s]  7==#< < G [ :4 :6 :DJ!&F:  G [ :4 :6 :r)N)r%r&r'r __classcell__rs@r*rr&s ;;r)rFcptd|t|fi|xsi\}}t|xsdddtjj ddk(r?t jjr!t jjdnd|d }|j||j| y) Nr DownloadingBTrr)descunit unit_scalepositiondisable)r)callback) rrrr4rgetmultiprocessingcurrent_process _identityget_filename)rb temp_filer{rr}rr5rs r* fsspec_getr0s%u&=>8!6B8HB)Mzz~~XY]``//1;;(779CCBG#   HKKinnxK8r)c |i}|tj}t|tr t |}t j |dd} d} t|d} t jj|| } t jj| r|s|s| St||} ||| d<t||} |r&| jddxs| jd dnd} t|| } t jj|| } t jj| r|s| S| d z}t|5t jj| r |s | cdddS| d z}t|d 5}t j#|d |j$t'|||||dddt j#d|d| t)j*j$| t j#d| || d}| dz}t|dd5}t-j.||dddddd| S#1swYxYw#1swYxYw#1swY| SxYw)a Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the path to the cached file. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk NT)exist_ok)rc)roz user-agent)r{ETagrcz.lockr+zw+bzB not found in cache or force_download set to True, downloading to )r{rr}zstoring z in cache at zcreating metadata file for )rbrcz.jsonwr\)encoding)rHF_DATASETS_CACHErCrrBr4makedirsrir5rSrrrrropenloggerrrrshutilmovejsondump)rbryrwrsrzror{r|r}responsercrf cache_pathheaders lock_pathincomplete_pathrmeta meta_path meta_files r*rrBsM0,, )T" N KK D)H D$Cd3Hi2J ww~~j!.1EBG * 3@HIQHLL & D(,,vt*DW[D$C.Hi2J ww~~j!.W$I ) ' 77>>* %n'' %}4/5 ) wY KK3%abkbpbpaqr s sI]iu v w  hse= => INNJ/ 1*>?D)( )S7 3 'y IIdI & ')'.  w w ' ')'. sC #I06I05I.docstring_decorators1WWV_v-rzz?U[]^  r)r(rr s` r*add_start_docstringsr  r)cfd}|S)Ncp|j |jnddzdjz|_|Sr)rrSrs r*r z/add_end_docstrings..docstring_decorators1$&JJ$:bjjfLrwwW]^  r)r(r s` r*add_end_docstringsrr r)c&td|DS)Nc3PK|]}|jj ywr)statst_size)rOr5s r*rQz(estimate_dataset_size..s5ttyy{""5s$&)sum)pathss r*estimate_dataset_sizers 5u5 55r)fct} |jd}|s t|S||z }|jdr t|SB)Nr ) bytearrayreadrabytes)rresbs r*readlinersT +C  FF1I  : q <<   : r))txtcsvrjsonltsvconllconlluorigparquetpklpicklerelxmlarrowr]zipz(? 3#I-OOD)!, - r)c |jd|j t }|jdt t D]W}tj|dt |z }||cStj|dt |z }|Jtd|dy#ttjf$rYywxYw)zQread the magic number from a file-like object and return the compression protocolrNzCompression protocol 'z' not implemented.) seekAttributeErrorioUnsupportedOperationrMAGIC_NUMBER_MAX_LENGTHrange$MAGIC_NUMBER_TO_COMPRESSION_PROTOCOLr0MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOLNotImplementedError)rr>i compressions r**_get_extraction_protocol_with_magic_numberrOs q 6612LFF1I * +`:>>|LiNehiNi?jk  " FJJ{mK]&^_ _ ` B33 4sB##B?>B?urlpathrlct|}|jdd}t|}|tvs|dvs|j dry|t vr t |St ||\}} tj|fi|xsi5}t|cdddS#1swYyxYw#t$r0|jtjrt|dzdwxYw)Nr~rrrrkS If the repo is private or gated, make sure to log in with `huggingface-cli login`.)rBrrBASE_KNOWN_EXTENSIONSra!COMPRESSION_EXTENSION_TO_PROTOCOLrfsspecrrOrrrr)rPrlr5rr{rs r*rrs'lG == q !D#D)I**  & ==; < 7 70;;@ZijG_  [[ <_%: < A=a@ A A A    f00 1#pp   s*+B& B B&B#B&#B&&9Cct|jd^}}t|r tjj |g|St j |g|}dj |g|zS)u# This function extends os.path.join to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xjoin function allows you to apply the join on the first path of the chain. Example:: >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt") zip://folder1/file.txt::https://host.com/archive.zip r~)rBrr:r4r5rSrR)aprs r*xjoinrY.sb& FLL EAQww||A""" NN1 !q !yy!q!!r)c>t|jd^}}t|r7tjj t |j}ntj |}|jdr|dz }dj|g|zS)u# This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xdirname function allows you to apply the dirname on the first path of the chain. Example:: >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip") zip://folder1::https://host.com/archive.zip r~://) rBrr:r4r5rYrrTrRrarSrWrs r*xdirnamer^Is|& FLL EAQ GGOODG,,. /   a  zz# T  99aS1W r)ct|jd^}}t|rtjj |St ||\}}|jd^}}t|fi|^}}|j |S)aExtend `os.path.exists` function to support both local and remote files. Args: urlpath (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r~rk)_as_strrr:r4r5rrr)rPrlmain_hop rest_hopsr{r_s r*xexistsrdhs#7+11$7HyXww~~h''#DW^m#n &}}T2976o6Qyy""r)ct|jd^}}t|r6tjj t |jStj |S)u This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xbasename function allows you to apply the basename on the first path of the chain. Example:: >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip") file.txt r~) rBrr:r4r5rrrTrRr]s r* xbasenamerf}sU& FLL EAQwwQ 0 0 233!!!$$r)cFt|jd^}}t|r6tjjt |j Stj|\}}dj|jdr|dzn|g|z|fS)u, This function extends os.path.split to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplit function allows you to apply the xsplit on the first path of the chain. Example:: >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1::https://host.com/archive.zip', 'file.txt') r~r[r\) rBrr:r4r5rrTrRrSra)rWrtails r*xsplitris& FLL EAQww}}T!W--/00//!$4yyajjo!d(1=ABDHHr)ct|jd^}}t|r6tjj t |jStj |\}}dj|g|z|fS)u8 This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplitext function allows you to apply the splitext on the first path of the chain. Example:: >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1/file::https://host.com/archive.zip', '.txt') r~) rBrr:r4r5splitextrrTrRrS)rWrexts r* xsplitextrmsr& FLL EAQwwQ 0 0 233##A&3yy!q!3&&r)ct|jd^}}t|rtjj |St ||\}}|jd^}}t|fi|^}}|j |S)zExtend `os.path.isfile` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r~rk)rBrr:r4r5isfilerr)r5rlrarbr{rrcs r*xisfilerps}t9??40HyXww~~d## A$Xg ho#zz$/943?3Qyy""r)ct|jd^}}t|rtjj |St ||\}}|jd^}}t|fi|x^}}^}} |j|}|1t||5}t|j}ddd|S|S#t$rtd|wxYw#1swY|SxYw)zExtend `os.path.getsize` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `int`: optional r~rkzNo such file: N)rBrr:r4r5getsizerrsizerrxopenr=r) r5rlrarbr{rrcrsrs r*xgetsizerust9??40HyXwwt$$ A$Xg ho#zz$/9#Drwwxu5]BGGOOT\D]]OTy  U1A1A$1G1JKsZ\ZaZaZiZijrZssr)c|jtjfd} ||_|S#t$r+|t j }||_fd|_Y|SwxYw)Nc d}tddzD] } |i|}|St d|#ttjtj j tj jf$r\}|}tjdtjd|ddtjtjYd}~d}~wwxYw)Nrz4Got disconnected from remote data host. Retrying in zsec [rK]zServer Disconnected)rIr#asyncio TimeoutErrorrequests exceptionsConnectionErrorTimeoutrwarningrSTREAMING_READ_RETRY_INTERVALtimesleep)rrdisconnect_errretryouterr max_retriesrs r*read_with_retriesz?_add_retries_to_file_obj_read_method..read_with_retries)s1kAo. ME AD+F+ # M ""78n L$$$##33##++  A "%J6KoKoJppuv{u||}J~KKLM 6??@@ As/A C:ACCct|Sr)getattr)rcattr orig_file_objs r*z6_add_retries_to_file_obj_read_method..Dsw}d/Kr))rrSTREAMING_READ_MAX_RETRIESrErF RawIOBase __getattr__)file_objrrrrs @@@r*$_add_retries_to_file_obj_read_methodr%sg ==D33K*L) O L <<>) K O Ls00A$#A$cg}i}|jdD]4}t||\}}|j||j|6dj |fS)Nr~rk)r,_prepare_single_hop_path_and_storage_optionsappendupdaterS)rPrlprepared_urlpathprepared_storage_optionshopr{s r*rrHsm!}}T"9KCapq_$ ''89 99% & 77r)c8|dn |j}|jtjr7d|vr3d|t tjdzdj dddz}d|vr|j ddnd}|,||jvr|j|j}nV|R||jvrD|jjDcic]\}}|tjvr|| }}}ni}|d vr|jd i}d d i||d <d |vrrtj|d}|jjD]1\} } | jds|d| zz }|j} d| i|}3d|vr|dz }|jdra|jdi} ddi| |d<nE|dk(r@|tjd|}tj t#j$dkrd|d<|r||i}||fScc}}w)aT Prepare the URL and the kwargs that must be passed to the HttpFileSystem or HfFileSystem In particular it resolves google drive URLs It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths. Storage options are formatted in the form {protocol: storage_options_for_protocol} Nz /resolve/rmr@rrfile>httphttps client_kwargs trust_envTzdrive.google.com )timeoutdownload_warningz &confirm=cookieszconfirm=z &confirm=tz"https://raw.githubusercontent.com/rzAccept-Encodingidentityhf)rornz0.21.0default block_size)rorrrr=rLrr{copyrrUavailable_protocolspoprheadrHF_HUB_VERSIONrr) rPrlrorr{ option_name option_valuerrrrrrs r*rrTsd$+D1F1FE&,,-+2HGC(:(:$;a$?$ABJJ;X[]^__*/7*:w}}U#A&H"x?3R3R'R)99(CHHJ  $9X9X)X.=-L-L-R-R-T ) \&"<"<">>  %  $$'++OR@ ,7+O+O(  ("}))'2)>H ((..0 N1<< 23{Q.G&..G'0'&M_&MO  N (<'   B C%)))R8G*;Z)S7)SOI & T **    7==#: :,5OL )#_5 O ##G s#Hrkrct|}|jd^}}t|r"|jddt ||g|i|St ||\}}i||xsi} t j|g|d|i|j } t| } | S#t$r } t| dk(r td| d} ~ wt$r0|jtjrt|dzdwxYw) aExtend `open` function to support remote files using `fsspec`. It also has a retry mechanism in case connection fails. The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co Args: file (`str`): Path name of the file to be opened. mode (`str`, *optional*, default "r"): Mode in which the file is opened. *args: Arguments to be passed to `fsspec.open`. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Keyword arguments to be passed to `fsspec.open`. Returns: file object r~rNrkmodezCannot seek streaming HTTP filezStreaming is not possible for this dataset because data host server doesn't support HTTP range requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)rR)r`rr:rrrrUrrBr@rrrrr) rrrlrrfile_strrarbr{rrs r*rtrts#"t}H#>>$/HyX <&Hd4T4V44=hXghD/ 2 2?0b 2F;;t@@$@@EEG 4H=H O!  q66 6+y    ??6-- .#mm   s')B C=&C2;;BKKPRO\ijL hZs<.9:YFGjjjs!C2c+NKt|jd^}}t|rtj|fi|Ed{yt ||\}}|jd^}}t |fi|^}}|jdd}|jdr|j|sgSt|jtr |jn|jd} |j|fi|D]%\} } } dj| d| g|z| | f'y7߭w)auExtend `os.walk` function to support remote files. Args: urlpath (`str`): URL root path. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Additional keyword arguments forwarded to the underlying filesystem. Yields: `tuple`: 3-tuple (dirpath, dirnames, filenames). r~NrkrrrK) r`rr:r4walkrrrxrwrCrrBrS) rPrlrrarbr{rrcryrdirpathdirnames filenamess r*xwalkrs%#7+11$7HyX778.v...$EW^m#n &}}T2976o6Q^^E*2.   C *)=I",R[[#">2;;BKKPRO,3BGGJ,I&,I Z (GXy))zWI67)CDhPYY Y Z /sAD%D#C D%ceZdZdZfdZddeefdZddeefdZdZ e ddZ e de fd Z e de fd Ze de fd Zd Zd ee dfddfdZd e ddfdZfdZxZS)xPathzHExtension of `pathlib.Path` to support both local paths and remote URLs.ct|}|jd^}}t|r|S|j dd}t j d|}||jdrdz }|Sdz }|S)Nr~\rKrr[r\r1)r__str__rr:rL#SINGLE_SLASH_AFTER_PROTOCOL_PATTERNsubra)rpath_strrarb path_as_posixrs r*rz xPath.__str__ s7?$'~~d39  "O ((s3 ;??}U !7!7!<D CED r)rlc.tt||S)zExtend `pathlib.Path.exists` method to support both local and remote files. Args: download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` rk)rdrB)rrls r*rz xPath.existsss4y/BBr)c#(K|j}|jd^}}t|r#t|j |Ed{y|rD|d}t ||\}}|jdd|i}dj ||g|dd}nd}tt||fi|xsi^}} |j t||} | D]7} t|dj |jd| g|z9y7w)a]Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. download_config : mainly use token or storage_options to support different platforms and auth types. Yields: [`xPath`] r~Nrrkrr) rTrr:rrrrSrrYtyper) rpatternrl posix_pathrarbrPr{rrcrrs r*rz xPath.glob"s ]]_ )//59  "H~**73 3 3#A,+LWfu+v(#*==#7#:O"L!YY''JIabM'JK "&uZ9Uo>SQSUFBGGE(G$<=M - ]  d4jr{{m3|n,M+NQZ+Z![\\ ] 4sA DDCDc .|jd|zfi|S)zRglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. Yields: [`xPath`] z**/)r)rrrs r*rglobz xPath.rglob>styy3F33r)r.cRt|t|jS)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: [`xPath`] )rr^rTrs r*parentz xPath.parentIs tDz(4==?344r)clt|jjddjS)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r~r)rrTrrrs r*rz xPath.nameR+T]]_2248;<AAAr)clt|jjddjS)zStem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r~r)rrTrstemrs r*rz xPath.stem[rr)clt|jjddjS)zSuffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r~r)rrTrsuffixrs r*rz xPath.suffixds+T]]_2248;<CCCr)c2tt|g|i|S)aExtend :func:`xopen` to support argument of type :obj:`~pathlib.Path`. Args: **args: Arguments passed to :func:`fsspec.open`. **kwargs: Keyword arguments passed to :func:`fsspec.open`. Returns: `io.FileIO`: File-like object. )rtrB)rrrs r*rz xPath.openmsSY0000r)rX.cTt|t|jg|S)zExtend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`. Args: *p (`tuple` of `str`): Other path components. Returns: [`xPath`] )rrYrTrrXs r*joinpathzxPath.joinpathys%tDz% 4!455r)c$|j|Sr)rrs r* __truediv__zxPath.__truediv__s}}Qr)c Rt|jd^}}t|r't|tt||St|dj t|t|j |jg|zS)Nr~) rBrr:rr with_suffixrSrrT)rrrarbrs r*rzxPath.with_suffixs"4yt49  "4:c%'"5f"=>? ?tDz$))ZT$Z h0G0S0STZ0[%\%e%e%g$hkt$tuvvr)r)r.r)r%r&r'rrr rrrrpropertyrrBrrrrtuplerrrrrs@r*rr sR Ch~&> C]Xn-E]8 455BcBBBcBBDDD 1 65c? 6w 6 S W wwr)rcrt|tr t|Sttt|Sr)rCrrB)r5s r*r`r`s("4/3t9JSs4y9I5JJr)cddl}t|dr|j|g|i|St|}|jt |d|g|i|SNrrrbrk)r4hasattrrrBrt)filepath_or_bufferrlrrr4s r* xgzip_openrs_!6*tyy+=d=f== !34tyy14Yk\`kdjkkr)cddl}t|dr|j|g|i|St|}|jt |d|g|i|Sr)numpyrloadrBrt)rrlrrnps r* xnumpy_loadrs_!6*rww);D;F;; !34rwwu/WiZ^ibhiir)c ddl}t|dr|j|fi|St|}|j dddk(rt |||d<|jt |d|fi|S)NrrrNinferrkr)pandasrread_csvrBrrrtrrlrpds r*xpandas_read_csvrsz!6*r{{-888 !34 ::mW - 8$<=Oap$qF= !r{{5!3T?[f_effr)c ddl}t|dr |j|fi|St |} |jt|d|fi|S#t$r-|jt |j fi|cYSwxYw#t$r9|jt t|d|j fi|cYSwxYwr)rr read_excelrrrrBrtrs r*xpandas_read_excelrs!6* O 2==!3>v> >!!34  2=='94Q`!alekl l  O 2==);)@)@)B!CNvN N O   2==0$X]]_`dj  s"AB 3BB ?C  C c ddlm}t|dr|j|fi|St |}|jt |d|fi|S)Nrrr)rrl)pyarrow.parquetr'r read_tablerBrt)rrlrpqs r*xpyarrow_parquet_read_tabler sU !6*r}}/:6:: !34r}}U#5DRabmflmmr)c ddlm}t|dr|j|fi|S|jt |d|fi|Sr)scipy.iorFrloadmatrt)rrlrsios r* xsio_loadmatrsH!6*s{{-888s{{5!3T?[f_effr)ct|drtj||St|d|5}tj||cdddS#1swYyxYw)aExtend `xml.etree.ElementTree.parse` function to support remote files. Args: source: File path or file object. parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `xml.etree.ElementTree.Element`: Root element of the given source document. r)parserrrkN)rETrrt)sourcerrlrs r* xet_parsersPvvxxv.. 64 A .Q88Af- . . .s AAc t|dr*tjjj|fi|St |d|5}tjjj|fi|cdddS#1swYyxYw)aExtend `xml.dom.minidom.parse` function to support remote files. Args: filename_or_file (`str` or file): File path or file object. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`. Returns: :obj:`xml.dom.minidom.Document`: Parsed document. rrrkN)rr+domminidomrrt)filename_or_filerlrrs r*xxml_dom_minidom_parsersm(ww$$%5@@@ #T? K 6q77??((5f5 6 6 6s *A99Bc eZdZdZedZedZedee ddffdZ e d de de e dee ddffd Zed d Zed de e ddfd Zy)ArchiveIterablezIAn iterable of (path, fileobj) from a TAR archive, used by `iter_archive`c#$Ktj|d}|D]o}|j}|js |#tj j |jdrR|j|}||fg|_ q~yw)Nzr|*)fileobjrr]__) tarfilerrisregr4r5rr extractfilemembers)rstreamtarinfo file_pathrs r* _iter_tarzArchiveIterable._iter_tarsae4 G I==? ww *55kB))'2HX% %FN  sBBc#,Ktj|}|jD]h}|j}|j r |#t j j|jdrR|j|}||fjyw)Nr) zipfileZipFileinfolistrfis_dirr4r5rrr)rzipfmemberr&rs r* _iter_zipzArchiveIterable._iter_zipsq!mmo &FI}} ww *55kByy(HX% % &sBBr.Nc#Kt|}|dk(r|j|Ed{y|j|Ed{y77w)Nr-)rOr/r')clsrrNs r*_iter_from_fileobjz"ArchiveIterable._iter_from_fileobjsE@C % }}Q' ' '}}Q' ' ' ( 's!%A AA A A  A rPrlc#Kt||}t|d|d5}|dk(r|j|Ed{n|j|Ed{dddy7'7#1swYyxYww)Nrkrr)rlrr-)rrtr/r')r1rPrlrNrs r*_iter_from_urlpathz"ArchiveIterable._iter_from_urlpath&st/wX 7D/a P ,TUe#==+++==+++  , ,++  , ,s>A1A%A!A%A#A% A1!A%#A%%A.*A1c(||j|Sr)r2)r1rs r*from_bufzArchiveIterable.from_buf3s3))733r)c*||j||Sr)r4)r1urlpath_or_bufrls r* from_urlpathzArchiveIterable.from_urlpath7s3))>?KKr)r)r.r)r%r&r'r staticmethodr'r/ classmethodrrr2rBr rr4r6r9r(r)r*rrsS  & &(itT0A&B((GK , ,,4^,D , 5$$ % , ,44L8N;SL_pLLr)rc veZdZdZe ddeeeefdee de eddffdZ eddee ddfdZ y) FilesIterablez8An iterable of paths from a list of directories or filesNurlpathsrlr.c #Kt|ts|g}|D]}t||r|t||rt ||D]\}}}t |Dcgc]}|j dr|c}|ddt|j drPt |D]"}|j drt||$t|ycc}ww)Nrkr) rClistrprzrsortedrrfrYr) r1r>rlrPrrrrYrfs r*_iter_from_urlpathsz!FilesIterable._iter_from_urlpaths?s(D) zH 1Gw@ A49'Sb4c 70GXy"()qgQXQcQcdoQp')q"rHQK )44[A $*9$57#..{;$#GX66 7 7(00! 1 *rsACC .C 2A*Cc*||j||Sr)rB)r1r>rls r* from_urlpathszFilesIterable.from_urlpathsWs3**HoFFr)r) r%r&r'rr;r rBr@r rrrBrDr(r)r*r=r=<s{BZ^1S$s)^,1?G?W1 3d? #11.Gh~6NGZiGGr)r=r)NNF)NFNTNNNF)r)NN)rrrrFrrr4rRrerr rxml.dom.minidomr+r)collections.abcrr itertoolsrpathlibrrtypingrr r r unittest.mockr urllib.parser xml.etreerrrUrhuggingface_hub.errorsr fsspec.corerr fsspec.utilsrhuggingface_hub.utilsrrr packagingrr1rrdownload.download_configr filesystemsrrr _filelockrrrtrackr aiohttp.client_exceptionsr!r# ImportError Exception get_loggerr%rINCOMPLETE_SUFFIXrBr,boolr8r:r=rErUrZrirrrrrrr callbacksrrrr rrrrrS extensionsrNrrT-SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOLrcompilerrfromhexrJrKmaxrHr@rrOrrYr^rdrfrirmrpintrurzr~rrrrrtr@rrrrrr`rrrrr rrrrr=)fs_classrs00r*rfs     %'00!' 1%SS"51#/ L   H %! Cds3s4sn3n4nYcYdYJAJ!J 6666,C,C,4x2 x2vsDy1A(Bc$CsDyAQ8R^b 8C=  e ;6##00;9(  T Tn6  " %0 !,,  x000 % 5 %!,1((1 S8,,,11- H_$_8X%6%6$_!&0bjj&=# MM*u MM*u MM*u MM(U MM&6 MM.!4 MM*u MM*v ($ U40BDtu   cc`Xc]`$cH^#S#8N+C#*%4I6'6#8N#;#t#(H^$<8$(>":$d$.t" H?C 8  8#+N#; 8 3S$sCx.() )* 8?C5$ 5$#+N#;5$ 3S$sCx.() )*5$p,RV,,x7O,^F3F.)AFTRUYF2!&SWk8Pk4ZH^$<Z8AwDLAwHK%T5()KW[l8N;SlX\jH^:R gH^V6$>L2>LBG0GW(  i  ~ 1 %`s$ Q584R :4R6R5R R