
    bi                         d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZ	d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZmZ  ee      Zd Z G d de      Zy)    N)ArgumentParser)	Generator)rmtree)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)DatasetInfosDict)dataset_module_factoryget_dataset_builder_class)VerificationMode)ERROR
get_loggerc                    t        | j                  | j                  | j                  | j                  | j
                  | j                  xs | j                  | j                  | j                  | j                  | j                  
      S )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cachenum_proc)argss    Q/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/commands/test.py_test_command_factoryr      sf    		)$//!!     c                   \    e Zd ZdZedefd       Zdededededed	ed
ededede	fdZ
d Zy)r   Fparserc                 >   | j                  dd      }|j                  dt        d d       |j                  dt        d d       |j                  d	t        d d
       |j                  ddd       |j                  ddd       |j                  ddd       |j                  ddd       |j                  ddd       |j                  dt        d d       |j                  ddd       |j                  dt        d       |j	                  t
               y )NtestzTest dataset loading.)helpz--namezDataset processing name)typedefaultr$   z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr$   z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz
--num_proczNumber of processesz--save_infoszalias to save_infor   zName of the dataset to download)r%   r$   )func)
add_parseradd_argumentstrintset_defaultsr   )r!   test_parsers     r   register_subcommandzTestCommand.register_subcommand&   sN   ''5L'M  THa b  A	 	! 	
 	  S	 	! 	
 	  Lm n  ,5m 	! 	
 	  $D 	! 	

 	  !5lQk l  U 	! 	

 	  CLa b  K_ `  ;\ ]  &; <r   r   r   r   r   r   r   r   r   r   r   c                     || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        |	r|st        d       t        d       |rd| _        y y )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cache	_num_procprintexit)selfr   r   r   r   r   r   r   r   r   r   s              r   __init__zTestCommand.__init__K   s{      
#!'%%9"!1'!yV
 G)-D& r   c           
         	
 t        j                  d      j                  t                j                  " j
                  rt        d       t        d        j                   j                  c}	t        |      
t        
       j
                  r!j                  rt        j                        nd}dt        t        d d f   f	
 fd}t         |             D ]  \  }}t        d|j                   j"                   d|dz    d| d	       t$        j&                  j)                  t$        j&                  j+                  |j-                         t.        j                   j0                              |_        |j5                   j6                  st8        j:                  nt8        j<                   j>                  rt@        jB                  nt@        jD                   jF                  
       |jI                           jJ                  rt$        j&                  jM                  |      st$        j&                  jO                  |      n|}t%        jP                  |d       tS        di |j                   j"                  |jT                  ijW                  |       t        dt$        j&                  j+                  |t.        j                   jX                                 jZ                  st$        j&                  jM                  |j\                        r7t^        ja                  d|j\                          tc        |j\                         t$        j&                  j+                   j\                  t.        j                   jd                        }t$        j&                  jM                  |      st^        ja                  d|        tc        |        t        d       y )NfilelockzABoth parameters `config` and `all_configs` can't be used at once.r2   returnc               3   &  K   j                   rj                  rt        j                        D ]u  \  } }dj                  v r, dj                  j
                  dj                   @ d|j                  j                  j
                  dj                   w y dj                  v r, dj                  j
                  dj                   y  dj                  j
                  dj                   y w)Nconfig_name)r   r   )rE   r   r    )r7   BUILDER_CONFIGS	enumeratebuilder_kwargsr5   r6   r   )iconfigbuilder_clsrE   moduler?   s     r   get_buildersz%TestCommand.run.<locals>.get_buildersv   s      [%@%@!*;+F+F!G IAv$(=(==) &*oo%)^^ %33  * (.&*oo%)^^ %33	  !F$9$99%r$..r\b\q\qrr% $/"&//!% !//	 s   DDzTesting builder 'z' (/))download_modeverification_moder   T)exist_okzDataset card saved at zClearing cache at zTest successful.rF   )3logging	getLoggersetLevelr   r4   r7   r=   r>   r3   r
   r   rG   lenr   r   rH   rK   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer:   r   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr9   r   	NO_CHECKS
ALL_CHECKSr<   
as_datasetr8   isdirbasenamemakedirsr	   infowrite_to_directoryREPOCARD_FILENAMEr;   r5   loggerwarningr   DOWNLOADED_DATASETS_DIR)r?   rY   
n_buildersrN   jbuildersave_infos_dirdownload_dirrL   rE   rM   s   `       @@@r   runzTestCommand.runl   s   *%..u5::!d&7&7UVG MM4::k'-/79=9J9J{OjOjS445pq
	id(BC 	 	6 $LN3 !	)JAw%gnn&9&9%:#a!eWAj\QRST$&GGNNW<<>@j@jk%G! ((-- +@@!22-- #3"<"<%00 )   
 ?Aww}}T?R!1!1$!7X\NT: GGNN$7$7#FGZZ[ij.rww||NHOOLmLm/n.opq   77==!3!34NN%78J8J7K#LM7--.!ww||DOOX__=d=de77==.NN%7~#FG<(C!	)F 	 !r   N)__name__
__module____qualname____test__staticmethodr   r0   r,   boolr-   r@   rt   rF   r   r   r   r   #   s    H"=N "= "=H.. . 	.
 . . . #. . . .BH"r   r   )rT   rX   argparser   collections.abcr   shutilr   datasets.configr]   datasets.builderr   datasets.commandsr   "datasets.download.download_managerr   datasets.infor	   datasets.loadr
   r   datasets.utils.info_utilsr   datasets.utils.loggingr   r   ru   rl   r   r   rF   r   r   <module>r      sL     	 # %   + 4 ; * K 6 4 
H	Q"( Q"r   