
    Pi                         d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZ	d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZmZ  ee          Zd Z G d de          ZdS )    N)ArgumentParser)	Generator)rmtree)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)DatasetInfosDict)dataset_module_factoryget_dataset_builder_class)VerificationMode)ERROR
get_loggerc                     t          | j        | j        | j        | j        | j        | j        p| j        | j        | j	        | j
        | j        
  
        S )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cachenum_proc)argss    j/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/datasets/commands/test.py_test_command_factoryr      sQ    	)$/!      c                   d    e Zd ZdZedefd            Zdededededed	ed
ededede	fdZ
d ZdS )r   Fparserc                    |                      dd          }|                    dt          d d           |                    dt          d d           |                    d	t          d d
           |                    ddd           |                    ddd           |                    ddd           |                    ddd           |                    ddd           |                    dt          d d           |                    ddd           |                    dt          d           |                    t
                     d S )NtestzTest dataset loading.)helpz--namezDataset processing name)typedefaultr$   z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr$   z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz
--num_proczNumber of processesz--save_infoszalias to save_infor   zName of the dataset to download)r%   r$   )func)
add_parseradd_argumentstrintset_defaultsr   )r!   test_parsers     r   register_subcommandzTestCommand.register_subcommand&   s   ''5L'MM  THa bbb  A	 	! 	
 	
 	
 	  S	 	! 	
 	
 	
 	  Lm nnn  ,5m 	! 	
 	
 	
 	  $D 	! 	
 	
 	

 	  !5lQk lll  U 	! 	
 	
 	

 	  CLa bbb  K_ ```  ;\ ]]]  &; <<<<<r   r   r   r   r   r   r   r   r   r   r   c                     || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        |	r |st          d           t          d           |r	d| _        d S d S )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cache	_num_procprintexit)selfr   r   r   r   r   r   r   r   r   r   s              r   __init__zTestCommand.__init__K   s      
#!'%%9"!1'! 	y 	V  
 GGG 	.)-D&&&	. 	.r   c           
      8   	
 t          j        d                              t                      j        % j        rt          d           t          d            j         j        c}	t          |          
t          
           j        rj        rt          j                  nd}dt          t          d d f         f	
 fd}t           |                      D ]\  }}t          d|j        j         d|dz    d| d	           t$          j                            t$          j                            |                                t.          j        j                            |_        |                     j        st8          j        nt8          j         j        rt@          j!        nt@          j"         j#        
           |$                                  j%        rt$          j        &                    |          st$          j        '                    |          n|}t%          j(        |d           tS          di |j        j        |j*        i+                    |           t          dt$          j                            |t.          j        j,                               j-        rt$          j        &                    |j.                  r6t^          0                    d|j.                    tc          |j.                   t$          j                             j.        t.          j        j2                  }t$          j        &                    |          r,t^          0                    d|            tc          |           t          d           d S )NfilelockzABoth parameters `config` and `all_configs` can't be used at once.r2   returnc               3   x  K   j         rkj        rdt          j                  D ]M\  } }dj        v r dj        j        dj        V  + d|j        j        j        dj        V  Nd S dj        v r dj        j        dj        V  d S  dj        j        dj        V  d S )Nconfig_name)r   r   )rE   r   r    )r7   BUILDER_CONFIGS	enumeratebuilder_kwargsr5   r6   r   )iconfigbuilder_clsrE   moduler?   s     r   get_buildersz%TestCommand.run.<locals>.get_buildersv   sn       [%@ !*;+F!G!G  IAv$(===)k &*o%)^  %3      *k (.&*o%)^  %3	       !F$999%+r$.rr\b\qrrrrrrr%+ $/"&/!%  !/	      r   zTesting builder 'z' (/))download_modeverification_moder   T)exist_okzDataset card saved at zClearing cache at zTest successful.rF   )3logging	getLoggersetLevelr   r4   r7   r=   r>   r3   r
   r   rG   lenr   r   rH   rK   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer:   r   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr9   r   	NO_CHECKS
ALL_CHECKSr<   
as_datasetr8   isdirbasenamemakedirsr	   infowrite_to_directoryREPOCARD_FILENAMEr;   r5   loggerwarningr   DOWNLOADED_DATASETS_DIR)r?   rY   
n_buildersrN   jbuildersave_infos_dirdownload_dirrL   rE   rM   s   `       @@@r   runzTestCommand.runl   sN   *%%..u555:!d&7!UVVVGGG M4:k'--/779=9Jq{OjqS4555pq
	id(BC 	 	 	 	 	 	 	 	 	6 $LLNN33 !	) !	)JAwSgn&9SSa!eSSjSSSTTT$&GNNW<<>>@jkk% %G! ((-3l@@!2-#1"2"<"<%0 )       
  r?Aw}}T?R?R!\!1!1$!7!7!7X\NT:::: GGGN$7#FGGZZ[ijjjprw||NHOLm/n/nppqqq   )7==!344 /NN#L8J#L#LMMM7-...!w||DOX_=dee7==.. )NN#F#F#FGGG<((( !!!!!r   N)__name__
__module____qualname____test__staticmethodr   r0   r,   boolr-   r@   rt   rF   r   r   r   r   #   s        H"=N "= "= "= \"=H.. . 	.
 . . . #. . . . . . .BH" H" H" H" H"r   r   )rT   rX   argparser   collections.abcr   shutilr   datasets.configr]   datasets.builderr   datasets.commandsr   "datasets.download.download_managerr   datasets.infor	   datasets.loadr
   r   datasets.utils.info_utilsr   datasets.utils.loggingr   r   ru   rl   r   r   rF   r   r   <module>r      sB    				 # # # # # # % % % % % %           + + + + + + 4 4 4 4 4 4 ; ; ; ; ; ; * * * * * * K K K K K K K K 6 6 6 6 6 6 4 4 4 4 4 4 4 4 
H		  Q" Q" Q" Q" Q"( Q" Q" Q" Q" Q"r   