o
    8wi                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZ	d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZmZ eeZdd ZG dd deZdS )    N)ArgumentParser)	Generator)rmtree)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)DatasetInfosDict)dataset_module_factoryget_dataset_builder_class)VerificationMode)ERROR
get_loggerc                 C   s4   t | j| j| j| j| j| jp| j| j| j	| j
| j
S )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cachenum_proc)args r   S/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/datasets/commands/test.py_test_command_factory   s   
r   c                   @   s\   e Zd ZdZedefddZdedededed	ed
edededede	fddZ
dd ZdS )r   Fparserc                 C   s   | j ddd}|jdtd dd |jdtd dd |jd	td d
d |jdddd |jdddd |jdddd |jdddd |jdddd |jdtd dd |jdddd |jdtdd |jtd d S )NtestzTest dataset loading.)helpz--namezDataset processing name)typedefaultr    z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr    z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz
--num_proczNumber of processesz--save_infoszalias to save_infor   zName of the dataset to download)r!   r    )func)
add_parseradd_argumentstrintset_defaultsr   )r   test_parserr   r   r   register_subcommand&   sB   zTestCommand.register_subcommandr   r   r   r   r   r   r   r   r   r   c                 C   sf   || _ || _|| _|| _|| _|| _|| _|| _|	| _|
| _	|	r*|s*t
d td |r1d| _d S d S )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cache	_num_procprintexit)selfr   r   r   r   r   r   r   r   r   r   r   r   r   __init__K   s$   
zTestCommand.__init__c              	      s  t dt jd urjrtd td jj}t	|t
 jr2 jr2t jnd}dttd d f f fdd}t| D ]\}}td|jj d|d  d	| d
 tjtj| tjj|_|jjsxtjntjjrt j!nt j"j#d |$  j%rtj&|stj'|n|}tj(|dd t)di |jj|j*i+| tdtj|tjj,  j-rtj&|j.rt/0d|j.  t1|j. tjj.tjj2}tj&|rt/0d|  t1| qJtd d S )NfilelockzABoth parameters `config` and `all_configs` can't be used at once.r-   returnc                  3   s    j r7 jr7t jD ](\} }djv r$ djjdjV  q d|jjjdjV  qd S djv rL djjdjV  d S  djjdjV  d S )Nconfig_name)r   r   )r>   r   r   r   )r2   BUILDER_CONFIGS	enumeratebuilder_kwargsr0   r1   r   )iconfigbuilder_clsr>   moduler:   r   r   get_buildersv   s6   



 z%TestCommand.run.<locals>.get_builderszTesting builder 'z' (/))download_modeverification_moder   T)exist_okzDataset card saved at zClearing cache at zTest successful.r   )3logging	getLoggersetLevelr   r/   r2   r8   r9   r.   r	   r
   r?   lenr   r   r@   rC   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer5   r   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr4   r   	NO_CHECKS
ALL_CHECKSr7   
as_datasetr3   isdirbasenamemakedirsr   infowrite_to_directoryREPOCARD_FILENAMEr6   r0   loggerwarningr   DOWNLOADED_DATASETS_DIR)r:   rR   
n_buildersrG   jbuildersave_infos_dirdownload_dirr   rD   r   runl   sN   "$	
zTestCommand.runN)__name__
__module____qualname____test__staticmethodr   r,   r(   boolr)   r;   rm   r   r   r   r   r   #   s4    $	

!r   )rM   rQ   argparser   collections.abcr   shutilr   datasets.configrV   datasets.builderr   datasets.commandsr   "datasets.download.download_managerr   datasets.infor   datasets.loadr	   r
   datasets.utils.info_utilsr   datasets.utils.loggingr   r   rn   re   r   r   r   r   r   r   <module>   s     