o
    .i#                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZmZ eeZdd ZG dd deZ dS )    N)ArgumentParser)	Generator)Path)copyfilermtree)Optional)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)dataset_module_factoryimport_main_class)VerificationMode)ERROR
get_loggerc                 C   s8   t | j| j| j| j| j| jp| j| j| j	| j
| j| jS )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cachenum_proctrust_remote_code)args r   J/home/ubuntu/.local/lib/python3.10/site-packages/datasets/commands/test.py_test_command_factory   s   
r    c                   @   sd   e Zd ZdZedefddZdedededed	ed
edededede	de
e fddZdd ZdS )r   Fparserc                 C   s   | j ddd}|jdtd dd |jdtd dd |jd	td d
d |jdddd |jdddd |jdddd |jdddd |jdddd |jdtd dd |jdddd |jdddd |jdtdd |jtd  d S )!NtestzTest dataset implementation.)helpz--namezDataset processing name)typedefaultr#   z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr#   z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz
--num_proczNumber of processesz--trust_remote_codez6whether to trust the code execution of the load scriptz--save_infoszalias to save_infor   zName of the dataset to download)r$   r#   )func)
add_parseradd_argumentstrintset_defaultsr    )r!   test_parserr   r   r   register_subcommand(   sH   zTestCommand.register_subcommandr   r   r   r   r   r   r   r   r   r   r   c                 C   sl   || _ || _|| _|| _|| _|| _|| _|| _|	| _|
| _	|| _
|	r-|s-td td |r4d| _d S d S )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset script.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cache	_num_proc_trust_remote_codeprintexit)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   __init__P   s&   
zTestCommand.__init__c              	      sb  t dt jd urjrtd td jj}t	|j
dtj jr6 jr6t jnd}dttd d f f fdd}t| D ]\}}td|jj d	|d  d
| d tjtj| tjj|_|jjs|tjntj j!rt"j#nt"j$j%d |&  j'r|'  j'rtj  tjj(}t)|jd }tj||}tj*|rtj+|}	ntj*|r|}	ntj,|r|}	n	d }	td|  |	d urtj|	tjj(}
t-||
 td|
  j.r*tj,|j/rt01d|j/  t2|j/ tjj/tjj3}tj,|r*t01d|  t2| qNtd d S )NfilelockzABoth parameters `config` and `all_configs` can't be used at once.r0   )r   returnc                  3   s    j r7 jr7t jD ](\} }djv r$ djjdjV  q d|jjjdjV  qd S djv rL djjdjV  d S  djjdjV  d S )Nconfig_name)r   r   )rB   r   r   r   )r5   BUILDER_CONFIGS	enumeratebuilder_kwargsr3   r4   r   )iconfigbuilder_clsrB   moduler>   r   r   get_builders}   s6   



 z%TestCommand.run.<locals>.get_builderszTesting builder 'z' (/))download_modeverification_moder   z.pyzDataset card saved at zClearing cache at zTest successful.)4logging	getLoggersetLevelr   r2   r5   r<   r=   r1   r   r;   r   module_pathrC   lenr   r   rD   rG   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer8   r
   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr7   r   	NO_CHECKS
ALL_CHECKSr:   
as_datasetr6   REPOCARD_FILENAMEr   isfiledirnameisdirr   r9   r3   loggerwarningr   DOWNLOADED_DATASETS_DIR)r>   rV   
n_buildersrK   jbuilderdataset_readme_pathr   combined_pathdataset_diruser_dataset_readme_pathdownload_dirr   rH   r   runs   sl   
"$	

zTestCommand.runN)__name__
__module____qualname____test__staticmethodr   r/   r+   boolr,   r   r?   rr   r   r   r   r   r   %   s8    '	

#r   )!rP   rU   argparser   collections.abcr   pathlibr   shutilr   r   typingr   datasets.configrZ   datasets.builderr   datasets.commandsr	   "datasets.download.download_managerr
   datasets.loadr   r   datasets.utils.info_utilsr   datasets.utils.loggingr   r   rs   rg   r    r   r   r   r   r   <module>   s"    