o
    ॵi                     @   s   d dl Z d dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ e ZG d	d
 d
e jZG dd de jZG dd deZG dd deZG dd deZdS )    N)ABCabstractmethod)load_dataset)HubApi)DatasetContextConfig)OssDownloader)EXTENSIONS_TO_LOAD)
get_loggerc                   @   s   e Zd ZdZdZdZdZdS )LocalDataLoaderTypezQ Supported data loader types for local dataset: huggingface, PyTorch, Tensorflow hf_data_loadertorch_data_loadertf_data_loaderN)__name__
__module____qualname____doc__HF_DATA_LOADERTORCH_DATA_LOADERTF_DATA_LOADER r   r   i/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/data_loader/data_loader_manager.pyr
      s
    r
   c                   @   s   e Zd ZdZdZdZdS )RemoteDataLoaderTypezI Supported data loader types for remote dataset: huggingface, modelscope r   ms_data_loaderN)r   r   r   r   r   MS_DATA_LOADERr   r   r   r   r      s    r   c                   @   s2   e Zd ZdZdefddZedejfddZ	dS )	DataLoaderManagerz Data loader manager, base class.dataset_context_configc                 C   s
   || _ d S Nr   selfr   r   r   r   __init__#   s   
zDataLoaderManager.__init__data_loader_typec                 C   s   d S r   r   )r   r!   r   r   r   r   &   s   zDataLoaderManager.load_datasetN)
r   r   r   r   r   r    r   enumEnumr   r   r   r   r   r       s
    r   c                       6   e Zd ZdZdef fddZdejfddZ  Z	S )LocalDataLoaderManagerz+Data loader manager for loading local data.r   c                    s   t  j|d d S Nr   )superr    r   	__class__r   r   r    .   s   zLocalDataLoaderManager.__init__r!   c                 C   s   | j j}| j j}| j j}| j j}| j j}| j j}| j j}| j j}	| j j	}
| j j
}tj|rGtj|d d}|tv rGd }|g}t|}|tjkr_t|f|||||||	j|
dd	|S dtjj d)N   .T)	namerevisionsplitdata_dir
data_files	cache_dirdownload_mode	streamingignore_verificationsz!Expected local data loader type: )r   dataset_namesubset_nameversionr.   r/   r0   cache_root_dirr2   use_streamingconfig_kwargsospathisfilesplitextstripr   getr
   r   r   value)r   r!   r5   r6   r7   r.   r/   r0   r8   r2   r9   input_config_kwargsfile_extr   r   r   r   1   sB   

z#LocalDataLoaderManager.load_dataset
r   r   r   r   r   r    r"   r#   r   __classcell__r   r   r(   r   r%   +   s    r%   c                       r$   )RemoteDataLoaderManagerz,Data loader manager for loading remote data.r   c                    s   t  j|d t | _d S r&   )r'   r    r   apir   r(   r   r   r    \   s   z RemoteDataLoaderManager.__init__r!   c                 C   s   | j j}| j j}| j j}| j j}| j j}| j j}| j j}| j jj	}	| j j
}
| j j}|tjkrJt|f||||||	|
dd|}| jj|||
d |S |tjkret| j d}|  | jj|||
d |jS dtjj	 dtjj	 d| d)	NT)r,   r-   r.   r/   r0   r2   r3   r4   )r5   	namespacer9   r   z"Expected remote data loader type: /z
, but got z .)r   r5   rH   r6   r7   r.   r/   r0   r2   rA   r9   r:   r   r   r   rG   dataset_download_statisticsr   r   processdataset)r   r!   r5   rH   r6   r7   r.   r/   r0   download_mode_valr9   rB   dataset_retoss_downloaderr   r   r   r   `   s^   



z$RemoteDataLoaderManager.load_datasetrD   r   r   r(   r   rF   Y   s    rF   )r"   r;   abcr   r   datasetsr   r   modelscope.hub.apir   4modelscope.msdatasets.context.dataset_context_configr   -modelscope.msdatasets.data_loader.data_loaderr   modelscope.utils.constantr   modelscope.utils.loggerr	   loggerr#   r
   r   r   r%   rF   r   r   r   r   <module>   s   .