o
    ॵix                     @   sd   d dl mZ d dlmZ d dlmZmZ d dlmZ d dl	m
Z
 G dd deZG dd	 d	eZd
S )    )DownloadManager)StreamingDownloadManager)cached_pathis_relative_path)DataDownloadConfig)OssUtilitiesc                       sN   e Zd Zdef fddZdededefddZdededefdd	Z  ZS )
DataDownloadManagerdownload_configc                    s   t  j|j|j|dd d S )NT)dataset_namedata_dirr	   record_checksums)super__init__r
   r   selfr	   	__class__ c/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/download/download_manager.pyr      s   
zDataDownloadManager.__init__url_or_filenamereturnc                 C   sB   t |}t|j|j|j|jd}t|r|j||dS t||dS N)
oss_configr
   	namespacerevision)r	   )	strr   r   r
   r   versionr   downloadr   )r   r   r	   oss_utilitiesr   r   r   	_download   s   zDataDownloadManager._downloadc                 C   s   |  ||S Nr   )r   r   r	   r   r   r   _download_single'   s   z$DataDownloadManager._download_single)	__name__
__module____qualname__r   r   r   r   r"   __classcell__r   r   r   r   r      s    
r   c                       sJ   e Zd ZdZdef fddZdedefddZdedefd	d
Z  Z	S )DataStreamingDownloadManagerz$The data streaming download manager.r	   c                    s   t  j|j|j||jd d S )N)r
   r   r	   	base_path)r   r   r
   r   	cache_dirr   r   r   r   r   0   s   
z%DataStreamingDownloadManager.__init__r   r   c                 C   sN   t |}t| jj| jj| jj| jjd}t|r |j|| jdS t	|| jdS r   )
r   r   r	   r   r
   r   r   r   r   r   )r   r   r   r   r   r   r   7   s   z&DataStreamingDownloadManager._downloadc                 C   s
   |  |S r    r!   )r   r   r   r   r   r"   G   s   
z-DataStreamingDownloadManager._download_single)
r#   r$   r%   __doc__r   r   r   r   r"   r&   r   r   r   r   r'   -   s
    r'   N)"datasets.download.download_managerr   ,datasets.download.streaming_download_managerr   datasets.utils.file_utilsr   r   .modelscope.msdatasets.download.download_configr   %modelscope.msdatasets.utils.oss_utilsr   r   r'   r   r   r   r   <module>   s   !