o
    ॵi	                     @   sL   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ G dd de	Z
dS )    N)Pool)tqdm)OssUtilities)
UploadModec                   @   sb   e Zd ZdededefddZdededed	efd
dZdedededededed	efddZ	dS )DatasetUploadManagerdataset_name	namespaceversionc                 C   s:   ddl m} | }|j||d|d}t||||d| _d S )Nr   )HubApiF)r   r   check_cookierevision)
oss_configr   r   r   )modelscope.hub.apir
   !get_dataset_access_config_sessionr   oss_utilities)selfr   r   r	   r
   _hub_api_oss_config r   \/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/utils/upload_utils.py__init__   s   zDatasetUploadManager.__init__object_namelocal_file_pathupload_modereturnc                 C   s   | j j||d|d}|S )NToss_object_namer   indicate_individual_progressr   r   upload)r   r   r   r   
object_keyr   r   r   r      s   zDatasetUploadManager.uploadobject_dir_namelocal_dir_pathnum_processes	chunksizefilter_hidden_filesc              
      s    fdd}g }t |D ]0\}	}
}|D ](}|r|drqt j||	|ddd|}t j|	|}|||f qqt|d}t	t
|j|||dt|d	}W d    t|S 1 sbw   Y  t|S )
Nc                    s     j j| d | d dd d S )Nr      Fr   r   )argsr   r   r   r   
run_upload*   s   
z3DatasetUploadManager.upload_dir.<locals>.run_upload. r&   /)	processes)r$   )total)oswalk
startswithpathjoinreplacestripappend
ThreadPoollistr   imaplen)r   r!   r"   r#   r$   r%   r   r)   
files_listrootdirsfiles	file_namer   r   poolresultr   r(   r   
upload_dir&   s0   
zDatasetUploadManager.upload_dirN)
__name__
__module____qualname__strr   r   r   intboolrB   r   r   r   r   r      s$    


	r   )r/   multiprocessing.dummyr   r7   r   %modelscope.msdatasets.utils.oss_utilsr   modelscope.utils.constantr   objectr   r   r   r   r   <module>   s   