o
    ॵiZ                     @   s   d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZmZ d dlmZ e Zd	Zd
ZdZdZdZdZG dd dZdS )    )print_functionN)hash_url_to_filename)HubApi)DataDownloadConfig)MS_CACHE_HOME)"DEFAULT_DATA_ACCELERATION_ENDPOINTMetaDataFields
UploadMode)
get_loggerAccessIdAccessSecretSecurityTokenBucket	BackupDirDirc                
   @   s`   e Zd Zdd Zdd Zdd Zedd Zd	ed
e	fddZ
dededededef
ddZdS )OssUtilitiesc                 C   sv   | j |d || _|| _|| _tjtd| _t	
 | _d| _d| _d| _tj| jd| _tj| jd| _t | _d S )N)
oss_configztmp/resumable_storei   i      )root)_do_initdataset_name	namespacerevisionospathjoinr   resumable_store_root_pathmultiprocessing	cpu_countnum_threads	part_sizemultipart_thresholdmax_retriesoss2ResumableDownloadStoreresumable_store_downloadResumableStoreresumable_store_uploadr   api)selfr   r   r   r    r*   Y/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/utils/oss_utils.py__init__   s$   
zOssUtilities.__init__c                 C   s   |t  | _|t | _|t | _tddkrt| _	n	d|d  d| _	|t
 | _t| j| j| j}tj|| j	| jdd| _|t | _|t | _d S )NENABLE_DATASET_ACCELERATIONTruezhttps://Regionz.aliyuncs.comx   )connect_timeout)	ACCESS_IDkeyACCESS_SECRETsecretSECURITY_TOKENtokenr   getenvr   endpointBUCKETbucket_namer#   StsAuthr   bucketDIRoss_dirBACK_DIRoss_backup_dir)r)   r   authr*   r*   r+   r   1   s   




zOssUtilities._do_initc                 C   s2   t d | jj| j| jd| jd}| | d S )Nz"Reloading sts token automatically.T)r   r   check_cookier   )loggerinfor(   !get_dataset_access_config_sessionr   r   r   r   )r)   oss_config_refreshr*   r*   r+   _reload_sts@   s   
zOssUtilities._reload_stsc                 C   s8   |rt dt| t|  }td|ddd d S d S )Nd   z{0}%  T)endflush)intfloatprintformat)consumed_bytestotal_bytesrater*   r*   r+   _percentageI   s   zOssUtilities._percentageoss_file_namedownload_configc              
   C   s2  |j }tj| j|}tj| j|}|j}d}|r(|j|}|r(|t	j
}d}		 z?|	d7 }	|r5|}
n
| j|r=|n|}
t|
d d}tj||}|jsUtj|shtj| j|
|| j| j| j| j| jd W |S  ty } z!|jddkr~|   |	| jkrtd	|  |W Y d }~nd }~ww q+)
NFr   T   )etag)storemultiget_thresholdr    progress_callbackr   status  zFailed to download )	cache_dirr   r   r   r?   rA   splitmeta_args_mapgetr   ARGS_BIG_DATAr=   object_existsr   force_downloadexistsr#   resumable_downloadr%   r!   r    rT   r   	Exception__dict__rH   r"   rD   warning)r)   rU   rV   r^   candidate_keycandidate_key_backupr_   big_data	args_dictretry_countfile_oss_keyfilename
local_pather*   r*   r+   downloadO   s^   	
zOssUtilities.downloadoss_object_namelocal_file_pathindicate_individual_progressupload_modereturnc           
   
   C   s   d}t j| j|}|r| j}nd }	 z6|d7 }| j|}|tjkr5|r5t	
d| d|j d W |S tj| j||| j| j| j|| jd W |S  tym }	 z|	jdd	kr]|   || jkrc W Y d }	~	nd }	~	ww q)
Nr   TrW   zSkip z in case of z mode.)rY   r!   r    r[   r   r\   r]   )r   r   r   r?   rT   r=   rc   r	   APPENDrD   rE   valuer#   resumable_uploadr'   r!   r    r   rg   rh   ra   rH   r"   )
r)   rt   ru   rv   rw   rn   
object_keyr[   existrr   r*   r*   r+   upload~   sH   	
zOssUtilities.uploadN)__name__
__module____qualname__r,   r   rH   staticmethodrT   strr   rs   boolr	   r~   r*   r*   r*   r+   r      s     	


/r   )
__future__r   r   r   r#   datasets.utils.file_utilsr   modelscope.hub.apir   .modelscope.msdatasets.download.download_configr   modelscope.utils.config_dsr   modelscope.utils.constantr   r   r	   modelscope.utils.loggerr
   rD   r2   r4   r6   r:   r@   r>   r   r*   r*   r*   r+   <module>   s$   