o
    ॵiZ(                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZ d dlZd dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlmZ d d	lmZ d d
l m!Z! d dl"m#Z# e! Z$									 						dde%fddZ&dS )    N)contextmanager)partial)Path)urljoinurlparse)config)	hash_url_to_filename"get_authentication_headers_for_urlftp_headfsspec_head	http_head!_raise_if_offline_mode_is_enabledftp_get
fsspec_gethttp_get)FileLock)MS_DATASETS_CACHE)
get_logger)ModelScopeConfigFd   T
deprecatedreturnc           '         s  |dkrt d| dt |}
|du rt}t|trt|}tj|dd |r1t	| t
| j}n| }d}d}d}d}d}d}t|dd}tj||}tj|rX|sX|sX|S t| |
d	}|durf||d
< |sPt
| j}|dkrwt| }n|dvrt| |d}|r|ddp|ddnd}d}zt }t| d|||	||d}|jdkr|r|jdnd}|j D ]\}}|drd| v r| d| 7 } |j}qd}d| v rd| vr| d7 } nX|jdkrd| v s|jdkrd| v s|jdkrtd| std|js|jdkrd| v rd}t d|   n|jdkr5t!j"| v r5|
du r5t#d |  d!W n t$t%j&j'fyO } z|}W Y d}~nd}~ww |stj|r_|s_|S |rjt(d"| d#|dur||jd$kr|t(d%|  t)d&|   |durt#d'|  d(t*| d)|durt#d'|  d*|j d)t#d'|  t||}tj||}tj|r|s|S |d+ }t+| tj|r|s|W  d   S |d,  t,dB fd.d/	}d0}|rt-|d1d2}tj rt. j/}| d} t d3| j0  |dkrt1| |  nH|dvrBt23t4}!d4|!j5v r9t4| | |||d5 n.t4| | ||d6 n%t23t6}"d4|"j5v r[t6| | |||||	||d7	 nt6| | |||||	|d8 W d   n	1 srw   Y  t d9|  d:|  t78| j0| t9d;}#t9|# t:|d;|# @  t d<|  | |d=}$|d> }%t;|%d?d@dA}&t<=|$|& W d   n1 sw   Y  W d   |S W d   |S 1 sw   Y  |S )Ca  
    Given a URL, look for the corresponding file in the local cache.
    If it's not there, download it. Then return the path to the cached file.

    Return:
        Local path (string)

    Raises:
        FileNotFoundError: in case of non-recoverable file
            (non-existent or no cache on disk)
        ConnectionError: in case of unreachable url
            and no cache on disk
    r   z'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.
You can remove this warning by passing 'token=z
' instead.NT)exist_okF)etag)tokenz
user-agentftp)httphttps)storage_optionsETagr   )allow_redirectsproxiestimeoutmax_retriesheaderscookies   download_warningzdrive.google.comz	&confirm=zconfirm=z
&confirm=ti  zfirebasestorage.googleapis.comi  i  z7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url i  zUnauthorized for URL zU. Please use the parameter `token=True` after logging in with `huggingface-cli login`z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.i  zCouldn't find file at zTried to reach zCouldn't reach z ()z (error z.lockz.incompletew+bc                 3   s8    t  | }|V  W d    d S 1 sw   Y  d S )N)open)modefincomplete_path ]/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/msdatasets/utils/hf_file_utils.pytemp_file_manager   s   "z,get_from_cache_ms.<locals>.temp_file_managerr   za+b)r+   zDownloading to disable_tqdm)r   descr2   )r   r3   )	temp_filer!   resume_sizer$   r%   r#   r3   r2   )r4   r!   r5   r$   r%   r#   r3   zstoring z in cache at i  zcreating metadata file for )urlr   z.jsonwzutf-8)encoding)r)   )>warningswarnFutureWarningr   
isinstancer   strosmakedirsr   r   pathr   joinexistsr	   schemer
   r   getr   get_cookiesr   status_coder$   r%   items
startswithrematchr6   loggerinfor   HF_ENDPOINTConnectionErrorOSErrorrequests
exceptionsTimeoutFileNotFoundErrorr   reprr   r   r   statst_sizenamer   inspect	signaturer   
parametersr   shutilmoveumaskchmodr*   jsondump)'r6   	cache_dirforce_downloadr!   etag_timeoutresume_download
user_agentlocal_files_onlyuse_etagr#   r   use_auth_tokenignore_url_paramsr   download_descr2   
cached_url	connectedresponser%   r   
head_errorrC   filename
cache_pathr$   kve	lock_pathr1   r5   r4   fsspec_get_sighttp_get_sigr]   meta	meta_path	meta_filer/   r-   r0   get_from_cache_ms   sT  


 
	
"








,


 
L
LLrz   )NFNr   FNFTr   Nr   FNNF)'r_   r>   rI   r[   r9   rX   
contextlibr   	functoolsr   pathlibr   urllib.parser   r   rP   datasetsr   datasets.utils.file_utilsr   r	   r
   r   r   r   r   r   r   filelockr   modelscope.utils.config_dsr   modelscope.utils.loggerr   modelscope.hub.apir   rK   r=   rz   r/   r/   r/   r0   <module>   sH   ,