o 5tiU@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZddlmZddlmZddlmZddlmZmZmZmZmZddlmZddlZdd lmZdd l m!Z!dd l"m#Z#d d l$m%Z%m&Z&ddl$m'Z'e'(e)Z*dZ+ede,eZ-dideeee,fde,fddZ.de,de/fddZ0de,de/fddZ1de,de/fddZ2de-de-fddZ3djd"e,d#e,de,fd$d%Z4 dkd"e,d#e,deej5e6ffd&d'Z7dide,d(e,d)ee,de,fd*d+Z8d,e,d-e,de,fd.d/Z9d0e,de,fd1d2Z:did3d4Z; dide,fd5d6Zdid:e,d;eee,e/fde=fdd?d?e@ZAdid@ee,fdAdBZBddejCjDdddCd fdDeeEdEee=dFeejCjDdGeeeFdHeFdIeGdJeGfdKdLZH  C MdldNe,d:e,dHeFdIeGdJeGdOeGdej5fdPdQZIdmdRdSZJdmdTdUZK dndWdXZL dodej5fdYdZZMdid:e,d;eee,e/fdee,fd[d\ZN   ]  !   dpde,fd^d_ZOd`daZPdbdcZQdddeZRdfejSfdgdhZTdS)qz Utilities for working with the local dataset cache. This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp Copyright by the AllenNLP authors. N)closingcontextmanager)partial)sha256)Path)ListOptionalTypeTypeVarUnion)urlparse)DownloadConfig)ExtractManager)FileLock) __version__config)logging .incompleteThf_modules_cachereturncCs|dur|ntj}t|}|tjvrEtj|tj|ddtjtj |dsEt tj |dd Wd|S1s@wY|S)z Add hf_modules_cache to the python path. By default hf_modules_cache='~/.cache/huggingface/modules'. It can also be set with the environment variable HF_MODULES_CACHE. This is used to add modules such as `datasets_modules` NTexist_okz __init__.pyw) rHF_MODULES_CACHEstrsyspathappendosmakedirsexistsjoinopen)rr&M/home/ubuntu/.local/lib/python3.10/site-packages/evaluate/utils/file_utils.pyinit_hf_modules)s   r(url_or_filenamecCst|}|jdvS)N)httphttpss3gshdfsftp)r scheme)r)parsedr&r&r' is_remote_url<s r2cCs$t|jdkptjt|jdS)Nz:/)r r0r!rismountr)r&r&r' is_local_pathAs$r6cCst|jdko tj| S)Nr3)r r0r!risabsr5r&r&r'is_relative_pathHsr8rcCs6tjtjtjt|}t|trt|S|S)z'Convert relative path to absolute path.)r!rabspath expanduser expandvarsr isinstancer)r abs_path_strr&r&r'relative_to_absolute_pathLs r>FT identifierfilenamecCs6|r |rtjntj}n|rtjntj}d|||fSN/)r"CLOUDFRONT_DATASETS_DISTRIB_PREFIXS3_DATASETS_BUCKET_PREFIX!CLOUDFRONT_METRICS_DISTRIB_PREFIXS3_METRICS_BUCKET_PREFIXr$)r?r@use_cdndatasetendpointr&r&r' hf_bucket_urlRsrJcCstt||||d|dS)N)r?r@rGrH) max_retries) http_headrJ)r?r@rGrHrKr&r&r' head_hf_s3ZsrMnamerevisioncCs|ptj}tjj|||dS)NrrNrO)rHUB_DEFAULT_VERSIONHUB_EVALUATE_URLformatrPr&r&r' hf_hub_urlcs rT base_name pathnamescGs8t|rtj|gdd|DRSt|g|RS)Ncss(|]}t|tjddVqdSrBN)rreplacer!seplstrip).0pathnamer&r&r' js&z#url_or_path_join..)r2 posixpathr$ras_posix)rUrVr&r&r'url_or_path_joinhsr` url_or_pathcCs&t|r |d|dStj|SrA)r2rindexr!rdirname)rar&r&r'url_or_path_parentos rdcCsV|d}t|}|}|r |d}t|}|d|7}|dr)|d7}|S)a Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) utf-8.z.py)encoder hexdigestendswith)urletag url_bytesurl_hashr@ etag_bytes etag_hashr&r&r'hash_url_to_filenamevs   rpc Ks|dur td i|}|jptj}t|trt|}t|tr#t|}t|r@t|||j |j |j |j |j |j|j|j|jd }ntj|rI|}nt|rUtd|dtd|d|durc|S|jrrt|jdj||jd}|S) a Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and return the path to the cached file. If it's already a local path, make sure the file exists and then return the path. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk ValueError: if it couldn't parse the url or filename correctly requests.exceptions.ConnectionError: in case of internet connection issue N) cache_dirforce_downloadproxiesresume_download user_agentlocal_files_onlyuse_etagrKtoken download_descz Local file z doesn't existzunable to parse z as a URL or as a local path)rq) force_extractr&)r rqrDOWNLOADED_EVALUATE_PATHr<rrr2get_from_cacherrrsrtrurvrwrKrxryr!rr#r6FileNotFoundError ValueErrorextract_compressed_filerextractrz)r)download_configdownload_kwargsrq output_pathr&r&r' cached_pathsB     rrucCsdtdtj}|dtj7}tjr|dtj7}tjr'|dtj7}tjr2|dtj 7}t |t rJ|dd dd | D7}|St |trU|d|7}|S) Nz datasets/z ; python/z ; pyarrow/z; torch/z ; tensorflow/z; jax/z; css"|] \}}|d|VqdSrWr&)r[kvr&r&r'r]s z*get_datasets_user_agent..)rr PY_VERSIONPYARROW_VERSIONTORCH_AVAILABLE TORCH_VERSION TF_AVAILABLE TF_VERSION JAX_AVAILABLE JAX_VERSIONr<dictr$itemsr)ruuar&r&r'get_datasets_user_agents " rrjrxcCs.i}|tjrddlm}||dtd}|S)zHandle the HF authenticationr)build_hf_headersevaluate)rx library_namelibrary_version) startswithr HF_ENDPOINThuggingface_hub.utilsrr)rjrxheadersrr&r&r'"get_authentication_headers_for_urls   rc@s eZdZdS)OfflineModeIsEnabledN)__name__ __module__ __qualname__r&r&r&r'rsrmsgcCs(tjrt|dur ddt|dS)zaRaise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_EVALUATE_OFFLINE is True.NzOffline mode is enabled.zOffline mode is enabled. )rHF_EVALUATE_OFFLINErr)rr&r&r'!_raise_if_offline_mode_is_enableds  r? func_args func_kwargs exceptions status_codesrKbase_wait_time max_wait_timec Cs|pd}|pi}d} z||i|WS|yU} z6||ks'|r)| jj|vr)| t||d|} t|d| d||dt| |d7}WYd} ~ nd} ~ wwq ) Nr&rTrz timed out, retrying in zs... []r)response status_codeminloggerinfotimesleep) funcrrrrrKrrretryerr sleep_timer&r&r'_retrys    r$@methodtimeoutc Kstd|d\}}|sd|d7}ztjd |||d|} d}Wn>tjjtjjfya} z,||kr7| t|d|d||dt ||d |d} t | WYd } ~ nd } ~ ww|r | S) aWrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. Note that if the environment variable HF_EVALUATE_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. Args: method (str): HTTP method, such as 'GET' or 'HEAD'. url (str): The URL of the resource to fetch. max_retries (int): Maximum number of retries, defaults to 0 (no retries). base_wait_time (float): Duration (in seconds) to wait before retrying the first time. Wait time between retries then grows exponentially, capped by max_wait_time. max_wait_time (float): Maximum amount of time between two retries, in seconds. **params: Params to pass to :obj:`requests.request`. Tried to reach )rFr)rrjrTz request to z timed out, retrying... [rrNr&) rrequestsrequestupperrConnectTimeoutConnectionErrorrrrrr) rrjrKrrrparamstriessuccessrrrr&r&r'_request_with_retrys"  rcCsltd|z$ttjj||d}|dWdWdS1s$wYWdSty5YdSw)NrrrFT)rrurllibrurlopenread Exception)rjrrr&r&r'ftp_head9s   rc Cstd|z1td|d|jttjj||d}t ||WdWdS1s1wYWdStj j yK}zt |dd}~ww)NrzGetting through FTP z into r) rrrrNrrrrshutil copyfileobjerrorURLErrorr)rj temp_filerrer&r&r'ftp_getCs& rY@c  Cst|pi}t|dd|d<|dkrd|dd|d<td|d |||||d } | jd kr1dS| jd } | durA|t| nd} tj d d | ||pLdt d} | j ddD]} | t | || qZWddS1stwYdS)N user-agentrurzbytes=d-RangeGETT)rrjstreamrsrcookiesrKrizContent-LengthB Downloading)unit unit_scaletotalinitialdescdisablei) chunk_size)copydeepcopyrgetrrrintrtqdmis_progress_bar_enabled iter_contentupdatelenwrite)rjrrs resume_sizerrrrKrrcontent_lengthrprogresschunkr&r&r'http_getMs@   "rc Cs>t|pi}t|dd|d<td|||||||d}|S)NrrHEAD)rrjrsrrallow_redirectsrrK)rrrrr)rjrsrrrrrKrr&r&r'rLos rLcCs@t||d}t||dd}||jr|jd}|Sd}|S)Nrx)rrKETag)rrLraise_for_statusokrr)rjrxrrrkr&r&r' request_etags rdc  s~|durtj}t|trt|}tj|ddd} d} d}d}d}t|dd}tj ||}tj |r:|s:|s:|St || d}|durH||d<|s| drTt |} zt|d||| |d } | jd kr|rl| jd nd}| jD]\}}| d rd |vr|d|7}| j}qsd} d |vrd|vr|d7}nN| jdkrd|vs| jdkrd |vs| jdkrtd|std| js| jdkrd|vrd} td|n| jdkrtj|vr| durtd|dWnttjjfy}z|}WYd}~nd}~ww| s_tj |r|s|S|rtd|d| dur.| jdkr.td|t d ||durGtd!|d"t!|d#| durXtd!|d$| jd#td!|t||}tj ||}tj |rw|sw|S|d%}t"||r|d&t#fd'd(}|}tj rt$j%}n d)}n t&t'j(|dd*}d)}|+}t|d+|j)| drt*||n t+||||||| | d,Wdn 1swYtd-|d.|t,-|j)|td/|||d0}|d1}t.|d2d3d4}t/0||Wdn1swYWd|SWd|S1s8wY|S)5a Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the path to the cached file. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk NTrF)rkrrzftp://)rrsrrKrrdownload_warningzdrive.google.comz &confirm=zconfirm=z &confirm=tizfirebasestorage.googleapis.comiiz7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url izUnauthorized for URL zY. Please use the parameter ``token=True`` after logging in with ``huggingface-cli login``z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.izCouldn't find file at rzCouldn't reach z ()z (error z.lockrc3s8td }|VWddS1swYdS)Nza+b)r%)fincomplete_pathr&r'_resumable_file_managers "z/get_from_cache.._resumable_file_managerr)dirdeletezB not found in cache or force_download set to True, downloading to )rsrrrrKrzstoring z in cache at zcreating metadata file for )rjrkz.jsonrre)encoding)1rHF_EVALUATE_CACHEr<rrr!r"rprr$r#rrrrLrrrrrrematchrjrrrrOSErrorrrTimeoutr}rreprrrstatst_sizertempfileNamedTemporaryFilerNrrrmover%jsondump)rjrqrrrs etag_timeoutrtrurvrwrKrxry connectedrrrk head_errorr@ cache_pathrrrr lock_pathrtemp_file_managerrrmeta meta_path meta_filer&rr'r|s                    0 00r|cfdd}|S)Ncs(dd|jdur|jnd|_|SNr3z )r$__doc__fndocstrr&r'docstring_decorator@$z1add_start_docstrings..docstring_decoratorr&r"r#r&r!r'add_start_docstrings? r&cr)Ncs(|jdur|jnddd|_|Sr)rr$rr!r&r'r#Hr$z/add_end_docstrings..docstring_decoratorr&r%r&r!r'add_end_docstringsGr'r(cCstdd|DS)Ncss|]}|jVqdSN)r r )r[rr&r&r'r]Psz(estimate_dataset_size..)sum)pathsr&r&r'estimate_dataset_sizeOsr,rcCs>t} |d}|s t|S||7}|dr t|Sq)NTr ) bytearrayrribytes)rresbr&r&r'readlineSs  r2r))FT)FTr)rrrr)r)NrNNrrN)NNNTrr) NFNrFNFTrNN)Urriorr!r^rrrrrr contextlibrr functoolsrhashlibrpathlibrtypingrrr r r urllib.parser rdatasetsr datasets.utils.extractrdatasets.utils.filelockrr3rrr get_loggerrrINCOMPLETE_SUFFIXrrr(boolr2r6r8r>rJResponserrMrTr`rdrprrrrrrrrRequestExceptiontuplerfloatrrrrrrLrr|r&r(r, RawIOBaser2r&r&r&r's                C$      '  # (  7