o 8wiq@sddlZddlZddlmZddlmZmZddlmZm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'm(Z(ddl)m*Z*dd l+m,Z,e(e-Z.Gd d d Z/dS) N)Iterable)OptionalUnion)!SINGLE_FILE_COMPRESSION_PROTOCOLSArchiveIterable FilesIterable_get_extraction_protocol_get_path_extension!_prepare_path_and_storage_optionsis_relative_pathurl_or_path_join xbasenamexdirname xet_parsexexistsxgetsizexglob xgzip_openxisdirxisfilexjoinxlistdir xnumpy_loadxopenxpandas_read_csvxpandas_read_excelxPathxpyarrow_parquet_read_tablexrelpath xsio_loadmatxsplit xsplitextxwalkxxml_dom_minidom_parse) get_logger) map_nested)DownloadConfigc @seZdZdZdZ    d"deedeedeedeefdd Ze d d Z d d Z dedefddZ ddZ dedefddZddZdeeejfdeefddZdeeeefdeefddZddZd d!ZdS)#StreamingDownloadManagera Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives. Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract data, but they rather return the path or url that could be opened using the `xopen` function which extends the built-in `open` function to stream data from remote files. TN dataset_namedata_dirdownload_config base_pathcCs:||_||_|p tjd|_|pt|_d|_d|_ dS)N.F) _dataset_name _data_dirospathabspath _base_pathr(r,downloaded_sizerecord_checksums)selfr*r+r,r-r8i/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/datasets/download/streaming_download_manager.py__init__9s   z!StreamingDownloadManager.__init__cCs|jSN)r0r7r8r8r9 manual_dirGsz#StreamingDownloadManager.manual_dircCst|j|dd}|S)aUNormalize URL(s) of files to stream data from. This is the lazy version of `DownloadManager.download` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') ``` T map_tuple)r&_download_singler7 url_or_urlsr8r8r9downloadKsz!StreamingDownloadManager.downloadurlpathreturncCs t|}t|rt|j|}|Sr;)strr r r4)r7rDr8r8r9r@_s z)StreamingDownloadManager._download_singlecCst|j|dd}|S)aAdd extraction protocol for given url(s) for streaming. This is the lazy version of `DownloadManager.extract` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> extracted_files = dl_manager.extract(downloaded_files) ``` Tr>)r&_extract)r7rBurlpathsr8r8r9extractfsz StreamingDownloadManager.extractcCst|}t||jd}|dd}t|}|dvs|dr'td|d|dur-|S|tvrUtj |dd}d|vrI|d| dn|}|d |d|S|d |S) Nr,z::r)tgztar)z.tar.gzz.tar.bz2z.tar.xzz+Extraction protocol for TAR archives like 'z' is not implemented in streaming mode. Please use `dl_manager.iter_archive` instead. Example usage: url = dl_manager.download(url) tar_archive_iterator = dl_manager.iter_archive(url) for filename, file in tar_archive_iterator: ...r.z://z://::) rFr r,splitr endswithNotImplementedErrorrr1r2basenamerindex)r7rDprotocolr2 extension inner_filer8r8r9rG|s  z!StreamingDownloadManager._extractcCs|||S)a0Prepare given `url_or_urls` for streaming (add extraction protocol). This is the lazy version of `DownloadManager.download_and_extract` for streaming. Is equivalent to: ``` urls = dl_manager.extract(dl_manager.download(url_or_urls)) ``` Args: url_or_urls (`str` or `list` or `dict`): URL(s) to stream from data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. )rIrCrAr8r8r9download_and_extractsz-StreamingDownloadManager.download_and_extracturlpath_or_bufcCs$t|dr t|Stj||jdS)aNIterate over files within an archive. Args: urlpath_or_buf (`str` or `io.BufferedReader`): Archive path or archive binary file object. Yields: `tuple[str, io.BufferedReader]`: 2-tuple (path_within_archive, file_object). File object is opened in binary mode. Example: ```py >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> files = dl_manager.iter_archive(archive) ``` readrJ)hasattrrfrom_buf from_urlpathr,)r7rVr8r8r9 iter_archives  z%StreamingDownloadManager.iter_archiverHcCstj||jdS)aIterate over files. Args: urlpaths (`str` or `list` of `str`): Root paths. Yields: str: File URL path. Example: ```py >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') >>> files = dl_manager.iter_files(files) ``` rJ)r from_urlpathsr,)r7rHr8r8r9 iter_filessz#StreamingDownloadManager.iter_filescCdSr;r8r<r8r8r9manage_extracted_filesz/StreamingDownloadManager.manage_extracted_filescCr^r;r8r<r8r8r9get_recorded_sizes_checksumsr`z5StreamingDownloadManager.get_recorded_sizes_checksums)NNNN)__name__ __module__ __qualname____doc__ is_streamingrrFr(r:propertyr=rCr@rIrGrUrioBufferedReaderrtupler[listr]r_rar8r8r8r9r)/s6   " r))0rhr1collections.abcrtypingrrutils.file_utilsrrrr r r r r rrrrrrrrrrrrrrrrrrr r!r"r#r$ utils.loggingr%utils.py_utilsr&r,r(rbloggerr)r8r8r8r9s  !