o
    8wi                     @   s   d dl Z d dlmZ d dlmZmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddl m!Z! ddl"m#Z# e!e$Z%erbdd	l&m'Z' dd
ee fddZ(dddZ)dS )    N)wraps)TYPE_CHECKINGOptional   )DownloadConfig)	xbasenamexdirname	xet_parsexexistsxgetsizexglob
xgzip_openxisdirxisfilexjoinxlistdirxnumpy_loadxopenxpandas_read_csvxpandas_read_excelxPathxpyarrow_parquet_read_tablexrelpathxsio_loadmatxsplit	xsplitextxwalkxxml_dom_minidom_parse)
get_logger)patch_submoduleDatasetBuilderdownload_configc                    s$  t | }t|dr|jrt|jtr j|j_ j|j_dS  fdd}t|d|t	
  t|d|t
  t|d|t
  t|d|t
  t|d	t
  t|d
t
  t|dt
  t|dt
  t|dt
  t|dt
  t|d|t
  t|d|t
  t|d|t
  t|d|t
  t|dt
  t|d|t
  t|d|t
  t|d|tdgd
  t|d|tdgd
  t|d|tdgd
  t|d|t
  t|d|t
  |j !dst|d|t"
   |_dS )a?  Extend the module to support streaming.

    We patch some functions in the module to use `fsspec` to support data streaming:
    - We use `fsspec.open` to open and read remote files. We patch the module function:
      - `open`
    - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module
      functions:
      - `os.path.join`
      - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator)

    The patched functions are replaced with custom functions defined to work with the
    :class:`~download.streaming_download_manager.StreamingDownloadManager`.

    Args:
        module_path: Path to the module to be extended.
        download_config: Mainly use `token` or `storage_options` to support different platforms and auth types.
    _patched_for_streamingNc                    s    t   fdd}d|_|S )Nc                     s   | d i|S )Nr"    )argskwargs)r"   functionr$   O/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/datasets/streaming.pywrapperG   s   z?extend_module_for_streaming.<locals>.wrap_auth.<locals>.wrapper	wrap_auth)r   _decorator_name_)r'   r)   r"   )r'   r(   r*   F   s   z.extend_module_for_streaming.<locals>.wrap_authopenz
os.listdirzos.walkz	glob.globzos.path.joinzos.path.dirnamezos.path.basenamezos.path.relpathzos.path.splitzos.path.splitextzos.path.existszos.path.isdirzos.path.isfilezos.path.getsizezpathlib.Pathz	gzip.openz
numpy.loadzpandas.read_csv__version__)attrszpandas.read_excelzscipy.io.loadmatzxml.etree.ElementTree.parsezxml.dom.minidom.parsezdatasets.packaged_modules.zpyarrow.parquet.read_table)#	importlibimport_modulehasattrr#   
isinstancer   tokenstorage_optionsr   r   startr   r   r   r   r   r   r   r   r   r
   r   r   r   r   r   r   r   r   r   r	   r   __name__
startswithr   )module_pathr"   moduler*   r$   r,   r(   extend_module_for_streaming*   s@   


	
r;   builderr!   c                    sd   t | j| jd}t| j|d ddlm   fddt| jdd D }|D ]}t||d q'dS )zExtend the dataset builder module and the modules imported by it to support streaming.

    Args:
        builder (:class:`DatasetBuilder`): Dataset builder instance.
    )r5   r4   r,   r   r    c                    s(   g | ]}t | r|j jkr|jqS r$   )
issubclass
__module__).0clsr    r$   r(   
<listcomp>}   s    z8extend_dataset_builder_for_streaming.<locals>.<listcomp>N)	r   r5   r4   r;   r>   r<   r!   type__mro__)r<   r"   parent_builder_modulesr:   r$   r    r(   $extend_dataset_builder_for_streamingn   s   
rE   )N)r<   r!   )*r0   	functoolsr   typingr   r   download.download_configr   utils.file_utilsr   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utils.loggingr   utils.patchingr   r7   loggerr<   r!   r;   rE   r$   r$   r$   r(   <module>   s    dD