o
    civ                   	   @   s   d dl Z d dlZd dlZd dlmZmZmZmZmZ d dl	m
Z
mZ er(d dlZdedeee  defddZ	dd	eeee f d
ddeee df fddZdddefddZdd ZdefddZdefddZdedefddZdd Zdd ZdS )    N)TYPE_CHECKINGListOptionalTupleUnion)RetryingPyFileSystem_resolve_custom_schemepath
extensionsreturnc                    sN   |du st |tsJ t||du rdS dd |D }t fdd|D S )a  Check if a path has a file extension in the provided list.

    Examples:
        >>> _has_file_extension("foo.csv", ["csv"])
        True
        >>> _has_file_extension("foo.CSV", ["csv"])
        True
        >>> _has_file_extension("foo.csv", ["json", "jsonl"])
        False
        >>> _has_file_extension("foo.csv", None)
        True

    Args:
        path: The path to check.
        extensions: A list of extensions to check against. If `None`, any extension is
            considered valid.
    NTc                 S   s   g | ]	}d |   qS ).)lower.0ext r   Q/home/ubuntu/.local/lib/python3.10/site-packages/ray/data/datasource/path_util.py
<listcomp>$   s    z'_has_file_extension.<locals>.<listcomp>c                 3   s    | ]
}   |V  qd S N)r   endswithr   r	   r   r   	<genexpr>%   s    z&_has_file_extension.<locals>.<genexpr>)
isinstancelisttypeany)r	   r
   r   r   r   _has_file_extension   s
   r   paths
filesystemzpyarrow.fs.FileSystemc                 C   s  ddl }ddlm}m}m}m} t| tr| g} t| tj	r$t| g} n t| t
r2tdd | D r:td|  dt| dkrDtd|r{t||s{d	| }zddl}dd
lm}	 W n tyh   t|dw t||jjsut|d|||}g }
| D ]}t|}z	|||\}}W n` |jjy } zRdt|v r|t||\}}t|}n8dt|v rtjj|ddj}|dv rzddl}dd
lm}	 W n ty   tddw |||	 }|}n  W Y d}~nd}~ww |du r|}t|st |}|!|}|
"| q|
|fS )at  
    Resolves and normalizes all provided paths, infers a filesystem from the
    paths and ensures that all paths use the same filesystem.

    Args:
        paths: A single file/directory path or a list of file/directory paths.
            A list of paths can contain both files and directories.
        filesystem: The filesystem implementation that should be used for
            reading these files. If None, a filesystem will be inferred. If not
            None, the provided filesystem will still be validated against all
            filesystems inferred from the provided paths to ensure
            compatibility.
    r   N)
FileSystemFSSpecHandlerPyFileSystem_resolve_filesystem_and_pathc                 s   s    | ]	}t |t V  qd S r   )r   str)r   pr   r   r   r   E   s    z0_resolve_paths_and_filesystem.<locals>.<genexpr>zIExpected `paths` to be a `str`, `pathlib.Path`, or `list[str]`, but got `z`.zMust provide at least one path.zThe filesystem passed must either conform to pyarrow.fs.FileSystem, or fsspec.spec.AbstractFileSystem. The provided filesystem was: HTTPFileSystemzCannot parse URIz#Unrecognized filesystem type in URIFallow_fragmentshttphttpsz.Please install fsspec to read files from HTTP.)#pyarrow
pyarrow.fsr   r    r!   r"   r   r#   pathlibPathr   r   
ValueErrorlenfsspecfsspec.implementations.httpr&   ModuleNotFoundError	TypeErrorspecAbstractFileSystemr   libArrowInvalid_encode_url_decode_urlurllibparseurlparseschemeImportError_is_http_filesystem_unwrap_protocolnormalize_pathappend)r   r   par   r    r!   r"   err_msgr2   r&   resolved_pathsr	   resolved_filesystemresolved_pather?   r   r   r   _resolve_paths_and_filesystem(   s   




rK   fsc                 C   sx   ddl m}m} zddl}ddlm} W n
 ty   Y dS w t| tr(| 	 } t| |s/dS t| j
|o;t| j
j|S )zKReturn whether ``fs`` is a PyFileSystem handled by a fsspec HTTPFileSystem.r   )r    r!   Nr%   F)r-   r    r!   r2   r3   r&   r4   r   r   unwraphandlerrL   )rL   r    r!   r2   r&   r   r   r   rA      s   

rA   c                 C   s   t jdkrt| rt|  S tjj| dd}|j	r d|j	 nd}|j
}|jdkr7d|j
v r7|j
dd }|j}t jdkra|sat|d	kra|d
 dkra|d  ra|dd dv ra|dd }|| | S )z2
    Slice off any protocol prefixes on path.
    win32Fr'   ? s3@   r   /         ):z:/N)sysplatform_is_local_windows_pathr.   r/   as_posixr<   r=   r>   querynetlocr?   splitr	   r1   isalpha)r	   parsedr_   r`   parsed_pathr   r   r   rB      s$   

rB   c                 C   s   t j| jdkS )NrQ   r<   r=   r>   r?   r   r   r   r   _is_url   s   rf   c                 C   s   t j| }|jdv S )Nr)   re   )r	   rc   r   r   r   _is_http_url   s   
rg   c                 C   sn   t jdkrdS t| dkr| d dkrdS t| dkr5| d dkr5| d	 d
ks-| d	 dkr5| d  r5dS dS )z5Determines if path is a Windows file-system location.rO   FrW   r   \TrU   rZ   rX   rV   )r[   r\   r1   rb   r   r   r   r   r]      s   

r]   c                 C   s   t jj| ddS )Nz/:)safe)r<   r=   quoter   r   r   r   r:      s   r:   c                 C   s   t j| S r   )r<   r=   unquoter   r   r   r   r;      s   r;   r   )r.   r[   r<   typingr   r   r   r   r   ray.data._internal.utilr   r   r,   r#   boolr   rK   rA   rB   rf   rg   r]   r:   r;   r   r   r   r   <module>   s.    
k!