o
    'i6                    @   s
  d dl m Z mZmZ d dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZmZmZ d dlmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z# ej$%ej$&e'Z(G dd	 d	eZ)ej*d
d Z+ej*dd Z,ej*dd Z-ej*dd Z.ej*dd Z/ej*dd Z0ej*dd Z1ej*dd Z2ej*dd Z3dZ4ej*dd Z5ej*dd  Z6ej*d!d" Z7ej*d#d$ Z8ej*d%d& Z9ej*ej:dd'd(ej:dd)d(ej:dd*d(ej:dd+ej;j<d,ej:dd-ej;j=d,ej:dd.ej;j>d,ej:d d/ej;j6d,ej:dd0d(ej:dd1d(ej:dd2d(ej:d"d3d(ej:d$d4d(ej:d&d5ej;j<d,gd6d7d8 Z?ej*d9d: Z@ej*d;d< ZAej*d=d> ZBej*d?d@ ZCdAdB ZDdCdD ZEdEdF ZFdGdH ZGdIdJ ZHej;j<dKdL ZIdMdN ZJdOdP ZKdQdR ZLdSdT ZMdUdV ZNdWdX ZOdYdZ ZPd[d\ ZQd]d^ ZRd_d` ZSdadb ZTdcdd ZUdedf ZVdgdh ZWdidj ZXdkdl ZYdmdn ZZdodp Z[dqdr Z\dsdt Z]dudv Z^dwdx Z_dydz Z`ej;jej;ad{dde`fdd|e`fd}dejbfd}d~ejbfgdd Zcdd Zddd Zeej;jej;addde`fdd|e`fd}dejffd}d~ejffgdd Zgej;jej;addde`e`fdd|e`e`fd}dejbejffd}d~ejbejffgdd Zhdd Zidd Zjdd Zkdd Zldd Zmej;j=dd Znej;j<dd Zoej;j<dd Zpej;j<dd Zqej;j>dd Zrej;j6dd Zsej;addedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfgdd Ztdd Zuej;addddde'gdd Zvej;j<dd Zwej;j=dd Zxdd Zydd ZzddÄ Z{ddń Z|ddǄ Z}ddɄ Z~dd˄ Zdd̈́ Zddτ Zddф Zddӄ Zej;j<ddՄ Zej;j<ddׄ Zej;j<ddل Zej;j<ddۄ Zdd݄ Zej;j<dd߄ Zej;j<dd Zej;jej;jej;j<dd Zej;j<ej;jddej;je dddd Zdd Zdd Zdd ZdS )    )datetimetimezone	timedeltaN)urlopen)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_userrunning_on_musllinux)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_files)find_free_portc                   @   s   e Zd Zd'ddZdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&S )(DummyHandler*   c                 C   s
   || _ d S N)_value)selfvalue r   S/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/pyarrow/tests/test_fs.py__init__/   s   
zDummyHandler.__init__c                 C   s   t |tr| j|jkS tS r   
isinstancer   r   NotImplementedr   otherr   r   r   __eq__2      
zDummyHandler.__eq__c                 C   s   t |tr| j|jkS tS r   r    r#   r   r   r   __ne__7   r&   zDummyHandler.__ne__c                 C      dS )Ndummyr   r   r   r   r   get_type_name<      zDummyHandler.get_type_namec                 C   s   |S r   r   r   pathr   r   r   normalize_path?   r,   zDummyHandler.normalize_pathc                 C   s~   g }|D ]8}d|v r| t|tj qd|v r"| t|tj qd|v r0| t|tj qd|v r;| t  qt|S )Nfiledirnotfoundbadtype)appendr   r   File	DirectoryNotFoundobjectIOError)r   pathsinfor.   r   r   r   get_file_infoB   s   zDummyHandler.get_file_infoc                 C   sZ   |j dkr|jr
g S t|j tdtjddtdtjg}|jr+|tdtjddg7 }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr   r   r5   r6   	recursive)r   selectorinfosr   r   r   get_file_info_selectorR   s   


z#DummyHandler.get_file_info_selectorc                 C   s4   |dkr|du s
J d S |dkr|du sJ d S t )NrF   Tnon-recursiveF)r9   )r   r.   rF   r   r   r   
create_dirb   s
   zDummyHandler.create_dirc                 C      |dksJ d S )N
delete_dirr   r-   r   r   r   rM   j      zDummyHandler.delete_dirc                 C   s   | dst|dksJ d S )N/delete_dir_contents)strip
ValueError)r   r.   missing_dir_okr   r   r   rP   m   s   
z DummyHandler.delete_dir_contentsc                 C   s   d S r   r   r*   r   r   r   delete_root_dir_contentsr   r,   z%DummyHandler.delete_root_dir_contentsc                 C   rL   )Ndelete_filer   r-   r   r   r   rU   u   rN   zDummyHandler.delete_filec                 C      |dksJ |dksJ d S )N	move_frommove_tor   r   srcdestr   r   r   movex      zDummyHandler.movec                 C   rV   )Ncopy_file_fromcopy_file_tor   rY   r   r   r   	copy_file|   r]   zDummyHandler.copy_filec                 C   *   d|v rt || dd}t|S )Nr2   z:input_streamutf8rE   encodepaBufferReaderr   r.   datar   r   r   open_input_stream      
zDummyHandler.open_input_streamc                 C   ra   )Nr2   z:input_filerb   rc   rg   r   r   r   open_input_file   rj   zDummyHandler.open_input_filec                 C      d|v rt |t S Nr2   rE   re   BufferOutputStreamr   r.   metadatar   r   r   open_output_stream      zDummyHandler.open_output_streamc                 C   rl   rm   rn   rp   r   r   r   open_append_stream   rs   zDummyHandler.open_append_streamN)r   )__name__
__module____qualname__r   r%   r'   r+   r/   r<   rI   rK   rM   rP   rT   rU   r\   r`   ri   rk   rr   rt   r   r   r   r   r   .   s&    
r   c                    s   t t  fdddddS )Nc                        |    S r   as_posixptempdirr   r   <lambda>       zlocalfs.<locals>.<lambda>Tfspathfnallow_move_dirallow_append_to_filedictr   requestr~   r   r}   r   localfs   s   
r   c                    s"   t ttt  fdddddS )Nc                    rx   r   ry   r{   r}   r   r   r      r   zpy_localfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   r}   r   
py_localfs   s   
r   c                 C   s   t t dd dddS )Nc                 S      | S r   r   r{   r   r   r   r          zmockfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   mockfs   s   r   c                 C   s   t ttt dd dddS )Nc                 S   r   r   r   r{   r   r   r   r      r   zpy_mockfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   r   r   	py_mockfs   s   r   c                    s   t tdd fdddddS )NTuse_mmapc                    rx   r   ry   r{   r}   r   r   r      r   z#localfs_with_mmap.<locals>.<lambda>r   r   r   r   r}   r   localfs_with_mmap   s   
r   c                 C   s"   t tt||d dd dddS )Nr   c                 S   r   r   r   r{   r   r   r   r      r   z!subtree_localfs.<locals>.<lambda>Tr   )r   r   str)r   r~   r   r   r   r   subtree_localfs   s   r   c              
   c   s    | j jd ddlm} |d \}}d}|| d| ddtd	d
dd}z|| W n tyK } zt	d| d|  W Y d }~nd }~ww t
||jdddV  || d S )Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousretry_time_limit
project_idzCould not create directory in : Fr   )configpyarrowrequires
pyarrow.fsr   r   rK   OSErrorpytestskipr   __add__rM   )r   
gcs_serverr   hostportbucketr   er   r   r   gcsfs   s2   "r   c           	      c   sz    | j jd ddlm} |d \}}}}d}|||| d| dddd	}|| t||jd
d
dV  || d S )Ns3r   S3FileSystemr   r   r   r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFr   )	r   r   r   r   r   rK   r   r   rM   )	r   	s3_serverr   r   r   r   r   r   r   r   r   r   s3fs   s*   
r   c                 C   s    d}t t||d |jdddS )Nzpyarrow-filesystem/prefix/r   Fr   )r   r   r   )r   r   prefixr   r   r   subtree_s3fs  s   r   a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]

        },
        {
            "Effect": "Deny",
            "Action": [
                "s3:DeleteObject"
            ],
            "Resource": [
                "arn:aws:s3:::no-delete-bucket*"
            ]
        }
    ]
}c                 c   s    | j jd ddlm} |d \}}}}| d| }d}d}	|||||||d}
|
|	 t|
|	jd	d	d
V  |
|	 d S )Nazurer   AzureFileSystemr   r   r   r   account_nameaccount_keyblob_storage_authoritydfs_storage_authorityblob_storage_schemedfs_storage_schemeTr   )	r   r   r   r   r   rK   r   r   rM   )r   azure_serverr   r   r   r   r   azurite_authorityazurite_scheme	containerr   r   r   r   azurefs7  s,   
r   c                 C   sX   | j jd t std ddlm} |\}}}||||d}t	|dd ddd	S )
NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                 S   r   r   r   r{   r   r   r   r   a  r   zhdfs.<locals>.<lambda>Tr   )
r   r   r   re   have_libhdfsr   r   r   r   r   )r   hdfs_connectionr   r   r   r   r   r   r   r   r   T  s   

r   c                    s4   t d}|d}ttt| fdddddS )Nfsspecr0   c                    rx   r   ry   r{   r}   r   r   r   m  r   z#py_fsspec_localfs.<locals>.<lambda>Tr   )r   importorskip
filesystemr   r   r   r   r~   r   r   r   r}   r   py_fsspec_localfsg  s   



r   c                 C   sH   t jddd}|jdkrt d |d}ttt|dd d	d	d
S )Nr   z0.7.5)
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemmemoryc                 S   r   r   r   r{   r   r   r   r   |  r   z$py_fsspec_memoryfs.<locals>.<lambda>Tr   )r   r   __version__r   r   r   r   r   r   r   r   r   py_fsspec_memoryfss  s   



r   c           	      c   sz    t d}|d \}}}}d}|j||td| d| dd}tt|}|| t||jdd	d
V  || d S )Nr   r   r   zhttp://r   )endpoint_url)keysecretclient_kwargsFTr   )	r   r   r   r   r   r   rK   r   rM   )	r   r   r   r   r   r   r   r   r   r   r   r   py_fsspec_s3fs  s$   

r   zLocalFileSystem())idzLocalFileSystem(use_mmap=True)z$SubTreeFileSystem(LocalFileSystem())r   )r   marksr   r   r   z_MockFileSystem()z-PyFileSystem(ProxyHandler(LocalFileSystem()))z-PyFileSystem(ProxyHandler(_MockFileSystem()))z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 C   s   |  | jS r   )getfixturevalueparamr   r   r   r   filesystem_config  s   <r   c                 C      | d S Nr   r   r   r   r   r   r        r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   sz   t | jtsJ t | jtsJ | jdksJ | jt| j d ks&J | jj}|d us0J |	d t
dks;J d S )Nr   g    eA)r!   mtimer   mtime_nsintr   approx	timestamptzinfo	utcoffsetr   )	file_infor   r   r   r   check_mtime  s   
r   c                 C   s    | j d u sJ | jd u sJ d S r   )r   r   r   r   r   r   check_mtime_absent  s   r   c                 C   s"   | j d u rt|  d S t|  d S r   )r   r   r   r   r   r   r   check_mtime_or_absent  s   
r   c                 C   s   | j dkrtjdd d S d S )Npy::fsspec+('s3', 's3a')zNot working with fsspec's s3fsreason)	type_namer   xfailr   r   r   r   skip_fsspec_s3fs     
r   c                 C   s   | j dkrtj|d d S d S )Nabfsr   )r   r   r   )r   r   r   r   r   
skip_azure	  r   r  c              	   C   s  ddl m} t| tdd | d \}}}}|dd| d| dd}|d	 tjtjd
d |d W d    n1 s?w   Y  tjtjdd |	d W d    n1 s\w   Y  tjt
dd |dd W d    n1 syw   Y  tjtjdd" |d}|d W d    n1 sw   Y  W d    n1 sw   Y  |d}|d W d    n1 sw   Y  tjt
dd |d W d    d S 1 sw   Y  d S )Nr   r   test_fs_limited_user
limited123r   r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucketzRequest ID:zexisting-bucket/test-filezexisting-bucket/test-file-copyznon-existing-bucket/test-file   testzno-delete-bucket/test-file)r   r   r	   _minio_limited_policyrK   r   raisesre   ArrowIOErrorrM   r   r`   rr   writerU   )r   r   r   r   _r   fr   r   r   +test_s3fs_limited_permissions_create_bucket  sB   
"r  c                  C   s,  t dtj} td}|jdksJ |jdksJ |jtj	ks!J |j
d u s(J t| tdtjddd}|jdks<J |jdksCJ |jtjksKJ |j
dksRJ |jd	ksYJ t| td
tj| d}|jd
kslJ |jd
kssJ |jtjks{J |j
d u sJ |j| ksJ |jdksJ t| d S )NR]foo/barbarzfoo/baz.txtr?   g  `A)typer@   r   zbaz.txtl    t<|+. foo)r  r   l    4 |+. )r   fromtimestampr   utcr   r.   	base_namer  r   Unknownr@   r   r5   r   r   r6   r   )dtr;   r   r   r   test_file_info_constructor2  s0   
r  c                   C   s6   t t t  W d    d S 1 sw   Y  d S r   )r   r	  	TypeErrorr   r   r   r   r   'test_cannot_instantiate_base_filesystemO  s   "r  c                  C   s   t  } t  }t }| | sJ | |sJ tt | d W d    n1 s,w   Y  | |   kr<|ks?J  J | dksEJ ||ksKJ |t ksRJ td| td| ks^J td| td|ksjJ td| td| ksvJ d S )Nstring   /basez/other)r   r   equalsr   r	  r  r   )fs0fs1fs2r   r   r   test_filesystem_equalsT  s   r#  c                 C   sH   t jtdd | d  W d    n1 sw   Y  | d us"J d S )Nzgot NoneTyper  )r   r	  r  r  r   r   r   r   test_filesystem_equals_noneh  s   r$  c                  C   s|   t  } td| }|jdksJ |j| ksJ t|dsJ tdt  }|jdks,J |j| ks3J t|ds<J d S )Nr  z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathbase_fsrepr
startswith)r   subfsr   r   r   test_subtree_filesystemo  s   
r*  c                 C   sT   | j dd dkrtjdd || }||}t|ts!J || s(J d S )N::mock"MockFileSystem is not serializabler   )	r   splitr   r   dumpsloadsr!   r   r  )r   pickle_module
serializedrestoredr   r   r   test_filesystem_pickling  s   

r5  c                 C   s  | j dd dkrtjdd t|  |d}|d}|d}| | | | W d    n1 s5w   Y  | |}|d	 W d    n1 sOw   Y  ||	| }|
|||g\}}	}
|jtjksoJ |	jtjkswJ |
jtjksJ d S )
Nr+  r,  r-  r.  r   	a/aa/aaa/a/bbc.txtr  )r   r/  r   r   r   rK   rr   r  r1  r0  r<   r  r   r6   r5   )r   r   r2  aaabbcfpr4  aaa_infobb_infoc_infor   r   r   ,test_filesystem_is_functional_after_pickling  s$   
r@  c                  C   s,   t  } | jdks
J t } | jdksJ d S )Nlocalr-  )r   r   r   r   r   r   r   test_type_name  s   rB  c                 C   s   |  ddks	J d S )Nr  )r/   r   r   r   r   test_normalize_path  s   rC  c              	   C   sp   G dd d}dd| t  i g dd t g}|D ]}tt | | W d    n1 s0w   Y  qd S )Nc                   @   s   e Zd ZdS )z-test_non_path_like_input_raises.<locals>.PathN)ru   rv   rw   r   r   r   r   Path  s    rD     g?c                   S   r(   )NrE  r   r   r   r   r   r     r   z1test_non_path_like_input_raises.<locals>.<lambda>)tuplepathlibrD  r   r	  r  rK   )r   rD  invalid_pathsr.   r   r   r   test_non_path_like_input_raises  s   rI  c                 C   s  |d}|d}|d}|d}|  | | | W d    n1 s%w   Y  | |}|d W d    n1 s?w   Y  | ||||g\}}}	}
|j|ksXJ dt|v s`J |jdksgJ | jdkru|jt	j
kstJ n|jt	jks}J d	t|v sJ |jd u sJ t| |jt|ksJ |jd
ksJ |jdksJ |jt	jksJ dt|v sJ |jdksJ | jdvrt| |	jt|ksJ |	jdksJ |	jdksJ |	jt	jksJ dt|	v sJ |	jdksJ | jdvrt|	 |
jt|ksJ |
jdksJ |
jdksJ |
jt	j
ks!J |
jd u s)J |
jd u s1J dt|
v s:J t|
 | |}|j|jksLJ |j|jksUJ d S )Nr6  r7  r8  zzzr  r9   r   zFileType.Directoryr:  zFileType.Filer   )zpy::fsspec+memoryr   txtr  zFileType.NotFound)rK   rr   r  r<   r.   r'  	extensionr   r  r   r7   r6   r@   r   r   r  r5   r   r   r   )r   r   r9  r:  r;  rJ  r<  r=  r>  r?  zzz_info	aaa_info2r   r   r   test_get_file_info  s`   




rP  c              	   C   s  |d}|d}|d}|d}|d}|d}z|  | | | W d    n1 s.w   Y  | | W d    n1 sCw   Y  |  | | | W d    n1 s]w   Y  |  | t|ddd	}|j|ksuJ | |}	| jd
krt|	dk nt|	dksJ |	D ]D}
|
j|s|
j|s|
j|r|
j	t
jksJ n#|
jd|s|
jd|r|
j	t
jksJ ntd|
j t|
 qt|dd}| |}	| jd
krt|	dksJ nt|	dksJ W | | d S W | | d S | | w )Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)rD   rF   r   r     rO   zunexpected path rF      )rK   rr   r   rC   r<   r   lenr.   endswithr  r   r5   rstripr6   rR   r   rM   )r   r   rC   file_afile_bdir_afile_cdir_brG   rH   r;   r   r   r    test_get_file_info_with_selector  s\   








r\  c                 C   s~   t |  |d}ttj | | W d    n1 sw   Y  | | | | |d}| j|dd | | d S )Nztest-directory/zdeeply/nested/test-directory/TrR  )r   r   r	  re   r
  rM   rK   )r   r   dr   r   r   test_create_dir*  s   

r^  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  ttj | | W d    d S 1 sHw   Y  d S )N
directory/directory/nested/)r   rK   rM   r   r	  re   r
  r   r   r]  ndr   r   r   test_delete_dir;  s   

"rc  c                 C   s   t |  |d}|d}| | | | | | | |}|jtjks(J |d}|d}|d}| | | | | |}|d W d    n1 sSw   Y  | | | |}|jtjksjJ d S )Nr_  r`  
directory2zdirectory2/nestedzdirectory2/nested/target-file   data)	r   rK   rM   r<   r  r   r7   rr   r  )r   r   r]  rb  dir_infor  sr   r   r   $test_delete_dir_with_explicit_subdirI  s&   







rh  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  | j|dd ttj | | W d    n1 sNw   Y  | | ttj | | W d    d S 1 sow   Y  d S )Nr_  r`  T)rS   )r   rK   rP   r   r	  re   r
  rM   ra  r   r   r   test_delete_dir_contentsi  s    


"ri  c                 C   s2  | d }| d }|d}|d}| | ttj |d W d    n1 s+w   Y  ttj |d W d    n1 sFw   Y  ttj |d W d    n1 saw   Y  |jddd	 |jddd	 |jddd	 ttj || W d    d S 1 sw   Y  d S )
Nr   r   r_  r`  rK  rO   //Taccept_root_dir)rK   r   r	  re   ArrowInvalidrP   r
  rM   )r   r   r   r]  rb  r   r   r   _check_root_dir_contents{  s(   
"rn  c                 C   s   t |  t | d S r   )rn  )r   r   r   r   r   test_delete_root_dir_contents  s   ro  c                 C   s^   |d}|d}|  | W d    n1 sw   Y  | || | | | | d S )Nztest-copy-source-fileztest-copy-target-file)rr   r`   rU   r   r   rg  tr   r   r   test_copy_file  s   
rr  c                 C   s   t | d |d}|d}| | |r<| || ttj | | W d    n1 s0w   Y  | | d S ttj | || W d    d S 1 sTw   Y  d S )N.Not implemented yet in for Azure. See GH-40025zsource-dir/ztarget-dir/)r  rK   r\   r   r	  re   r
  rM   )r   r   r   rg  rq  r   r   r   test_move_directory  s   

"rt  c                 C   s   t |  t| d |d}|d}| | W d    n1 s!w   Y  | || ttj | | W d    n1 sBw   Y  | | d S )Nrs  ztest-move-source-fileztest-move-target-file)	r   r  rr   r\   r   r	  re   r
  rU   rp  r   r   r   test_move_file  s   
ru  c                 C   s   |d}|  | W d    n1 sw   Y  | | ttj | | W d    n1 s4w   Y  |d}| | |d}|  |}|d W d    n1 s[w   Y  | | d S )Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-filere  )	rr   rU   r   r	  re   r
  rK   r  rM   )r   r   r|   r]  r  rg  r   r   r   test_delete_file  s   

rv  c                 C   r   r   r   )vr   r   r   identity  r,   rx  )compressionbuffer_size
compressor@   gzip   c           	      C   s   |d}dd }|  |}||| W d    n1 sw   Y  | |||}| }W d    n1 s:w   Y  ||ksEJ d S )Nzopen-input-streams   some data for reading
   )rr   r  ri   read)	r   r   ry  rz  r{  r|   rh   rg  resultr   r   r   test_open_input_stream  s   
r  c                 C   s   |d}dd }|  |}|| W d    n1 sw   Y  tdd }| |}| }W d    n1 s<w   Y  ||ksGJ | |}|| | }W d    n1 s`w   Y  |||d  ksoJ d S )Nzopen-input-file	   some data   r  )rr   r  rT  rk   r  seek)r   r   r|   rh   rg  	read_fromr  r  r   r   r   test_open_input_file  s   


r  c                 C   sB   |d}t t | | W d    d S 1 sw   Y  d S )Nzopen-input-stream-not-found)r   r	  rE   ri   )r   r   r|   r   r   r    test_open_input_stream_not_found  s   "r  )ry  rz  decompressorc                 C   s   |d}dd }|  |||}|| W d    n1 sw   Y  | |||}|t||ks7J W d    d S 1 sBw   Y  d S )Nzopen-output-streams   some data for writingr  )rr   r  ri   r  rT  )r   r   ry  rz  r  r|   rh   r  r   r   r   test_open_output_stream  s   "r  )ry  rz  r{  r  c                 C   s  |d}|d}|  |}	|	| W d    n1 sw   Y  |rf| j|||d}
|
d W d    n1 s<w   Y  | |}
|
 }W d    n1 sUw   Y  ||}|dksdJ d S ttj | j|||d W d    d S 1 sw   Y  d S )Nzopen-append-streams   already existing)ry  rz  s   
newly addeds   already existing
newly added)	rr   r  rt   ri   r  r   r	  re   ArrowNotImplementedError)r   r   ry  rz  r{  r  r   r|   initialrg  r  r  r   r   r   test_open_append_stream(  s,   
"r  c           	      C   s   |d}ddi}d}| j ||d}|| W d    n1 s!w   Y  | |}| |ks4J | }W d    n1 sBw   Y  | jdv sQd| jv rkt| d | D ]\}}|| | kshJ qZd S |i ksqJ d S )	Nzopen-output-stream-metadataContent-Typezx-pyarrow/testr  rq   )r   r   r  r-  zWAzure filesystem currently only returns system metadata not user metadata. See GH-40026)	rr   r  ri   r  rq   r   r  itemsrd   )	r   r   r|   rq   rh   r  got_metadatakrw  r   r   r    test_open_output_stream_metadataJ  s$   
r  c                   C   sD   t dd tt t dd W d    d S 1 sw   Y  d S )NFr   )xxx)r   r   r	  r  r   r   r   r   test_localfs_optionsa  s   
"r  c                 C   sf  | d }t   |d W d    n1 sw   Y  t   |d W d    n1 s/w   Y  t   |jddd W d    n1 sIw   Y  t   |d W d    n1 saw   Y  t   |d W d    n1 syw   Y  t   |dd W d    n1 sw   Y  t   |dd W d    d S 1 sw   Y  d S )Nr   z/non/existent/filez/non/existent/dirFrR  z/non/existentz/xxx)r   ri   rr   rK   rM   rU   r\   r`   )r   r   r   r   r   test_localfs_errorsi  s,   "r  c                 C   s   | d }t t}|j}|| | g\}}|j| jks#J |j	| j
ks-J t| |j	| j
ks;J t| d S r   )rG  rD  __file__parentr<   rz   r@   statst_sizer   st_mtime_nsr   )r   r   	file_pathdir_pathr   rf  r   r   r   test_localfs_file_info}  s   


r  c                 C   s`   t dtj}t|}|d W d    n1 sw   Y  |dg\}|j|ks.J d S )Nr  r  )r   r  r   r  r   rr   r<   r   )r   r  r   r;   r   r   r   test_mockfs_mtime_roundtrip  s   r  c              	   C   s  ddl m} t }|dd|ddddd	}t||sJ |jdks#J |jdks*J | | ||ks6J | }t||s@J | | ||ksLJ |d
d}t||sXJ | | ||ksdJ |dddd}t||ssJ | | ||ksJ t	
t |dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |t d W d    d S 1 sw   Y  d S )Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainACLr  default_metadataaccess)r  r   )r   r  acct)r   r  )r  )r   r   r   nowr!   r  r   r1  r0  r   r	  rR   )r2  r   r  r   r   r   r   test_gcs_options  sH   

"r  c                 C   sr  ddl m}m}m}m} |ddddddd	}t||sJ |jdks$J | | ||ks0J |d
dddd}t||s?J | | ||ksKJ ||ddd}t||sZJ ||ddd}t||siJ |d
d}t||suJ | | ||ksJ ||ksJ |dd}t||sJ | | ||ksJ |dd}t||sJ | | ||ksJ |ddddd}t||sJ | | ||ksJ ||ksJ |dd}t||sJ | | ||ksJ | | || ksJ |ddd}t||sJ | | ||ksJ |dddd}t||s(J | | ||ks5J |ddd}t||sCJ | | ||ksPJ |ddd}t||s^J | | ||kskJ ||ksrJ |ddd }t||sJ | | ||ksJ |d!d"}t||sJ | | ||ksJ ||d#d"ksJ || ksJ t	
t |dd$ W d    n	1 sw   Y  t	
t |dd% W d    n	1 sw   Y  t	
t |ddd& W d    n	1 sw   Y  t	
t |ddd' W d    n	1 s"w   Y  t	
t |ddd(d) W d    n	1 s?w   Y  t	
t |dddd* W d    n	1 s\w   Y  t	
t |d(dd+ W d    n	1 sxw   Y  t	
t |d,d-gd. W d    n	1 sw   Y  t	
t || d W d    d S 1 sw   Y  d S )/Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyr  r   token	us-east-2r  r  )r   r   session_tokenregionr   r   rolesessionr   d   )role_arnsession_nameexternal_idload_frequencyrQ  )max_attempts)retry_strategy)r  Tr  )background_writesr  r  r  )r  r  )allow_delayed_open)r   r   )r   r   )check_directory_existence_before_creationg      ?g      ?)request_timeoutconnect_timeout)r   force_virtual_addressingzca.pem)tls_ca_file_pathzother_ca.pem)r   )r   )r   r  )r   r  arn)r   r   r  )r   r   r   )r  r   r  r  r  )r   r  r  r   r  r!   r  r1  r0  r   r	  rR   )r2  r  r  r   r  r   r"  r   r   r   test_s3_options  s   




$r  c           
      C   s  ddl m} dddd}d}ddd	d}d
}||d}t||s"J ||||ks.J ||d}t||s:J ||||ksFJ ||d}t||sRJ ||||ks^J ||d}t||sjJ ||||ksvJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks)J ||||	ks6J |||	|ksCJ ||d}||d}	||	ksTJ ||||	ksaJ |||	|ksnJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks J ||||	ksJ |||	|ksJ ||d}| }	||	ks)J ||||	ks6J |||	|ksCJ ||d}| }	||	ksRJ ||||	ks_J |||	|kslJ ||d}| }	||	ks{J ||||	ksJ |||	|ksJ ||d}| }	||	ksJ ||||	ksJ |||	|ksJ tt |dd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 s.w   Y  tt	j
 |dd W d    n	1 sJw   Y  tt	j
 |ddddd W d    d S 1 skw   Y  d S )Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999r  i  zhttps://localhost:8080)proxy_options)r   r  #  r  )r   r   )r   r   )r   r   zhttpsB://localhost:9000httpA)r   r   r!   r1  r0  r   r	  r  KeyErrorre   rm  )
monkeypatchr2  r   proxy_opts_1_dictproxy_opts_1_strproxy_opts_2_dictproxy_opts_2_strr   r!  r"  r   r   r   test_s3_proxy_options#  s   



























$r  c                  C   s   ddl m}  | ddd}d}tjt|d}|d W d    n1 s%w   Y  d	t|jv r3d S | d
dd}|d d S )Nr   r   z
eu-north-1T)r  r   zWhen getting information for bucket 'arrow-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-1'.|NETWORK_CONNECTIONr  arrow-datasetsNETWORK_CONNECTION	us-east-1)r   r   r   r	  r   r<   r   r   )r   r   msgexcr   r   r   test_s3fs_wrong_region  s   r  c           	      C   s"  ddl m} |dd}t||sJ | | ||ksJ |ddd}t||s+J | | ||ks7J ||ks=J |ddd	d
ddd}t||sNJ | | ||ksZJ ||ks`J |ddd}t||smJ | | ||ksyJ ||ksJ |ddddd}t||sJ | | ||ksJ ||ksJ |ddd}t||sJ | | ||ksJ ||ksJ tjtdd |ddd W d    n1 sw   Y  tjtdd |ddd W d    n1 sw   Y  d}tjt|d |dddd W d    n	1 sw   Y  tjtdd |dddd W d    n	1 s4w   Y  tjt|d |dddd W d    n	1 sSw   Y  tt |dddd W d    n	1 spw   Y  tt |  W d    d S 1 sw   Y  d S )Nr   r   zfake-account-name)r   fakeaccountkey)r   r   zfake-accountfakeaccountzfake-blob-authorityzfake-dfs-authorityr  r   fakesastoken)r   	sas_tokenzfake-tenant-idzfake-client-idzfake-client-secret)r   	tenant_id	client_idclient_secret)r   r  zclient_id must be specifiedr  )r   r  )r   r  zInvalid Azure credential configuration: For ManagedIdentityCredential, provide only client_id. For ClientSecretCredential, provide tenant_id, client_id, and client_secret.)r   r  r  )r   r  r  )r   r  r  )r   r   r  )	r   r   r!   r1  r0  r   r	  rR   r  )	r2  r   r!  r"  fs3fs4fs5fs6invalid_msgr   r   r   test_azurefs_options  s   
$r  c                 C   s  ddl m} t std | \}}}d}d}d}d}	|||d|||d	}
||	||d|||}||	||d
|||}||	||d
|d ||}|||}|d| d| }|||dd}|||ddd}|||dt	dd}|||ddd}|||ddddid}|
|ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ t
t |  W d    n1 sw   Y  t
t |d W d    n	1 sw   Y  |
||||||||||fD ]}||||ks#J q| \}}}||||d}|tds;J |d| d| d| }|tdsSJ d S )Nr   r   r      i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}libhdfs)r   replicationrz  default_block_sizemerE  zhdfs://r   	localuser)r   
cache_path)r   kerb_ticketcache_path2
hdfs_tokenabcd)r   r  
extra_confrS  rO   z/?user=)r   r   re   r   r   r   from_uriformatrG  rD  r	  r  r1  r0  r<   r   )r   r2  r   r   r   r   r  rz  r  urihdfs1hdfs2hdfs3hdfs4hdfs5hdfs6hdfs7hdfs8hdfs9hdfs10hdfs11r   r   r   r   r   test_hdfs_options&  s   



r  )r  expected_klassexpected_pathzmock:rK  zmock:foo/barr  zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/éfile:/rO   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 C   s,   t | \}}t||sJ ||ksJ d S r   )r   r  r!   )r  r  r  r   r.   r   r   r   test_filesystem_from_urin  s   r  c                  C   s   t d\} }t| tsJ |dksJ t jdd\} }t| ts#J |dks)J t jdd\} }t| ts9J |dks?J d S )Nr  rO   )r  )r   r  r!   r   )r   r.   r   r   r    test_filesystem_from_uri_calling  s   r  r.   c                 C   sB   t | }t|\}} t|tsJ | |   ksJ d S r   )	rG  rD  r   r  r!   r   resolveabsoluterz   )r.   r|   r   r   r   r    test_filesystem_from_path_object  s   
r  c           
   	   C   s   ddl m} | d \}}}}d| d| d| d| d	}t|\}}t||s+J |dks1J || ||g\}	|	j|ksDJ |	jt	j
ksLJ d S )	Nr   r   r   s3://r   z0@mybucket/foo/bar?scheme=http&endpoint_override=z&allow_bucket_creation=Truemybucket/foo/bar)r   r   r   r  r!   rK   r<   r.   r  r   r6   )
r   r   r   r   r   r   r  r   r.   r;   r   r   r   test_filesystem_from_uri_s3  s   
r  c                 C   s   ddl m} | d \}}dd| d| d d }t|\}}t||s'J |d	ks-J || ||g\}|j|ks@J |jt	j
ksHJ d S )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idr  )r   r   r   r  r!   rK   r<   r.   r  r   r6   )r   r   r   r   r  r   r.   r;   r   r   r   test_filesystem_from_uri_gcs  s   
r  c                  C   sp   t  } t| }t|tsJ |jdksJ |j| u sJ tt td  W d    d S 1 s1w   Y  d S Nz	py::dummy)r   r   r!   r   handlerr   r	  r  )r  r   r   r   r   test_py_filesystem  s   
"r  c                  C   s   t d} t d}t d}t| }t| }t|}t|}||us"J ||us(J ||us.J ||ks4J ||ks:J ||ks@J |t ksGJ |t ksNJ d S )NrE  r  )r   r   r   r8   )handler1handler2handler3r!  r"  r  r  r   r   r   test_py_filesystem_equality  s   r  c                 C   s\   t  }t|}| |}| |}t|tsJ ||ksJ |j|ks%J |jdks,J d S r  )r   r   r0  r1  r!   r   r  r   )r2  r  r   r3  r4  r   r   r   test_py_filesystem_pickling  s   

r  c                  C   s   t  } t| }t|tsJ t| }d } | d usJ d }| d u s%J t  } t| }t| }d } | |ju s;J | d usBJ d }| d u sKJ d S r   )r   r   r!   weakrefrefr  )r  r   wrr   r   r   test_py_filesystem_lifetime  s    

r  c                  C   s  t  } t| }|dg\}|jdksJ |jtjksJ |dg\}|jdks+J |jtjks3J |dg\}|jdksAJ |jtjksIJ t	
t |dg W d    n1 s_w   Y  t	
t |dg W d    d S 1 s{w   Y  d S )Nzsome/dirz	some/filer2   r3   r  )r   r   r<   r.   r  r   r6   r5   r7   r   r	  r  r9   )r  r   r;   r   r   r    test_py_filesystem_get_file_info	  s"   "r  c                  C   s^  t  } t| }tdd}||}t|dksJ |d jdks"J |d jtjks,J |d j	dks5J |d jdks>J |d jtj
ksHJ |d j	d u sQJ tdd	d
}||}t|dksdJ |d jdksmJ |d jdksvJ |d jdksJ tdd}tt || W d    n1 sw   Y  tdd	d}||g ksJ d S )Nr=   )rC   r  r   r>   r?   rE  rA   T)rC   rF   rS  rB   r2   )rC   rD   )r   r   r   r<   rT  r.   r  r   r5   r@   r6   r   r	  rE   )r  r   rG   rH   r   r   r   )test_py_filesystem_get_file_info_selector   s.   



r  c               	   C   s   t  } t| }|jddd |jddd tt |d W d    n1 s*w   Y  |d |d d	D ]#}tt || W d    n1 sRw   Y  |j|dd
 q;|	d |
dd |dd d S )NrF   TrR  rJ   FfoobarrM   rP   )rK  rO   rj  rk  rU   rW   rX   r^   r_   )r   r   rK   r   r	  r9   rM   rP   rR   rU   r\   r`   )r  r   r.   r   r   r   test_py_filesystem_ops=  s"   


r   c                  C   ~   t t } | d}| dksJ W d    n1 sw   Y  tt | d W d    d S 1 s8w   Y  d S )Nsomefiles   somefile:input_streamr2   )r   r   ri   r  r   r	  rE   r   r  r   r   r   test_py_open_input_streamQ     
"r$  c                  C   r!  )Nr"  s   somefile:input_filer2   )r   r   rk   r  r   r	  rE   r#  r   r   r   test_py_open_input_fileZ  r%  r&  c                  C   D   t t } | d}|d W d    d S 1 sw   Y  d S Nr"  re  )r   r   rr   r  r#  r   r   r   test_py_open_output_streamc     
"r)  c                  C   r'  r(  )r   r   rt   r  r#  r   r   r   test_py_open_append_streamj  r*  r+  c                  C   s   ddl m}  tjdpd}| dd}|j|ksJ | ddd}|td}t|dks/J d	}|	|#}|
 }d
|v sAJ |d dksIJ |d dksQJ W d    d S 1 s\w   Y  d S )Nr   r   PYARROW_TEST_S3_REGIONr  Tr  )r   r  zarrow-datasets/nyc-taxiz8arrow-datasets/nyc-taxi/year=2019/month=6/part-0.parquetr  zLast-Modifieds   2025-11-26T10:28:55ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetr  r<   r   rT  ri   rq   )r   default_regionr   entriesr   r  mdr   r   r   test_s3_real_awsq  s$   
"r4  c                  C   s  t d\} }| jdksJ | |}d|dv sJ W d    n1 s'w   Y  t d\} }| jdks:J tjtdd t d	 W d    n1 sQw   Y  t d
\} }| jdksdJ t d\} }| |}tt |	  W d    d S 1 sw   Y  d S )Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2r  zBucket '.*' not foundr  z s3://x-arrow..nonexistent-bucketz0s3://x-arrow-nonexistent-bucket?region=us-east-3z	us-east-3zMs3://x-arrow-nonexistent-bucket/T.md?region=us-east-2&allow_delayed_open=true)
r   r  r  ri   r  r   r	  r9   rr   close)r   r.   r  streamr   r   r   !test_s3_real_aws_region_selection  s*   

"r9  c                  C   s   ddl m}  | ddksJ | ddksJ tjtdd | d	 W d    n1 s,w   Y  tjtdd | d
 W d    d S 1 sHw   Y  d S )Nr   resolve_s3_regionr  r  zmf-nwp-modelsr5  zNot a valid bucket namer  r  z	s3:bucket)r   r;  r   r	  rR   r:  r   r   r   test_resolve_s3_region  s   

"r<  c              
   C   s0  |d }|d }|d}| |}|d W d    n1 s!w   Y  | \}}}	}
d|	 d|
 d| d| d| 
}t|d	 }t|| t }||}| dksYJ W d    n1 scw   Y  t|d
 }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t|}t|| ||}| dksJ W d    n1 sw   Y  t|d }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t||ddd ||}| dksJ W d    d S 1 sw   Y  d S )Nr   r   r8  r  r  r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtrE  F
chunk_sizeuse_threads)rr   r  r   r   r   ri   r  r   )s3_connectionr   r~   r   r   r.   r  r   r   r   r   
source_urilocal_path1r   local_path2local_path3destination_urilocal_path4local_path5r   r   r   test_copy_files  sP   

$rK  c                    s|  t   | d }|   t|d }|d W d    n1 s$w   Y   t|d }|d W d    n1 sBw   Y   fdd}| d }|  tt|t| || | d	 }|  tt|t|  d
 || | d }|  tt|}tt|}t|| || | d }	|	  t||	 ||	 | d }
|
  t||
ddd ||
 d S )Nsourcefile1   test1file2   test2c                    s     t| d }| dksJ W d    n1 sw   Y    t| d }| dks3J W d    d S 1 s>w   Y  d S )NrM  rN  rO  rP  )ri   r   r  )destination_dirr  r   r   r   check_copied_files  s   "z5test_copy_files_directory.<locals>.check_copied_filesdestination1destination2)r>  r?  destination3destination4destination5rE  Fr@  )r   mkdirrr   r   r  r   r   )r~   
source_dirr  rS  destination_dir1destination_dir2destination_dir3rD  rH  destination_dir4destination_dir5r   rR  r   test_copy_files_directory  sD   

r`  c                  C      d} t tjd| g d S )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecoder   r   r   test_s3_finalize  s   rj  c                  C   ra  )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('arrow-datasets')
        rb  rc  rh  r   r   r    test_s3_finalize_region_resolver;  s   rk  c                  C   ra  )Na  if 1:
        import threading
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)
        threads = []
        fn = lambda: FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        for i in range(4):
            thread = threading.Thread(target = fn)
            threads.append(thread)
            thread.start()

        for thread in threads:
            thread.join()

        finalize_s3()
        rb  rc  rh  r   r   r   test_concurrent_s3fs_initO  s   rl  zUatexit(ensure_s3_finalized) will be called too late with bundled aws-sdk-cpp 1.11.587r   z?Leaking S3ClientFinalizer causes segfault on musl based systemsc               
   C   sN  z	t ddg W n ty   td Y nw t } dddd|  dtjt	dg}t j
|t jd	}zdd
|  d}t }d }t |d k rzt|}|jdksUJ W d    n1 s_w   Y  W n, ty~ } z|}td W Y d }~nd }~ww t |d k sHtd|d|  W |  n|  w | dksJ d S )Nuwsgiz	--versionz"uwsgi not installed on this Pythonz-iz--httpz
127.0.0.1:z--wsgi-filezwsgi_examples.py)stdinzhttp://127.0.0.1:z/s3/rQ     g?zCould not fetch r      )rd  re  rE   r   r   r   r.  r.   joinherePopenDEVNULLtimer   statusr   sleepfail	terminatewait)r   argsprocurl
start_timeerrorrespr   r   r   r   test_uwsgi_integrationh  s<   
	r  c                  C   s   zddl m}  ddlm} W n ty   td Y nw td\}}t	t
| }||ks1J |dks7J d}td| \}}t	t
|  }||ksPJ d S )	Nr   )r   MemoryFileSystemfsspec not installedz$fsspec+memory://path/to/data.parquetz/path/to/data.parquetzfile:///tmp/my.filezfsspec+)fsspec.implementations.localr   fsspec.implementations.memoryr  ImportErrorr   r   r   r  r   r   )r   r  r   r.   expected_fsr  r  r   r   r   test_fsspec_filesystem_from_uri  s   r  c                     sH  zddl m}  W n ty   td Y nw t|    jddd  jddd  jdi d	}|d
 W d    n1 sAw   Y   jdi d	}|d W d    n1 s]w   Y   fdd}|dt	j
ksqJ |dt	jkszJ |dt	j
ksJ    |dt	jksJ |dt	jksJ |dt	jksJ d S )Nr   r  r  test_dirTrR  ztest_dir/subdirztest_file.txtr  s   test contentztest_dir/nested_file.txts   nested contentc                    s     | gd jS )Nr   )r<   r  )r.   r   r   r   get_type  s   z6test_fsspec_delete_root_dir_contents.<locals>.get_type)r  r  r  r   r   r   rK   rr   r  r   r5   r6   rT   r7   )r  r8  r  r   r   r   $test_fsspec_delete_root_dir_contents  s,   
r  c                  C   sp   t d zddlm}  W n ty   t d Y nw td\}}tt	|  }||ks0J |dks6J d S )Nr   r   )HfFileSystemzhuggingface_hub not installedzFhf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquetzAdatasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet)
r   r   huggingface_hubr  r  r   r   r  r   r   )r  r   r.   r  r   r   r   $test_huggingface_filesystem_from_uri  s   
r  )r   r   r   r}  r.  rG  urllib.requestr   rd  rf  ru  r   r  r   re   pyarrow.tests.test_ior   pyarrow.tests.utilr   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   pyarrow.utilr   r.   dirnameabspathr  rr  r   fixturer   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   markr   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r#  r$  r*  r5  r@  rB  rC  rI  rP  r\  r^  rc  rh  ri  rn  ro  rr  rt  ru  rv  rx  parametrizecompressr  r  r  
decompressr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r$  r&  r)  r+  r4  r9  r<  rK  r`  rj  rk  	processes	threadingrl  r   skipifr  r  r  r  r   r   r   r   <module>   s  4i
	
	
	
	
	
	



 





;




#
;8 

	

	

	

%
g
 

]
G



		

!

43

 #