o
    پi                      @   sV  d dl Z d dlZejded ejded d dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZm Z!m"Z# e$ddZ%e$ddZ&e$d	d
Z'e$ddZ(e$ddZ)e& dZ*dZ+de& de( Z,de& de( dZ-de* de( Z.de% Z/de% dZ0de+ dZ1dZ2ej3ddddd Z4e	j5dd Z6e	j5dd Z7e	j5d d! Z8e	j5e&e(fd"d#Z9d$d% Z:d&d' Z;d(d) Z<d*d+ Z=d,d- Z>d.d/ Z?d0d1 Z@ejABd2e7e8e9gd3d4 ZCd5d6 ZDejABd2e7e8e9gejABd7dd8gd9d: ZEejABd2e8e9gd;d< ZFd=d> ZGejABd2e7e8e9gd?d@ ZHejABd2e7e8e9gdAdB ZIejABd2e7e8e9gdCdD ZJejABd2e9gdEdF ZKejABd2e7e8e9gdGdH ZLejABd2e8e9gdIdJ ZMejABd2e7e8e9gdKdL ZNejABd2e7e8e9gdMdN ZOejABd2e7e8e9gdOdP ZPejABd2e7e8e9gdQdR ZQejABd2e7e8e9gdSdT ZRejABd2e7e8e9gejABdUd8dgdVdW ZSejABd2e7gdXdY ZTejABd2e7e8e9gejABdZd8dgd[d\ ZUejABd2e7e8e9gd]d^ ZVejABd2e7e8e9gejABdZd8dgd_d` ZWejABdZd8dgdadb ZXejABdZd8dgdcdd ZYejAjZejABd2e8e9gdedf Z[dgdh Z\didj Z]ejABd2e7e8e9gdkdl Z^dmdn Z_dodp Z`e	j5dqdr Zadsdt ZbejABdue.e-e0gdvdw ZcejABdxdydzgejABd2e8e9gd{d| ZdejABd2e8e9gd}d~ ZeejABd2e7e8e9gejABdd8dgdd ZfejABd2e7e8e9gdd Zgdd ZhejABddd8gejABd7dd8gejABd2e7e8e9gdd ZiejABd7dd8gejABd2e7e8e9gdd ZjejAjZejABd2e7e8e9gdd Zkdd ZlejABd2e7e8e9gdd ZmejABd2e9gdd ZnejABd2e9gdd ZoejABd2e9gdd ZpejABd2e9gdd Zqdd Zrdd Zsdd Ztdd Zudd Zvdd ZwG dd dZxejABd2e8e9gdd Zydd ZzejABd2e8e9gdd Z{dS )    Nignore)category)_ops_azure_commonGCS_TEST_BUCKETz
csh-test-3AS_TEST_ACCOUNTcshteststorage2AS_TEST_ACCOUNT2cshteststorage3AS_TEST_CONTAINERtestcontainer2AS_TEST_CONTAINER2testcontainer3z-does-not-existcshteststorage4az:///gs://https://3.blob.core.windows.net/publiccontainer/test_cat.pngs   PNGsessionT)scopeautousec                   c   s@    t  dkrdtjv rtddddtjd  g d V  d S )NLinuxGOOGLE_APPLICATION_CREDENTIALSgcloudauthzactivate-service-accountz--key-file=)platformsystemosenvironsprun r#   r#   F/home/ubuntu/.local/lib/python3.10/site-packages/blobfile/_ops_test.pysetup_gcloud_auth3   s   
r%   c                 c   s(    t  }t |  d V  t | d S N)r   getcwdchdir)pathoriginal_pathr#   r#   r$   r(   C   s
   
r(   c                  c   sR    t  } t| tsJ tj| d}|V  W d    d S 1 s"w   Y  d S )N	file.name)tempfileTemporaryDirectory
isinstancestrr   r)   join)tmpdirr)   r#   r#   r$   _get_temp_local_pathK   s   
"r2   c                  c   sv    zddl m}  W n ty   td Y nw dt dddd td	D  }| | |d
 V  | 	| d S )Nr   gfiletensorflow not installedr   r    c                 s       | ]	}t tjV  qd S r&   randomchoicestringascii_lowercase.0ir#   r#   r$   	<genexpr>Z   s    
z%_get_temp_gcs_path.<locals>.<genexpr>   
/file.name)
tensorflow.ior4   ImportErrorpytestskipr   r0   rangemkdirrmtree)r4   r)   r#   r#   r$   _get_temp_gcs_pathS   s   


rJ   c                 c   sz    d dd tdD }d|  d| d| }|d V  d	d
ddd| d|d| dddg}tj|dt dkdd d S )Nr6   c                 s   r7   r&   r8   r=   r#   r#   r$   r@   d   s    z$_get_temp_as_path.<locals>.<genexpr>rA   r   z.blob.core.windows.net/r   rB   azstorageblobzdelete-batch--account-namez--sourcez	--patternz/*--auth-modeloginTWindows   )checkshelltimeout)r0   rG   r!   r"   r   r   )account	container	random_idr)   cmdr#   r#   r$   _get_temp_as_pathb   s$   
rZ   c           
      C   sN  d| v s	|  drrt [}t|tsJ t| \}}}tj	|d}t
|d}|| W d    n1 s9w   Y  ddddd	d
|d|d|d|ddg}tj|dt dktjtjdd W d    d S 1 skw   Y  d S zddlm}	 W n ty   td Y nw |	| d}|| W d    d S 1 sw   Y  d S )N.blob.core.windows.netr   tmpwbrK   rL   rM   uploadz--overwriterN   --container-name--name--filerO   rP   TrQ   rR   rS   rT   stdoutstderrrU   r   r3   r5   )
startswithr,   r-   r.   r/   azure
split_pathr   r)   r0   openwriter!   r"   r   r   DEVNULLrC   r4   rD   rE   rF   GFile)
r)   contentsr1   rV   rW   rM   filepathfrY   r4   r#   r#   r$   _write_contentsx   sR   

" "ro   c           	      C   s(  d| v rrt  `}t|tsJ t| \}}}tj|d}ddddd|d|d	|d
|ddg}t	j
|dt dkt	jt	jdd t|d}| W  d    W  d    S 1 s[w   Y  W d    d S 1 skw   Y  d S ddlm} || d}| W  d    S 1 sw   Y  d S )Nr[   r\   rK   rL   rM   downloadrN   r_   r`   ra   rO   rP   TrQ   rR   rb   rbr   r3   )r,   r-   r.   r/   rf   rg   r   r)   r0   r!   r"   r   r   rj   rh   readrC   r4   rk   )	r)   r1   rV   rW   rM   rm   rY   rn   r4   r#   r#   r$   _read_contents   sL   

"$rs   c                  C   s0   g d} | D ]\}}t |}||ksJ qd S )N))r   r6   )za/r6   )art   )a/bbr6   r6   )gs://ar6   )gs://a/r6   )	gs://a/b/r6   )gs://a/brv   )gs://a/b/c/test.filenametest.filename)z!https://a.blob.core.windows.net/br6   )z"https://a.blob.core.windows.net/b/r6   )z$https://a.blob.core.windows.net/b/c/r6   )z#https://a.blob.core.windows.net/b/cc)z1https://a.blob.core.windows.net/b/c/test.filenamer}   )bfbasename)	testcasesinput_desired_outputactual_outputr#   r#   r$   test_basename   s
   
r   c                  C   sB   t jdd} g d}|D ]\}}| |}||ksJ | qd S )NFoutput_az_paths))rt   r6   )ru   rt   )a/b/cru   )a/b/c/r   )z
a/b/c/////r   rw   )rx   rx   )ry   rx   )zgs://a/////rx   )r{   rx   )r|   
gs://a/b/c)gs://a/b/c/r{   )zgs://a/b/c/////r{   ))https://a.blob.core.windows.net/containerr   )z*https://a.blob.core.windows.net/container/r   )z.https://a.blob.core.windows.net/container/////r   )+https://a.blob.core.windows.net/container/br   )z;https://a.blob.core.windows.net/container/b/c/test.filename-https://a.blob.core.windows.net/container/b/c).https://a.blob.core.windows.net/container/b/c/r   )z3https://a.blob.core.windows.net/container/b/c//////r   )r   create_contextdirname)rv   r   r   r   r   r#   r#   r$   test_dirname   s   %
r   c                  C   s   t jdd} g d}|D ]3\}}}| ||}||ks$J | d| d|v r?t|}| ||}||ks?J | d| qd S )NFr   ))rt   rv   ru   )ru   r~   r   )a/b/r~   r   )r   c/r   )r   /c/r   )r6   r6   r6   )r   zd0123456789-._~!$&'()*+,:=@;z'gs://a/b/c/d0123456789-._~!$&'()*+,:=@;)rx   rv   r{   )r{   r~   r   )rz   r~   r   )rz   r   r   )rz   r   	gs://a/c/)rz   ../czgs://a/c)rz   ../c/r   )rz   ../../c/r   )r   rv   r   )r   r~   r   ),https://a.blob.core.windows.net/container/b/r~   r   )r   r   r   )r   r   ,https://a.blob.core.windows.net/container/c/)r   r   z+https://a.blob.core.windows.net/container/c)r   r   r   )r   r   r   )gs://test/a/bzc:dzgs://test/a/b/c:d)r   zc:d;zgs://test/a/b/c:d; zblob.core.windows.net)r   r   r0   _convert_https_to_az)rv   r   input_ainput_br   r   
az_input_ar#   r#   r$   	test_join  s   ;r   c                 C   s   |  dd ddS )Nr   r   r[   r6   )replacer)   r#   r#   r$   r   T  s   r   c                 C   s   t dd| S )Nz^az://(.*?)/z!https://\1.blob.core.windows.net/)resubr   r#   r#   r$   _convert_az_to_httpsX  s   r   ctxc                 C   s`   d}|  !}t || t|\}}tj| |ksJ W d    d S 1 s)w   Y  d S N   meow!)ro   r   get_urlurllibrequesturlopenrr   )r   rl   r)   url_r#   r#   r$   test_get_url\  s   
"r   c                  C   sH   t jt } | tsJ tt\}}t j| | ks"J d S r&   )	r   r   r   AZURE_PUBLIC_URLrr   re   AZURE_PUBLIC_URL_HEADERr   r   )rl   r   r   r#   r#   r$   test_azure_public_get_urle  s   r   	streamingFc              	   C   sJ  d}|  }t |dd}t t | t j|d|d}|| | t|ks-J W d    n1 s7w   Y  t j|d|d}| |ksMJ | t|ksWJ W d    n1 saw   Y  t j|d|d}t	|}d||ks|J W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )N   meow!
purr
a foldera.filer]   r   rq       )
r   r0   makedirsr   BlobFileri   telllenrr   list)r   r   rl   r)   wrlinesr#   r#   r$   test_read_writel  s&   
	"r   c                 C   sj  d}|  }t j|ddd9}t|d#}|dd}|| W d    n1 s,w   Y  W d    n1 s;w   Y  W d    n1 sJw   Y  t j|ddd<}t|d&}|dd}| |ksnJ W d    n1 sxw   Y  W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )	Nr   r]   Tr   r   zeggs.txtrq   r   )r   r   zipfileZipFilerh   ri   rr   )r   rl   r)   rn   zfmyfiler#   r#   r$   test_zipfile~  s.   "r   c               	   C   s*  d} t  }t|}t|dd}t|}ttt| t|d}||  W d    n1 s5w   Y  t|d}| | ksIJ W d    n1 sSw   Y  t|d}t	|}d|| kslJ W d    n1 svw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r   r   r]   rq   r   )
rZ   r   r   r0   r   r   r   ri   rr   r   )rl   r)   r   r   r   r#   r#   r$   test_az_path  s&   	"r   c              	   C   s   d}d}|  k}t j|ddd}|| W d    n1 s w   Y  t j|ddd}|| W d    n1 s=w   Y  t |d}| || ksSJ W d    n1 s]w   Y  W d    d S W d    d S 1 suw   Y  d S )N   meow!
s   purr
abFr   rq   r   r   ri   rr   )r   rl   additional_contentsr)   r   r   r#   r#   r$   test_append  s   "r   c                 C   sp   d}|  )}t || t|}|jt|ksJ tt |j dks&J W d    d S 1 s1w   Y  d S )Nr   rR   )ro   r   statsizer   abstimemtime)r   rl   r)   sr#   r#   r$   	test_stat  s   

"r   c                 C   s   d}|  4}t || t|}tt |j dksJ d}t||s'J t|j|ks1J W d    d S 1 s<w   Y  d S )Nr   rR      )ro   r   r   r   r   r   	set_mtime)r   rl   r)   r   	new_mtimer#   r#   r$   test_set_mtime  s   

"r   c              	   C   s   d}|  S}t |d}|| W d    n1 sw   Y  t |d td t j|ddd}t |}W d    n1 sDw   Y  |jdksPJ W d    d S 1 s[w   Y  d S )Nr   r]   r      Tr   )r   r   ri   r   r   sleepr   r   )r   rl   r)   rn   str#   r#   r$   test_azure_metadata  s   
"r   c                 C   s`   d}|  !}t || t|sJ t| t|rJ W d    d S 1 s)w   Y  d S r   )ro   r   existsremover   rl   r)   r#   r#   r$   test_remove  s   

"r   c              	   C   s   d}|  j}t |}t t |d t |d}t | t |s&J t | t |r2J t | t |dd}t|| tt	 t | W d    n1 sXw   Y  t 
| t | W d    d S 1 srw   Y  d S )Nr   fakedirnamer   name)r   r   rmdirr0   r   r   ro   rE   raisesOSErrorr   )r   rl   r)   dirpathnew_dirpathnew_filepathr#   r#   r$   
test_rmdir  s$   





"r   c                 C   sj   d}|  &}t |ddd}t | t |sJ tt |d| W d    d S 1 s.w   Y  d S )Nr   xtestfile)r   r0   r   r   ro   )r   rl   r)   r   r#   r#   r$   test_makedirs  s   
"r   c                 C   s   d}|  k}t |rJ t|| t |rJ |d }t | t |s)J t |d d r4J t |d dd}d|vrIt t | t t |}t|| t |s]J t |d d rhJ W d    d S 1 ssw   Y  d S )Nr   z.dirz	.otherdirsubdirr+   ://)r   isdirro   r   r0   r   )r   rl   r)   r   rm   r#   r#   r$   
test_isdir  s"   


"r   c              	   C   s  d}|  }t |}t | t |d}t |d}|| W d    n1 s,w   Y  t |d}t |d}|| W d    n1 sMw   Y  t t |d g d}ttt ||kslJ t	|}ttt ||ks}J W d    d S 1 sw   Y  d S )Nr   rt   r]   rv   r~   rt   rv   r~   )
r   r   r   r0   r   ri   sortedr   listdirr   )r   rl   r)   r   a_pathr   b_pathexpectedr#   r#   r$   test_listdir  s$   

"r   c              	      s  d}|  }t | t  d}t |d}|| W d    n1 s'w   Y  t  d}t |d}|| W d    n1 sHw   Y  t t  d ttt  }dd |D g dkslJ d	d |D  fd
ddD ks~J dd |D g dksJ dd |D g dksJ |d j	j
t|ksJ |d j	j
t|ksJ |d j	d u sJ W d    d S 1 sw   Y  d S )Nr   rt   r]   rv   r~   c                 S      g | ]}|j qS r#   )r   r>   er#   r#   r$   
<listcomp>:      z test_scandir.<locals>.<listcomp>r   c                 S   r   r#   r   r   r#   r#   r$   r   ;  r   c                       g | ]}t  |qS r#   r   r0   )r>   r   r   r#   r$   r   ;      c                 S   r   r#   )is_dirr   r#   r#   r$   r   <  r   )FFTc                 S   r   r#   )is_filer   r#   r#   r$   r   =  r   )TTFr   r      )r   r   r0   r   ri   r   r   r   scandirr   r   r   )r   rl   r)   r   r   r   entriesr#   r  r$   test_scandir-  s*   
$"r  c              	   C   s  d}|  }t |}t t |dd}|| W d    n1 s%w   Y  t t |dd}|| W d    n1 sDw   Y  t t |dd}|| W d    n1 scw   Y  t t |dd}|| W d    n1 sw   Y  t t |d t t |dd}|| W d    n1 sw   Y  ttt j|d	d
g dksJ W d    d S 1 sw   Y  d S )Nr   rt   r]   aarv   car~   zc/ar   )shard_prefix_length)rt   r	  rv   r~   r
  )	r   r   r   r0   ri   r   r   r   r   )r   rl   r)   r   r   r#   r#   r$   test_listdir_shardedC  s*   
$"r  topdownc           	   	   C   sR  d}|  }t |}t |d}t |d}|| W d    n1 s'w   Y  t t |d t |d}t |d}|| W d    n1 sQw   Y  |dgdgft |ddgg ft |ddg dgfg}|sytt|}tt j||d	|ksJ t	|}tt j||d	|ksJ W d    d S 1 sw   Y  d S )
Nr   rt   r]   zc/dzc/d/br~   drv   )r  )
r   r   r0   r   ri   r   r   reversedwalkr   )	r   r  rl   r)   r   r   r   r   r   r#   r#   r$   	test_walk]  s,   
"r  c              	      sN  d}|  }t | t  d}t |d}|| W d    n1 s'w   Y  t  d}t |d}|| W d    n1 sHw   Y   fdd}t . |ddg |dddg |ddg |d	dg |d
ddg |ddg W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r   r]   bbc                    sF   t  fdd|D }t tt| }||ks!J | d| d S )Nc                    r   r#   r   r>   pr  r#   r$   r     r  zAtest_local_glob.<locals>.assert_listing_equal.<locals>.<listcomp> != r   r   r   globr)   desiredactualr  r#   r$   assert_listing_equal  s   z-test_local_glob.<locals>.assert_listing_equal*ba*ab**)r   r   r0   r   ri   r(   )r   rl   r)   r   r   r   r  r#   r  r$   test_local_globv  s.   

"r   parallelc           	   	      st  d}|  }t | t  d}t |d}|| W d    n1 s(w   Y  t  d}t |d}|| W d    n1 sIw   Y   fdd}|t  dddg |t  ddg |t  d	dg |t  d
ddg |t  ddg t  d}t |d}|| W d    n1 sw   Y  t  dd}t t | t |d}|| W d    n1 sw   Y  t  ddd}d|vrt t | t |d}|| W d    n1 sw   Y  |t  ddg |t  ddg |t  dg d |t  dddg |t  dddg |t  dddg |t  dg  |t  ddg |t  d
g d |t  ddg |t  ddg |t  ddg |t  ddg |t  ddg |t  ddg |t  d dg W d    d S 1 sw   Y  d S )!Nr   r   r]   r  c                    sJ   t  fdd|D }t ttj| d}||ks#J | d| d S )Nc                    r   r#   r   r  r  r#   r$   r     r  z;test_glob.<locals>.assert_listing_equal.<locals>.<listcomp>r!  r  r  r  r   r!  r#   r$   r    s   z'test_glob.<locals>.assert_listing_equalr  r  r  r  test.txtr   	subsubdirr   z
*/test.txtsubdir/test.txtz*/*.txtz**.txt)r$  r&  subdir/subsubdir/test.txtz	sub**.txtr'  zsubdir/**.txtz**/*.txtz*/test)r   r  r   r$  zsubdir/z*/z*dirzsubdir/*dirzsubdir/subsubdirzsubdir/*dir/zsu*ir/*dir/)r   r   r0   r   ri   r   )	r   r!  rl   r)   r   r   r   r  rn   r#   r#  r$   	test_glob  sr   

$r(  c                 C   s  d}|  I}t |}t |d}t |d}|| W d    n1 s(w   Y  t |d}t |d}|| W d    n1 sIw   Y  t |d}t |d}|| W d    n1 sjw   Y  t |dd}t t | t |d}|| W d    n1 sw   Y  ttt t |d}|d j	dkr|d j
sJ |d	 j	dkr|d	 j
sJ |d
 j	dkr|d
 jsJ dD ]*}	dD ]%}
ttt t ||
}ttt jt ||
d|	d}||ksJ qqd|v r?ttt t |d}|d j	dkr|d j
sJ |d	 j	dkr-|d	 j
s/J |d
 j	dkr=|d
 jsGJ W d    d S W d    d S 1 sSw   Y  d S )Nr   r   r]   r  r$  r   *b*r   r   r  )r   r   )r  zb***zb**z**tr)  T)r!  r  r   r*  )r   r   r0   r   ri   r   r   r   scanglobr   r  r  )r   rl   r)   r   r   r   r   rn   r  r  patternnormal_entriesparallel_entriesr#   r#   r$   test_scanglob  s^   


	
   ($r/  c           	      C   s  d}|  }t |}t |d}t | t |d}t | d|vr.t t |d t t |dd}|| W d    n1 sHw   Y  t t |d t t |d	d}|| W d    n1 spw   Y  t t |d
 t t |dd}|| W d    n1 sw   Y  dd }|||ddgg f|ddgg ft |dg dgft |dg dgf|d
gdgft |d
g g fg t j||d |||dgg f|d
gdgft |d
g g fg W d    d S 1 sw   Y  d S )Nr   destroysaver   adirzadir/br]   bdirzbdir/bsomedirsomefilec                 S   s4   t t| }t|t|ksJ | d| d S )Nr  )r   r   r  r   r  r#   r#   r$   r  ,  s   &z)test_rmtree.<locals>.assert_listing_equalrv   r"  )r   r   r0   r   r   ri   rI   )	r   r!  rl   r)   rootdestroy_path	save_pathr   r  r#   r#   r$   test_rmtree  sP   



"r9  c                 C   s  dddd fD ]f}t  W}t  B}t  -}t }t }t }t }tttd}	t }
tt tj	||| d W d    n1 sNw   Y  tt tj	||| d W d    n1 skw   Y  t
|| ||f||f||f||f||f||	f|	|f||
fg}|D ]N\}}tj	||d| d}|t| ksJ t||ksJ tt tj	||| d W d    n1 sw   Y  tj	||d| d t||ksJ qW d    n1 sw   Y  W d    n1 sw   Y  W d    n	1 s	w   Y  W d    n	1 sw   Y  W d    n	1 s)w   Y  W d    n	1 s9w   Y  W d    n	1 sIw   Y  W d    n	1 sYw   Y  W d    n	1 siw   Y  qd S )	Nr   r       )rV   rW   r"  T)
return_md5r!  )	overwriter!  )r2   rJ   rZ   r
   r   rE   r   FileNotFoundErrorr   copyro   hashlibmd5	hexdigestrs   FileExistsError)r!  rl   local_path1local_path2local_path3	gcs_path1	gcs_path2as_path1as_path2as_path3as_path4r   srcdsthr#   r#   r$   	test_copyI  sh   D
        rO  c                 C   s  dddd fD ]}t  }t }t }tttd}tttd}t|| tj||| d t	j
t|d tj||| d W d    n1 sLw   Y  t	j
t|d tj||| d W d    n1 skw   Y  t	j
t|d tj||| d W d    n1 sw   Y  t	j
t|d tj||| d W d    n1 sw   Y  t	j
t|d tj||| d W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  qd S )Nr   r   r:  zfile.binr"  )match)r2   rZ   r   r   r0   AZURE_INVALID_CONTAINER"AZURE_INVALID_CONTAINER_NO_ACCOUNTro   r>  rE   r   r=  )r!  rl   
local_pathrH  rI  invalid_container_as_pathinvalid_account_as_pathr#   r#   r$   test_copy_invalidk  sB   


  rV  c              
   C   s*  dt j d }|  }t 1}tj|d}t|d}|| W d    n1 s,w   Y  t	j
||dd W d    n1 sCw   Y  t||ksPJ t }tj|d}t	j
||dd t||kslJ W d    n1 svw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   s   meow???r$  r]   Tr"  )commonPARALLEL_COPY_MINIMUM_PART_SIZEr,   r-   r   r)   r0   rh   ri   r   r>  rs   )r   rl   remote_pathr1   rS  rn   r#   r#   r$   test_parallel_copy_large_file  s$   

	"rZ  c                  C   sP   t  } tt|  t| d d tksJ W d    d S 1 s!w   Y  d S )N   )rZ   r   r>  r   rs   r   )rM  r#   r#   r$   test_copy_azure_public  s   "r\  c                  C   s2   d} t | s	J tt | d dksJ d S )Nz5gs://tfds-data/datasets/mnist/3.0.1/dataset_info.jsonrq   r   )r   r   r   r   rr   )rm   r#   r#   r$   test_gcs_public  s    r]  c                 C   sV   d}|  }t |rJ t|| t |sJ W d    d S 1 s$w   Y  d S r   )r   r   ro   r   r#   r#   r$   test_exists  s   
"r^  c               
   C   s   t  q} dd }dd }tj| ddd*}|| tj| ddd}|| W d    n1 s2w   Y  W d    n1 sAw   Y  t| d}| |ksUJ W d    n1 s_w   Y  W d    d S W d    d S 1 sww   Y  d S )N   miso     momor]   Tr   rq   )rJ   r   r   ri   rr   )r)   outer_contentsinner_contentsrn   r#   r#   r$   test_concurrent_write_gcs  s    

"rd  c                  C   s4  t  } tjdd}dd }dd }ttjB |j| ddd*}|| |j| ddd}|| W d    n1 s?w   Y  W d    n1 sNw   Y  W d    n1 s]w   Y  || d	}| |ksqJ W d    n1 s{w   Y  W d    d S W d    d S 1 sw   Y  d S )
N   azure_write_chunk_sizer_  r`  ra  r]   Tr   rq   )	rZ   r   r   rE   r   ConcurrentWriteFailurer   ri   rr   )r)   rv   rb  rc  rn   r#   r#   r$   test_concurrent_write_as  s(   
"ri  c                  c   s    t j } d V  | t _d S r&   )r   r    r>  )envr#   r#   r$   environ_context  s   

rk  c                  C   s   t dft d dft d dft d dftdftd dftd dftd dftdftd dftd dftd dftdftd dftd dftd dftdftd dftd dftd dfddg} | D ]\}}t||ksmJ q`d S )NFr   ///invalid.fileT)z/does-not-existF)r   T)rQ  GCS_INVALID_BUCKETrR  AZURE_VALID_CONTAINERGCS_VALID_BUCKETr   r   )r   r)   should_existr#   r#   r$   test_more_exists  s4   














rr  	base_pathc           
      C   s  dD ]<}| | }t | |drt}nt}tt| |dkrMdD ]'}ztt||  W q$ tjyK } zd|jv sAJ W Y d }~q$d }~ww ndD ]}tt||  qOt	
t tt| W d    n1 srw   Y  t|r~J t|rJ t	
| t| W d    n1 sw   Y  |dv rzt| W n tjy } zd|jv sJ W Y d }~n
d }~ww t| t	
t t| W d    n1 sw   Y  t	
t t| W d    n1 sw   Y  | tkr t	
tj t| W d    n	1 sw   Y  nt| t	
t t| W d    n	1 s;w   Y  t	
tj t| W d    n	1 sWw   Y  tt| t b}tj|d}t	
| t|| W d    n	1 sw   Y  t|d	}|d
 W d    n	1 sw   Y  t	
| t|| W d    n	1 sw   Y  W d    n	1 sw   Y  dD ]j}	t	
|% tj |d|	d}|!  W d    n	1 sw   Y  W d    n	1 sw   Y  t	
|& tj |d|	d}|d W d    n	1 s(w   Y  W d    n	1 s8w   Y  qqd S )N)r6   r   rl  rm  z/invalid/dir/r   r6   )r  r*  zWildcards cannot be used)r6   r   zCannot delete bucketr$  r   meow)FTrq   r   r]      meow)"printendswithIsADirectoryErrorr=  r   r   r  ErrormessagerE   r   r   r   r   r   r   NotADirectoryErrorrI   r   rR  r   r@  r   r  r,   r-   r   r)   r0   r>  rh   ri   r   rr   )
rs  suffixr)   expected_errorr,  r   r1   rS  rn   r   r#   r#   r$   test_invalid_paths  s   






r~  buffer_sizer   d   c              	   C   s  | }d}t |d}|| W d    n1 sw   Y  t j|d| d}|d W d    n1 s9w   Y  | dkrK|jjdksJJ n
|jjt|ksUJ t j|d| d%}|d |d |d |d |ddks{J W d    n1 sw   Y  | dkr|jjd	ksJ |jjd	ksJ n|jjdksJ |jjt|ksJ W d    d S W d    d S 1 sw   Y  d S )
Nr   r]   rq   )r  r   r[  @B r   r  )	r   r   ri   rr   raw
bytes_readr   seekrequests)r  r   r)   rl   r   r   r#   r#   r$   test_read_stats;  s6   



"r  c              	   C   s  t  }d}d}|  }tj|dd}|| W d    n1 s#w   Y  tj|dd|d}| |ks:J W d    n1 sDw   Y  t| }t	||t
|}t|d}| |kshJ W d    n1 srw   Y  t|d}|| W d    n1 sw   Y  tj|dd|d}| |ksJ W d    n1 sw   Y  tjdt d	dd|d}t| d
ksJ W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   s   purr!r]   )moderq   F)r  r   	cache_dirr   z:.blob.core.windows.net/publiccontainer/test_cat_no_md5.pngr   )r,   mkdtempr   r   ri   rr   r?  r@  rA  r0   r   rh   AS_EXTERNAL_ACCOUNTr   )r   r  rl   alternative_contentsr)   rn   content_hash
cache_pathr#   r#   r$   test_cache_dir[  sB   
"r  
use_randomc           
   
   C   s  d}d| d }d| d }|rt t|}t t|}|  }t|d}|| W d    n1 s9w   Y  t|dL}||}t|d}|| W d    n1 s`w   Y  d |j_|| 7 }t| dksyJ ||d | ||d   ksJ W d    n1 sw   Y  t|d}|| W d    n1 sw   Y  t|di}||}t|d}|| W d    n1 sw   Y  d |j_|| 7 }t| dksJ |d | |||d   }	|	d	s|	d
s|d | ||d   }	||	ksJ W d    n1 s&w   Y  W d    d S W d    d S 1 s?w   Y  d S )N             r  r]   rq   r   r   r   )
r   urandomr   r   r   ri   rr   r  _fre   )
r   r  
chunk_sizelong_contentsshort_contentsr)   rn   read_contentsf2r   r#   r#   r$   test_change_file_sizew  sN   

"
 $r  c              
   C   s4  d}d| d }d| d }|  }t |d}|| W d    n1 s(w   Y  t |dE}||}t |d}|| W d    n1 sOw   Y  d |j_|||7 }||d | |||d   ksqJ W d    n1 s{w   Y  W d    d S W d    d S 1 sw   Y  d S )Nr  r  r  r  r[  r]   rq   )r   r   ri   rr   r  r  )r   r  rl   r  r)   rn   r  r  r#   r#   r$   test_overwrite_while_reading  s*   

"r  c                  C   s   d} t  [}t|}t|9 t|ddtdt|dddfD ]}t|d}||  W d    n1 s;w   Y  q#W d    n1 sKw   Y  W d    d S W d    d S 1 scw   Y  d S )Nru  r   r+   z..z./file.namer]   )r2   r   r   r(   r0   r   r   ri   )rl   r)   r   rm   rn   r#   r#   r$   #test_create_local_intermediate_dirs  s$   

"r  binaryc              
   C   s  t jd}| 1}| rd}d}nd}d}tj|||d}W d    n1 s)w   Y  tj|||d}t| dksAJ W d    n1 sKw   Y  d}	| sY|	d}	tj|||d}||	 W d    n1 sqw   Y  tj|||d+}|d	|	d d	 ksJ | |	d	d  ksJ t| dksJ W d    n1 sw   Y  tj|||d4}t	t|	D ]}
|d	|	|
|
d	  ksJ qt| dksJ t| dksJ W d    n1 sw   Y  d
}	g d}| s
|	d}	dd |D }tj|||d}||	 W d    n	1 s#w   Y  tj|||d}|
 |ks:J W d    n	1 sEw   Y  tj|||d}dd |D |ks_J W d    n	1 sjw   Y  | rdD ]g}|jdd|t jd }	tj|||d}||	 W d    n	1 sw   Y  tj|||d)}|dd}d}	 ||}|dkrn||7 }q||	ksJ W d    n	1 sw   Y  qtnNdd	i}tj|||d}t|| W d    n	1 sw   Y  tj|||d}t||ksJ W d    n1 sw   Y  W d    d S W d    d S W d    d S 1 s?w   Y  d S )Nr   rq   r]   r   r   r   r   utf8r   s   meow!

mew!
)r      
s   mew!
c                 S   s   g | ]}| d qS )r  )decoder>   liner#   r#   r$   r     s    z(test_more_read_write.<locals>.<listcomp>c                 S   s   g | ]}|qS r#   r#   r  r#   r#   r$   r     s    )r:  iNa    r   dtyper  r   Trt   )npr9   RandomStater   r   r   rr   r  ri   rG   	readlinesrandintuint8tobytesjsondumpload)r  r   r   rngr)   	read_mode
write_moder   r   rl   r?   r   r   bufrv   objr#   r#   r$   test_more_read_write  s   

 


 0E$r  c              
   C   s  z
dd l }dd l}W n ty   td Y nw tjd}d}|jddt	|tj
d|}| }tj|d| d/}|j|dd d	d
g dd}	|D ]}
|	|
 qNW d    n1 s`w   Y  W d    n1 sow   Y  tj|d| d5}|j|dddgd}t|D ]\}}
t|
|| sJ qW d    n1 sw   Y  W d    n1 sw   Y  tj|d| d+}||}|jjd }t||D ]\}}
t|
 || sJ qW d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   z)av and imageio are required for this test)r  @   r  r  r  r  r]   )r  r   ffmpeg
libx264rgbbgr24)-fmp4z-crf0)formatqualitycodecpixelformatoutput_paramsrq   r  r  )r  input_params)avimageiorD   rE   rF   r  r9   r  r  prodr  reshaper   r   
get_writerappend_data
get_reader	enumeratearray_equalrh   streamsvideor  to_image)r   r   r  r  r  shape
video_datar)   wfr   framerfr   idxrW   streamr#   r#   r$   
test_video  sX    
$r  c              	   C   s   dd }|  N}t j|ddd}|| W d    n1 s w   Y  t j|ddd}|| ks6J W d    n1 s@w   Y  W d    d S W d    d S 1 sXw   Y  d S )N   0l        r]   Tr   rq   r   )r   rl   r)   rn   r#   r#   r$   test_large_file;  s   "r  c                     s  t  ʉt  dd } t d}||  W d    n1 s!w   Y   fdd}t|  }|  tjd u sAJ |tksJJ tj|ksTJ |tks]J t	 |  tjd u soJ t
 )}tjd|dd}| | ksJ W d    n1 sw   Y  W d    n1 sw   Y  tj|ksJ W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )	Ns   00re  r]   c                      s   t jdddd gdd d S )Ngsutilz-oz-GSUtil:parallel_composite_upload_threshold=1McpT)rS   )r!   r"   r#   rS  rY  r#   r$   create_composite_fileM  s   
z5test_composite_objects.<locals>.create_composite_filerq   F)r  r   )rJ   r2   rh   ri   r?  r@  rA  r   r   r   r,   r-   r   rr   )rl   rn   r  	local_md5r1   r#   r  r$   test_composite_objectsF  s6   

"r  c              	   C   s   d}t | }|  _}t|| t||ksJ t|d}|| W d    n1 s1w   Y  t||ks?J t|d}|| W d    n1 sUw   Y  t||kscJ W d    d S 1 snw   Y  d S )Nr   r]   )r?  r@  rA  ro   r   r   ri   )r   rl   	meow_hashr)   rn   r#   r#   r$   test_md5k  s   
"r  c                 C   s   d}t | }d}t | }|  V}t|| ttjj|}t	tjj||j
|s0J t|| t	tjj||j
|rBJ ttjj|}|j|ksQJ t| t	tjj||j
|rcJ W d    d S 1 snw   Y  d S )Nr   s   purr)r?  r@  rA  ro   rf   
maybe_statopsdefault_context_confmaybe_update_md5versionr   r   )r   rl   r  r  	purr_hashr)   r   r#   r#   r$   test_azure_maybe_update_md5{  s   


"r  c              
   C   s  d}d}|  }t |d| t |j}t j|d|d}tdD ]}|| q$W d    n1 s6w   Y  t |j}t j|d|d}|| W d    n1 sYw   Y  t |j}||ksjJ tt j% t j|d|d}|| W d    n1 sw   Y  W d    n1 sw   Y  t |d	 |ksJ t 
| W d    d S 1 sw   Y  d S )Ns   bark!s   ruffr]   )r  i  rq   )r   r   ri   r   r  rG   rE   r   VersionMismatchrr   r   )r   rl   r  r)   r  rn   r   new_versionr#   r#   r$   test_azure_etags  s2   "r  c              
   C   s  d}d}|  }zt | W n	 ty   Y nw t j|ddd}|| t |d u s1J || W d    n1 s@w   Y  t |}t j|ddd&}|t||ks^J t |}|d usiJ | |ksqJ W d    n1 s{w   Y  t |}||ksJ ||ksJ t | d}d}	t j|ddd}|| W d    n1 sw   Y  t j|ddd}
|
|	 W d    n1 sw   Y  t |}t |
}||ksJ t	t j
* t j|dd|d	}| |ksJ W d    n	1 sw   Y  W d    n	1 sw   Y  t j|ddd}| |	ks/J W d    n	1 s:w   Y  t |}||ksKJ t | t j|ddd}|d
d  W d    n	1 skw   Y  t j|ddddN}|ddksJ t j|ddd}|dd  W d    n	1 sw   Y  t	t j
 |d W d    n	1 sw   Y  W d    n	1 sw   Y  t | W d    d S 1 sw   Y  d S )Ns   hellos    worldr]   Tr   rq   s   hi theres	   overwrite)r   r  s   abcdefgh   rA   )r   r  s   abcdefghabcdefghs   ijklmnop)r   r   r=  r   ri   last_version_seenrr   r   rE   r   r  )r   part1part2r)   rn   written_versionpartial_read_versionread_versiondata1data2r  version1version2gr#   r#   r$   test_azure_etags_last_version  s|   









$r  c                 C   s   t jdtfdd}|  h}||K}tjd5}td|_|d t	
t |dd  W d    n1 s;w   Y  |jd	ksGJ W d    n1 sQw   Y  W d    n1 s`w   Y  W d    d S W d    d S 1 sxw   Y  d S )
Nfnc              
   s   sf    t jdd}|j| ddd}z
|V  W |d n|d w W d    d S 1 s,w   Y  d S )Nr  rf  r]   Tr      asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf)r   r   r   ri   )r  bfcrn   r#   r#   r$   trailer_file  s   "z5test_azure_buffer_error_finally.<locals>.trailer_filez#blobfile._azure.execute_api_requestasdfs   asdfr  r  r   )
contextlibcontextmanagerr/   unittestmockpatchRuntimeErrorside_effectri   rE   r   
call_count)r   r  r)   rn   r  r#   r#   r$   test_azure_buffer_error_finally  s"   


"r  c                  C   s0   d} dt dtfdd}|| t| ksJ d S )NzSun, 27 Sep 2009 18:41:57 GMTtextreturnc                 S   s   t j | ddd S )NGMTZz%a, %d %b %Y %H:%M:%S %z)datetimestrptimer   	timestamp)r  r#   r#   r$   ref_parse_timestamp  s
   z9test_azure_timestamp_parsing.<locals>.ref_parse_timestamp)r/   floatrf   _parse_timestamp)r  r  r#   r#   r$   test_azure_timestamp_parsing  s   r  c                 C   s   |  ttjj  d S r&   )putidr  r  r  get_http_pool)qr#   r#   r$   _get_http_pool_id  s   r  c                  C   st   t  } tjj }t|}t jt| fd}|	  |
  tjj }t|}|  }||ks2J ||ks8J d S )N)targetargs)mpQueuer  r  r  r  r  Processr  startr0   get)r  http1parent1r  http2parent2childr#   r#   r$   	test_fork  s   r#  c               
   C   s   d dt  dftjdt  dftdt dffD ]D\} }t }| d ur)t| }|) t	|d}|
 }|ts>J W d    n1 sHw   Y  W d    n1 sWw   Y  qd S )Nr   r   z+.blob.core.windows.net/private/test_cat.pngrq   )r  r   ry  r=  AS_INVALID_ACCOUNTr  nullcontextrE   r   r   rr   re   r   )errorr)   r   rn   rl   r#   r#   r$   test_azure_public_container#  s(   


r'  c               	   C   s   d t fttfttftjdt dffD ].\} }t }| d ur$t	
| }| t| tt| W d    n1 s<w   Y  qd S )Nr   z.blob.core.windows.net/private)ro  r=  rQ  rR  r   ry  r  r  r%  rE   r   rv  r   r  )r&  r)   r   r#   r#   r$   test_scandir_error8  s   
r(  c               
   C   sR  t  } t| d}|d W d    n1 sw   Y  t| dd}tj|ddd}| dks4J |d |d|d |d dksLJ tt	 |d	 W d    n1 saw   Y  tt	 |d
 W d    n1 s{w   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr]   ru  rq   r   r  )r  ends   eor   r   r  )
r2   rh   ri   rW  WindowedFilerr   r  rE   r   AssertionError)r)   rn   r  r#   r#   r$   test_windowed_fileG  s(   
&"r,  c                   @   s4   e Zd Zdd ZdeddfddZdefdd	ZdS )
Deadlinec                 C   s
   d | _ d S r&   	_deadlineselfr#   r#   r$   __init__[     
zDeadline.__init__deadliner  Nc                 C   s
   || _ d S r&   r.  )r1  r4  r#   r#   r$   set_deadline^  r3  zDeadline.set_deadlinec                 C   s   | j S r&   r.  r0  r#   r#   r$   get_deadlinea  s   zDeadline.get_deadline)__name__
__module____qualname__r2  r  r5  r6  r#   r#   r#   r$   r-  Z  s    r-  c              
   C   sL  d}|  }t || t }tj|jd}|t d  ||d}|  W d    n1 s3w   Y  t	d t
tj" ||d}|  W d    n1 sYw   Y  W d    n1 shw   Y  |d  ||d}|  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nru  )r6  r   rq   )ro   r-  r   r   r6  r5  r   r   rr   r   rE   r   DeadlineExceeded)r   rl   r)   r4  bf_ctxrn   r#   r#   r$   test_deadlinee  s.   





"r<  c                  C   s<   t  } | j}t| |  tt|}|  d S r&   )r  r   r  pickledumpsr  loads)r   r~   c2r#   r#   r$   test_pickle_configx  s   
rA  c              	   C   s   d}|  g}t |dd}t t | t |d}|| W d    n1 s+w   Y  t j|ddd}| |d d ksEJ | dksMJ W d    n1 sWw   Y  W d    d S W d    d S 1 sow   Y  d S )Nr   r   r   r]   rq   r   )	file_size)r   r0   r   r   r   ri   rr   r   )r   rl   r)   r   r   r#   r#   r$   test_read_with_size  s   "rC  )|r  warningsfilterwarningsDeprecationWarningFutureWarningr9   r;   r,   r   r  r  urllib.requestr   r?  r   
subprocessr!   multiprocessingr  r   r=  r   r   unittest.mockr   rE   numpyr  blobfiler   r   r  r   rf   r   rW  getenvr   r   r
   r   r   r$  r  ro  rQ  rR  rp  rn  r   r   fixturer%   r  r(   r2   rJ   rZ   ro   rs   r   r   r   r   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r   r(  r/  r9  rO  rV  slowrZ  r\  r]  r^  rd  ri  rk  rr  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  r'  r(  r,  r-  r<  rA  rC  r#   r#   r#   r$   <module>   sN  






+',G



	


	

	




*A
-5
!




@
(
L%	%



=

	