o
    پiP                     @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)Z)d dl*Z*d dl+m,Z- d dl+m.Z/ d dl+m0Z1 d d	l2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z> e? @eAd d
@eAdd@eAddZBdZCdZDd ZEdZFdZGdZHdZIdddZJG dd dZKdddZLdd"d#ZMdd$d%ZNdd'd(ZOdd)d*ZPdd3d4ZQdd9d:ZRdd=d>ZSdd@dAZTddEdFZUdddJdKZVddMdNZWG dOdP dPe!ZXG dQdR dRe!ZYG dSdT dTe!ZZddWdXZ[dd]d^Z\dd`daZ]ddbdcZ^ddddeZ_dddgdhZ`ddkdlZaddmdnZbddqdrZcddtduZdddzd{Zedd~dZfG dd dejgZheLe4e5eCeDeEeFdeGeHddddeIdddddddZidS )    )annotationsN)partial)
ModuleType)AnyBinaryIOCallableIteratorListLiteral
NamedTupleOptionalSequenceTextIOTupleUnioncastoverload)_azure)_common)_gcp)
CHUNK_SIZE DEFAULT_CONNECTION_POOL_MAX_SIZE!DEFAULT_MAX_CONNECTION_POOL_COUNTConfigDirEntryErrorRemoteOrLocalPathRequest RestartableStreamingWriteFailureStatget_log_threshold_for_errorpath_to_str	             i      
      fnCallable[..., object]argsr   c                 G  s   | |  d S N )r)   r+   r-   r-   E/home/ubuntu/.local/lib/python3.10/site-packages/blobfile/_context.py_execute_fn_and_ignore_resultI      r/   c                   @  sz  e Zd ZdpddZ					dqdrddZdsddZdtddZdudvddZ	dwdxd#d$Zdsd%d&Z	dydzd'd(Z
dyd{d)d*Zd|d,d-Zd|d.d/Zd|d0d1Zd}d3d4Z	d~dd8d9Z		ddd:d;Z	<	dddBdCZdtdDdEZddHdIZddKdLZdtdMdNZddQdRZdtdSdTZddVdWZddYdZZdd\d]Ze	^	^	^		dddhdiZe	^	^	^	^		dddldiZ	m					dddodiZdS )Contextconfr   c                 C  s
   || _ d S r,   )_conf)selfr2   r-   r-   r.   __init__N   s   
zContext.__init__FNsrcr   dst	overwriteboolparallelparallel_executor%Optional[concurrent.futures.Executor]
return_md5dst_versionOptional[str]returnc                 C  s  t |}t |}t|rtd| dt|r td| d|s/| |r/td| d|d ur?t|s?J d| d|rd }t|sKt|rQt|rQt}t|r`t|r`t	t
j|d}t|rkt|rktj}t|rt|rt
|\}	}
}
t
|\}}
}
|	|krt	t
j|d}|d ur|d u rtjjt| jjd}|| j||||d	W  d    S 1 sw   Y  n
|| j||||d	S t|rt|rtj| j|||d
S t|rt|r|d u sJ d| dt
j| j|||d
S tt D ]\}}zv| j|dddc}| j|dd|dI}t }	 |t}|dkr%n|r-|| | | q|rL|! W  d    W  d    W   S 	 W d    W d    W  d S 1 saw   Y  W d    n	1 sqw   Y  W q t"y } z9| jj#d ur|| jj#kr |t$| jt%|kr| j&d| d| d| d|dd	 t'(| W Y d }~qd }~ww d S )NIs a directory: ''zDestination 'z*' already exists and overwrite is disabledz8Destination version was specified, but destination path z! does not support a version check)r>   
mp_context)r=   )r6   r7   r=   rbT)	streamingwb)rF   version    zerror z% when executing a streaming write to z	 attempt z, sleeping for z.1fz seconds before retrying))r!   _guess_isdirIsADirectoryErrorexistsFileExistsError_is_azure_path_is_gcp_path_is_local_path_parallel_downloadr   azureparallel_uploadgcp
split_pathparallel_remote_copy
concurrentfuturesProcessPoolExecutormpget_contextr3   multiprocessing_start_methodremote_copy	enumeratecommonexponential_sleep_generatorBlobFilehashlibmd5readr   updatewrite	hexdigestr   retry_limitr    strlog_callbacktimesleep)r4   r6   r7   r8   r:   r;   r=   r>   copy_fnsrc_account_dst_accountexecutorattemptbackoffsrc_fdst_fmblockerrr-   r-   r.   copyQ   s   



"





$RzContext.copypathc                 C  s   t |}t|rtj|S t|r$t| j|}|d urdS | 	|S t
|r:t| j|}|d ur5dS | 	|S td| d)NTUnrecognized path: 'rB   )r!   rP   osrz   rL   rO   rT   
maybe_statr3   isdirrN   rR   r   )r4   rz   str-   r-   r.   rL      s   

zContext.existsri   c                 C  s^   t |}t|rt|\}}|dd S t|r)t|\}}}|dd S tj	|S N/)
r!   rO   rT   rU   splitrN   rR   r|   rz   basename)r4   rz   ro   objr-   r-   r.   r      s   zContext.basenamepatternIterator[str]c                 c  s\    t |rd|v sd|v sd|v rtdt|E d H  d S | j||dD ]}|jV  q%d S )N?[]'Advanced glob queries are not supported)r   r:   )rP   r   _local_globscanglobrz   )r4   r   r:   entryr-   r-   r.   glob   s   
zContext.globr   shard_prefix_lengthintIterator[DirEntry]c                 c  s   d|v sd|v sd|v rt dt|rEt|D ])}t|}t|j}t|| 	||| |r3d nt
|j|j|jd d ddV  qd S t|sNt|rd|vrat| j|}|d ur_|V  d S t|r{t|\}}	d|v rtt dt|d	}
nt|\}}}	d|v sd|v rt d
t| j||d	}
|dkrtd	t|	g}nX|sJ dg }dd tdD }|	jddd\}}td|d D ]6}tj||dD ],}d	}|D ]}|t|7 }q||k}|r|| }n|| d | }|td	t| qq|ryt| jj }|! }|D ]}|"| qt#|}|! }d}|j$t%| j|
||fd@ ||k re|& }t'|t(rA|j)V  nt'|trQ|"| |d7 }nt'|t*r\|d7 }nt d||k s2W d    n1 spw   Y  d S d S t+, }|D ]}|| qt#|dkr|- }t.| j|
|dD ]}t'|t(r|j)V  q|| qt#|dksd S d S t d| d)Nr   r   r   r   sizemtimectimerc   rH   )rz   nameis_diris_filestat*z'Wildcards cannot be used in bucket name z0Wildcards cannot be used in account or containerr   zQYou probably want to use parallel=True if you are setting shard_prefix_length > 0c                 S  s*   g | ]}|t td tdgvr|qS )r   r   )INVALID_CHARSunionord.0ir-   r-   r.   
<listcomp>  s    $z$Context.scanglob.<locals>.<listcomp>      )maxsplitrepeatinitializerinitargszInvalid result)r2   roottUnrecognized path 'rB   )/r   rP   r   r|   r   stat_moduleS_ISDIRst_moder   r   r   st_sizest_mtimest_ctimerO   rN   
_get_entryr3   rT   rU   combine_pathrR   	_GlobTask_split_pathranger   	itertoolsproductchrappendrZ   r[   r\   QueueputlenPool_glob_workerget
isinstance
_GlobEntryr   _GlobTaskCompletecollectionsdequepopleft_process_glob_task)r4   r   r:   r   filepathsr   r   bucketblob_prefixr   account	containerinitial_tasksvalid_charspattern_prefixpattern_suffixr   charsprefixcexactpatmp_ctxtasksr   tasks_enqueuedresults
tasks_donerdqr-   r-   r.   r      s   








&zContext.scanglobc                 C  sX   t |}t|rtj|S t|rt| j|S t|r$t	| j|S t
d| dNr{   rB   )r!   rP   r|   rz   r~   rO   rT   r3   rN   rR   r   r4   rz   r-   r-   r.   r~   K  s   zContext.isdirc                 c  s*    t |}| j||dD ]}|jV  qd S )N)r   )r!   scandirr   )r4   rz   r   r   r-   r-   r.   listdirV  s
   
zContext.listdirc                 c  s@   t |}t|st|r|ds|d7 }| |s#td| d| |s0td| dt|rqt	
|D ]5}| rPt|jt	j|jddd dV  q9| }t|jt	j|jddt|j|j|jd d ddV  q9d S t|szt|r|d	krt| j|dd
E d H  d S t| jj}| }| }d	}dd tdD }	td|d D ])}
tj|	|
dD ]}d}|D ]}|t|7 }q|
|k}||||f |d7 }qqd	}|j t!| j||fd* ||k r|" }|d u r|d7 }q|V  ||k sW d    d S W d    d S 1 sw   Y  d S t#d| d)Nr   ,The system cannot find the path specified: 'rB    The directory name is invalid: 'TF)r   rz   r   r   r   r   r   )r2   r   exclude_prefixc                 S  s$   g | ]}|t vr|td kr|qS r   )r   r   r   r-   r-   r.   r     s   $ z#Context.scandir.<locals>.<listcomp>r   r   r   r   r   r{   )$r!   rO   rN   endswithrL   FileNotFoundErrorr~   NotADirectoryErrorrP   r|   r   r   r   r   rz   abspathr   r   r   r   r   _list_blobs_in_dirr3   rZ   r[   r\   r   r   r   r   r   r   r   _sharded_listdir_workerr   r   )r4   rz   r   der   r   prefixesitemsr   r   r   r   r   r   r   r   r   r-   r-   r.   r   [  s   






$
zContext.scandirNonec                 C  sf   t |}t|rtj|dd d S t|rt| j| d S t|r+t	| j| d S t
d| d)NTexist_okr{   rB   )r!   rP   r|   makedirsrO   rT   	mkdirfiler3   rN   rR   r   r   r-   r-   r.   r     s   zContext.makedirsc                 C  s   t |}t|rt| d S t|r3|dr td| dt| j|}|s1t	d| dd S t
|rW|drDtd| dt| j|}|sUt	d| dd S td| d)Nr   rA   rB   r   r{   )r!   rP   r|   removerO   r   rK   rT   r3   r   rN   rR   r   )r4   rz   okr-   r-   r.   r     s$   

zContext.removec           	      C  sn  t |}t|rt| d S |ds|d7 }t|r$t|\}}nt|r1t	|\}}}nt
d| d|dkrEt
d| d| |}zt| W n tyZ   Y d S  tyb   Y n	w td| dt|rt|\}}ttjd||dd	d
d}t| j| d S t|rt	|\}}}tt	j|d||dd	dd}t	| j| d S t
d| d)Nr   r{   rB   r   zCannot delete bucket: 'zThe directory is not empty: '!/storage/v1/b/{bucket}/o/{object}r   objectDELETE)   urlmethodsuccess_codes/{container}/{blob}r   blob)   )r!   rP   r|   rmdirr   rO   rT   rU   rN   rR   r   r   nextr   StopIterationOSErrorr   	build_urlexecute_api_requestr3   )	r4   rz   ro   r  itr   reqr   r   r-   r-   r.   r    sN   



zContext.rmdirr   c                 C  s   t |}t|rt|}t|j|j|jd d dS t|r2t	
| j|}|d u r0td| d|S t|rKt
| j|}|d u rItd| d|S td| d)Nr   No such file: 'rB   r{   )r!   rP   r|   r   r   r   r   r   rO   rT   r}   r3   r   rN   rR   r   )r4   rz   r   r   r-   r-   r.   r     s   
zContext.statr   floatrH   c                 C  sz   t |}t|r|d u sJ tj|||fd dS t|r'tj| j|||dS t|r5t	j| j|||dS t
d| d)N)timesT)rz   r   rH   r{   rB   )r!   rP   r|   utimerO   rT   	set_mtimer3   rN   rR   r   )r4   rz   r   rH   r-   r-   r.   r    s   zContext.set_mtimec                   sn  t    std  dt rt  d S t s$t r ds- d7  t r< fdd}t	j
}nt rK fdd}tj
}ntd  d|r|d u rhtjjtjjd}|}n|}t }g }| | D ]}	|t|j|	}
||
 qvW d    n1 sw   Y  |D ]}
|
  qd S | D ]}	|j|	 qd S td  d)	Nr   rB   r   c                  3  st    t  \} }t j D ](}t|}t |\}}|| kr%||s'J tt jd| |dddd}|V  qd S )Nr   r   r   )r     r   )rT   rU   
list_blobsr3   _get_slash_path
startswithr   r
  )r   r  r   entry_slash_pathentry_bucket
entry_blobr  rz   r4   r-   r.   request_generator-  s"   z)Context.rmtree.<locals>.request_generatorc            	      3  s    t  \} }}t j D ].}t|}t |\}}}|| kr+||kr+||s-J tt j| d||dddd}|V  qd S )Nr  r  r   )r  r  r   )rR   rU   r  r3   r  r  r   r
  )	r   r   r  r   r  entry_accountentry_containerr  r  r  r-   r.   r  D  s(   
	r{   rC   )r!   r~   r   rP   shutilrmtreerO   rN   r   rT   r  rR   r   rW   rX   rY   rZ   r[   r3   r\   
contextlibnullcontextsubmitr/   r   result)r4   rz   r:   r;   r  r)   rq   contextrX   r  fr-   r  r.   r    sJ   




zContext.rmtreeTtoptopdownonerror#Optional[Callable[[OSError], None]]2Iterator[Tuple[str, Sequence[str], Sequence[str]]]c                 #  s   t |}|sd S t|rCtj|}tj|||dD ]"\}}}t|ts*J |	tj
r6|d d }|t|t|fV  qd S t|sLt|rtj|}|	ds[|d7 }|rt }|| t|dkr|   	dswJ t|rtjj dd}nt|rtjj dd}ntd| dg }g }|D ]}	t|	}
|
 krq|	jr||	j q||	j qt ||fV  | fdd	|D  t|dksld S d S t|rtj|}nt|rtj|}ntd| dg  g g}g g}|D ]}	t|	}||krq|t|d  }|d}|d d }| krz |d t  krT|d  |  |  fV       |d t  ks6 |krz|t  } | |g  |d | |g   |ksY|	j!r|d |	j qt dkr|d  |  |  fV      t dkst||  |  fV  t|dkrt|dksJ d S td| d)
N)r&  r'  r(  r   r   r   	delimiterr{   rB   c                 3  s     | ]}  |d  V  qdS )r   N)joinr   dirnamecurr4   r-   r.   	<genexpr>  s    zContext.walk.<locals>.<genexpr>)"r!   r~   rP   r|   rz   normpathwalkr   ri   r   sepsortedrO   rN   _normalize_pathr3   r   r   r   r   r   rT   r  rR   r   r  r   r   _strip_slashextendr   r-  popr   )r4   r&  r'  r(  r   dirnames	filenamesr   r  r   
entry_pathdirnames_stackfilenames_stackr  relpathpartsdirpathr/  r-   r0  r.   r4  v  s   










$zContext.walkc                 C  s@   t |}t|rt| j|S t|rt| j|S tj|S r,   )	r!   rO   rT   r/  r3   rN   rR   r|   rz   r   r-   r-   r.   r/    s   zContext.dirnamear+   c                 G  s(   t |}|}|D ]	}t| j||}q|S r,   )r!   _join2r3   )r4   rC  r+   outbr-   r-   r.   r-    s
   zContext.joinTuple[str, Optional[float]]c                 C  sZ   t |}t|rt| j|S t|rt| j|S t|r%d| d fS td| d)Nzfile://r{   rB   )	r!   rO   rT   get_urlr3   rN   rR   rP   r   r   r-   r-   r.   rH    s   zContext.get_urlc                 C  s  t |}t|rTt| j|}|d u rtd| d|j}|d ur$|S | |d}t	|
 }W d    n1 s<w   Y  |jd usHJ t| j||j| |S t|rt| j|}|d u rktd| d|j}|d u r| |d}t	|
 }W d    n1 sw   Y  |jd usJ t| j||j| |S | |d}t	|
 W  d    S 1 sw   Y  d S )Nr  rB   rE   )r!   rO   rT   r}   r3   r   rc   ra   r_   	block_md5hexrH   maybe_update_md5rN   rR   )r4   rz   r   hr%  r#  r-   r-   r.   rc     s:   $zContext.md5fileTextIO | BinaryIOc                 C  sN   |}t |tjr|j}t |tjtjfr|j}t |tjtj	fs$t
d|jS )Nz7File was not an Azure BlobFile opened in streaming mode)r   ioTextIOWrapperbufferBufferedReaderBufferedWriterrawrR   StreamingReadFileStreamingWriteFile
ValueError_version)r4   rM  actualr-   r-   r.   last_version_seen
  s   zContext.last_version_seenc                 C  8   |  |d}| W  d    S 1 sw   Y  d S )Nr   ra   rd   r4   rz   r%  r-   r-   r.   	read_text     $zContext.read_textbytesc                 C  r[  )NrE   r\  r]  r-   r-   r.   
read_bytes  r_  zContext.read_bytestextc                 C  <   |  |d}|| W d    d S 1 sw   Y  d S )Nwra   rf   )r4   rz   rb  r%  r-   r-   r.   
write_text     "zContext.write_textdatac                 C  rc  )NrG   re  )r4   rz   rh  r%  r-   r-   r.   write_bytes!  rg  zContext.write_bytes.modeLiteral['rb', 'wb', 'ab']rF   Optional[bool]buffer_sizeOptional[int]	cache_dir	file_sizer   c                 C     d S r,   r-   r4   rz   rj  rF   rm  ro  rp  rH   r-   r-   r.   ra   %     zContext.BlobFileLiteral['r', 'w', 'a']r   c                 C  rq  r,   r-   rr  r-   r-   r.   ra   2  rs  r   (Literal['r', 'rb', 'w', 'wb', 'a', 'ab']c                 C  s  t |}t|rtd| d|du r|dv }|dur$|dv s$J d|r.t|r.J dt|rEd|v rE| |dkrE| | | |du rM| jj}|r|d	vr\td
| d|durdtdt|rt	j
||d}d|v r{t	j||d}nrt	j||d}njt|r|rtd|dv rt| j|}nT|dv rtj| j||d}t	j||d}n?td| dt|r|dv rt| j||}n&|dv rtj| j|||d}t	j||d}ntd| dtd| dtt|}	d|v r|	S t	j|	dd}
t|
drt|
d| tt|
S d}d}|dvr!td| d|dur/|dvr/td| |}|dkr;d}t|sEt|r|}|dv rdt }| ||}| |rb| || nW|dv r|du rt }| ||}| || n;t|std| d| | t !|"d# }| || d}| || d }t$%| d}t|rt&| j|}|du rt'd!| d|j(dusJ |j(}|j!}n0t|rt&| j|}|du rt'd!| d|j(dusJ |j(}|j!}ntd| dd"}|du rd#}n| |||}| | }|r| j||d#d#d$}|dus:J d%| |||}t)j| |d#d& t)j*|rXt)+| nt),|| |du rt|rrt-| j||| nt|rt-| j||| n|dusJ | |||}W d   n	1 sw   Y  nt }| ||}nt|r|}ntd| dt.| |||||d'}d|v rt	j||d}nt	j||d}tt|}	d|v r|	S t	j|	dd}
tt|
S )(a  
        Open a local or remote file for reading or writing

        Args:
            path: local or remote path
            mode: one of "r", "rb", "w", "wb", "a", "ab" indicating the mode to open the file in
            streaming: the default for `streaming` is `True` when `mode` is in `"r", "rb"` and `False` when `mode` is in `"w", "wb", "a", "ab"`.
                * `streaming=True`:
                    * Reading is done without downloading the entire remote file.
                    * Writing is done to the remote file directly, but only in chunks of a few MB in size. `flush()` will not cause an early write.
                    * Appending is not implemented.
                * `streaming=False`:
                    * Reading is done by downloading the remote file to a local file during the constructor.
                    * Writing is done by uploading the file on `close()` or during destruction.
                    * Appending is done by downloading the file during construction and uploading on `close()` or during destruction.
            buffer_size: number of bytes to buffer, this can potentially make reading more efficient.
            cache_dir: a directory in which to cache files for reading, only valid if `streaming=False` and `mode` is in `"r", "rb"`.   You are reponsible for cleaning up the cache directory.
            file_size: size of the file being opened, can be specified directly to avoid checking the file size when opening the file.  While this will avoid a network request, it also means that you may get an error when first reading a file that does not exist rather than when opening it.  Only valid for modes "r" and "rb".  This valid will be ignored for local files.
            version: a version number of the file being opened, used to prevent overwriting a file that has changed since it was opened.  Only valid for modes "w", "wb", "a", "ab"

        Returns:
            A file-like object
        rA   rB   N)r   rE   z'Can only specify file_size when readingz1Cannot specify version when writing to local filerd  r   )rd  rG   r   rE   z"Invalid mode for streaming file: 'z,Cannot specify cache_dir for streaming filesrj  r   )rm  z$Cannot specify version for GCP files)rd  rG   )r   zUnsupported mode: ')r   rH   r{   rF  utf8)encoding_CHUNK_SIZE)rd  rG   r   rE   rC  abzInvalid mode: 'z%cache_dir only supported in read modez	local.tmp)rC  rz  z!cache_dir must be a local path: 'z.lockz.tmpr  FT)r8   r=   zfailed to return md5r   )ctx
local_pathrj  tmp_dirremote_pathrH   )/r!   rJ   rK   rP   r/  r   r3   default_buffer_sizer   rO  FileIOrR  rS  rO   NotImplementedErrorrT   rV  rU  rN   rR   r   r   rP  hasattrsetattrr   r   tempfilemkdtempr-  rL   ry   rb   rc   encoderg   filelockFileLockr}   r   rH   r|   rz   r   replacerK  
_ProxyFile)r4   rz   rj  rF   rm  ro  rp  rH   r%  binary_ftext_fr~  r}  local_filenamer|  path_md5	lock_pathtmp_pathremote_versionr   remote_hashperform_copyexpected_local_pathlocal_hexdigestr-   r-   r.   ra   ?  s$  !




















>




)r2   r   )FFNFN)r6   r   r7   r   r8   r9   r:   r9   r;   r<   r=   r9   r>   r?   r@   r?   )rz   r   r@   r9   )rz   r   r@   ri   )F)r   ri   r:   r9   r@   r   )Fr   )r   ri   r:   r9   r   r   r@   r   )r   )rz   r   r   r   r@   r   )rz   r   r   r   r@   r   )rz   r   r@   r   )rz   r   r@   r   r,   )rz   r   r   r  rH   r?   r@   r9   )FN)rz   r   r:   r9   r;   r<   r@   r   )TN)r&  r   r'  r9   r(  r)  r@   r*  )rC  r   r+   ri   r@   ri   )rz   r   r@   rG  )rM  rN  r@   r?   )rz   r   r@   r`  )rz   r   rb  ri   r@   r   )rz   r   rh  r`  r@   r   )...NN)rz   r   rj  rk  rF   rl  rm  rn  ro  r?   rp  rn  rH   r?   r@   r   )....NN)rz   r   rj  rt  rF   rl  rm  rn  ro  r?   rp  rn  rH   r?   r@   r   )r   NNNNN)rz   r   rj  ru  rF   rl  rm  rn  ro  r?   rp  rn  rH   r?   )__name__
__module____qualname__r5   ry   rL   r   r   r   r~   r   r   r   r   r  r   r  r  r4  r/  r-  rH  rc   rZ  r^  ra  rf  ri  r   ra   r-   r-   r-   r.   r1   M   sv    

e

m
G


:^
V
	


#



r1   msgri   r@   r   c                 C  s   t d|   d S )Nz
blobfile: )print)r  r-   r-   r.   default_log_fn#  s   r  rz   r9   c                 C  s   t j| }|jdkS )Ngs)urllibparseurlparseschemerz   r   r-   r-   r.   rO   '  s   
rO   c                 C  s,   t j| }|jdko|jdp|jdkS )Nhttpsz.blob.core.windows.netaz)r  r  r  r  netlocr   r  r-   r-   r.   rN   ,  s
   rN   Optional[ModuleType]c                 C  s   t | rtS t| rtS d S r,   )rO   rT   rN   rR   rz   r-   r-   r.   _get_module3  s
   r  c                 C  s   t | d u S r,   )r  r  r-   r-   r.   rP   <  r0   rP   r2   r   r6   r7   startr   r   src_file_sizec              	   C  s   t | }|j|d|dg}|| t|dD}|| d}		 tt||	 }
|
dks-J ||
}|dkrH|	|krGtd| d| d	|	 n|| |	t	|7 }	q W d    n1 s^w   Y  W d    d S W d    d S 1 svw   Y  d S )
NrE   )rp  zrb+r   TrI   z&read wrong number of bytes from file `z`, expected z
 but read )
r1   ra   seekopenminr   rd   r   rf   r   )r2   r6   r7   r  r   r  r{  rt   ru   
bytes_readnrw   r-   r-   r.   _download_chunk@  s0   



	"r  rq   concurrent.futures.Executorr=   r?   c                 C  sf  t | d}||}tj|dkrtjtj|dd t|d}|jdkr5||jd  |	d W d    n1 s?w   Y  t
|d	t pLd}tt|j| tj}	d}
g }|
|jk r|t| |||
t|	|j|
 |j}|| |
|	7 }
|
|jk sc|D ]}|  q|r||d
}tt|dW  d    S 1 sw   Y  d S d S )Nr2   r   Tr   rG   r   r       _max_workersrE   rw  )r1   r   r|   rz   r/  r   r  r   r  rf   getattr	cpu_countmaxmathceilr_   PARALLEL_COPY_MINIMUM_PART_SIZEr"  r  r  r   r#  ra   binasciihexlifyrI  decode)r2   rq   r6   r7   r=   r{  r   r%  max_workers	part_sizer  rX   futurer-   r-   r.   rQ   X  s8   







$rQ   s1s2c                 C  s<   t t| t|}t|D ]}| | || kr|  S q|S r,   )r  r   r   )r  r  lengthr   r-   r-   r.   _string_overlapy  s   r  	List[str]c                 C  sF   g }d}| D ]}||7 }|dkr| | d}q|dkr!| | |S )Nr   r   )r   )rz   rA  partr   r-   r-   r.   r     s   

r   r   r  r   c           
      c  s    t | r	tj}nt| rtj}ntd|  d| }|D ]0}t|}t||}||d  }|d | }t|d d D ]}	||	7 }||V  q<|V  |}qd S )Nr   rB   r   )	rO   rT   entry_from_dirpathrN   rR   r   r  r  r   )
r   r  r  previous_pathr   r  offsetr@  r1  r  r-   r-   r.   _expand_implicit_dirs  s$   
r  r   r   r5  c                 C  sr   dd t d| D }d}|D ]}|dkr|d| d7 }q|dkr'|d	7 }q|t |7 }qt || d
 S )Nc                 S  s   g | ]}|d kr|qS )r   r-   )r   r   r-   r-   r.   r     s    z$_compile_pattern.<locals>.<listcomp>z([*]+)r   r   z[^z]***z.*z?$)rer   escapecompile)r   r5  tokensregexptokr-   r-   r.   _compile_pattern  s   
r  r   c                 c  sd    | d\}}}t|}t|t| |ddD ]}t|}t||r/||kr,|jr,q|V  qd S )Nr   r2   rz   )r   r  )	partitionr  r  _list_blobsr  r9   matchr   )r2   r   r   ro   
re_patternr   r  r-   r-   r.   
_glob_full  s   r  c                   @  s   e Zd ZU ded< ded< dS )r   ri   r1  zSequence[str]remNr  r  r  __annotations__r-   r-   r-   r.   r     s   
 r   c                   @  s   e Zd ZU ded< dS )r   r   r   Nr  r-   r-   r-   r.   r     s   
 r   c                   @  s   e Zd ZdS )r   N)r  r  r  r-   r-   r-   r.   r     s    r   r   &Iterator[Union[_GlobTask, _GlobEntry]]c                 c  sD   |j |jd  }|jdd  }d|v r*t| || d| D ]}t|V  q d S d|v r~t|| }|d\}}}|| }	t| |	ddD ]4}t|}
|
|	krU|j	rUqGt
||
r{t|dkrht|V  qG|
|soJ t|
t|d  |V  qGd S t|dkr|| }	t| |	}|d urt|V  d S d S t||V  d S )Nr   r   r  r   r   r   r2   rz   r,  )r1  r  r  r-  r   r  r  r  r  r   r9   r  r   r  r   r   )r2   r   r   r1  r  r   r  r   ro   rz   r  r-   r-   r.   r     s:   
r   r   mp.Queue[_GlobTask]r   9mp.Queue[Union[_GlobEntry, _GlobTask, _GlobTaskComplete]]c                 C  s6   	 |  }t| ||dD ]}|| q|t  q)NT)r   r   )r   r   r   r   )r2   r   r   r   r   r   r-   r-   r.   r     s   r   r   c                 #  s   t j| }| ds| dr|t j7 }d|v r|dd }|t jrAt j|}|t js8|t j7 }|t|d  }n t j|}t j|}t|dkrYt j| }n|t|d  }|| }t	|t jd}t 
|D ]3\ }}	 fdd|D }
|
 fdd|	D 7 }
|
D ]}t||r|t jr|d d	 }|V  qqqd S t j| }t j|r|t jr|d d	 }|V  d S d S )
Nr   \r   r   )r5  c                   s    g | ]}t j |t j qS r-   )r|   rz   r-  r5  r.  r   r-   r.   r   #  s     z_local_glob.<locals>.<listcomp>c                   s   g | ]	}t j |qS r-   )r|   rz   r-  )r   filenamer  r-   r.   r   $  s    r   )r|   rz   r3  r   r5  r   r   r   r/  r  r4  r  r  rL   )r   normalized_patternr   base_dirr   rB  full_patternr  r;  r<  pathsrz   r-   r  r.   r     sF   

	
r   c                 C  s   |  dr| d d S | S r   )r   r  r-   r-   r.   r8  2  s   
r8  c                 C  s:   t | rtj| rdS t| st| r| drdS dS )zL
    Guess if a path is a directory without performing network requests
    Tr   F)rP   r|   rz   r~   rO   rN   r   r  r-   r-   r.   rJ   9  s
   rJ   r,  c                 c  sj    i }|d ur||d< t |rtj| ||dE d H  d S t|r-tj| ||dE d H  d S td| d)Nr,  r+  r{   rB   )rO   rT   r  rN   rR   r   )r2   rz   r,  paramsr-   r-   r.   r  D  s   r  r   r   c                 C  s   | j r| jd S | jS )Nr   )r   rz   )r   r-   r-   r.   r  Q  s   r  c                 C  s$   t |rtj| gt|R  S |S r,   )rN   rR   r   rU   r  r-   r-   r.   r7  U  s   r7  r   r   c                 c  s<    t | |}t| |ddD ]}|rt||krq|V  qd S )Nr   r  )r7  r  r  )r2   r   r   normalized_prefixr   r-   r-   r.   r   \  s   
r   Optional[DirEntry]c                 C  s   t | }t|r.t| |}|d ur"|drt|S t||S ||r,t|S d S t|rXt	| |}|d urL|drFt	|S t	||S ||rVt	|S d S t
d| d)Nr   r{   rB   )r1   rO   rT   r}   r   r  entry_from_path_statr~   rN   rR   r   )r2   rz   r{  r   r-   r-   r.   r   e  s(   







r   r   mp.Queue[Tuple[str, str, bool]]r   mp.Queue[Optional[DirEntry]]c           	      C  sn   	 | d\}}}|r|| }t| |}|d ur|| nt| || dd}|D ]}|| q)|d  q)NTF)r   )r   r   r   r   )	r2   r   r   baser   r   rz   r   r  r-   r-   r.   r     s   


r   rC  rF  c                 C  sR   t |rtj||S t|rt| ||S t|r!t| ||S t	d| dr   )
rP   r|   rz   r-  rO   rT   
join_pathsrN   rR   r   )r2   rC  rF  r-   r-   r.   rD    s   rD  c                      s,   e Zd Zd fddZd fddZ  ZS )r  r{  r1   r|  ri   rj  ru  r}  r?   r~  rH   r@   r   c                   s>   t  j||d || _|| _|| _|| _|| _d| _|| _d S )Nrv  F)	superr5   _ctx_mode_tmp_dir_local_path_remote_path_closedrX  )r4   r{  r|  rj  r}  r~  rH   	__class__r-   r.   r5     s   	
z_ProxyFile.__init__c              	     s   t | dr| jr
d S t   z*| jd ur'| jdv r'| jj| j| jd| j	d W | j
d ur9t| j t| j
 n| j
d urLt| j t| j
 w w d| _d S )Nr  )rd  rG   rC  rz  T)r8   r>   )r  r  r  closer  r  r  ry   r  rX  r  r|   r   r  )r4   r  r-   r.   r    s$   



z_ProxyFile.close)r{  r1   r|  ri   rj  ru  r}  r?   r~  r?   rH   r?   r@   r   )r@   r   )r  r  r  r5   r  __classcell__r-   r-   r  r.   r    s    r  TFspawn)rj   connection_pool_max_sizemax_connection_pool_countazure_write_chunk_sizegoogle_write_chunk_sizeretry_log_thresholdretry_common_log_thresholdrh   connect_timeoutread_timeoutoutput_az_paths&use_azure_storage_account_key_fallbackget_http_pooluse_streaming_readr  get_deadlinesave_access_token_to_diskr\   rj   Callable[[str], None]r  r  r  r  r  r  rh   rn  r   r  r  r  r  +Optional[Callable[[], urllib3.PoolManager]]r  r  r  Optional[Callable[[], float]]r  r\   c                 C  s   t di d| d|d|d|d|d|d|d|d	|d
|	d|
d|d|d|d|d|d|d|}t|dS )zm
    Same argument as configure(), but returns a Context object that has all the blobfile methods on it.
    rj   r  r  r  r  r  rh   r  r   r  r  r  r  r  r  r  r  r\   r  Nr-   )r   r1   )rj   r  r  r  r  r  r  rh   r   r  r  r  r  r  r  r  r  r\   r2   r-   r-   r.   create_context  sL   	

r  )r)   r*   r+   r   )r  ri   r@   r   )rz   ri   r@   r9   )rz   ri   r@   r  )r2   r   r6   ri   r7   ri   r  r   r   r   r  r   r@   r   )r2   r   rq   r  r6   ri   r7   ri   r=   r9   r@   r?   )r  ri   r  ri   r@   r   )rz   ri   r@   r  )r   ri   r  r   r@   r   r   )r   ri   r5  ri   )r2   r   r   ri   r@   r   )r2   r   r   ri   r   r   r@   r  )
r2   r   r   ri   r   r  r   r  r@   r   )r   ri   r@   r   )rz   ri   r@   ri   r,   )r2   r   rz   ri   r,  r?   r@   r   )r   r   r@   ri   )r2   r   rz   ri   r@   ri   )r2   r   r   ri   r   r9   r@   r   )r2   r   rz   ri   r@   r  )r2   r   r   r  r   r  r@   r   )r2   r   rC  ri   rF  ri   r@   ri   )$rj   r  r  r   r  r   r  r   r  r   r  r   r  r   rh   rn  r   rn  r  rn  r  r9   r  r9   r  r	  r  r9   r  r   r  r
  r  r9   r\   ri   )j
__future__r   r  r   concurrent.futuresrW   r   rb   rO  r   r  multiprocessingrZ   r|   r  r  r   r   r  rk   urllib.parser  	functoolsr   typesr   typingr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r  urllib3blobfiler   rR   r   r_   r   rT   blobfile._commonr   r   r   r   r   r   r   r   r   r   r    r!   setr   r   r   DEFAULT_AZURE_WRITE_CHUNK_SIZEDEFAULT_GOOGLE_WRITE_CHUNK_SIZEDEFAULT_RETRY_LOG_THRESHOLD"DEFAULT_RETRY_COMMON_LOG_THRESHOLDDEFAULT_CONNECT_TIMEOUTDEFAULT_READ_TIMEOUTDEFAULT_BUFFER_SIZEr/   r1   r  rO   rN   r  rP   r  rQ   r  r   r  r  r  r   r   r   r   r   r   r8  rJ   r  r  r7  r   r   r   rD  r  r  r  r-   r-   r-   r.   <module>   s   @8*
       
]



	


!




"

-




	

'