o
    پi^                    @   s<  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ dZ0dZ1dZ2d	Z3d
Z4dZ5dZ6dddddddZ7dee8ef fddZ9dee8 fddZ:de8de8de8de8fddZ;	ddee8e8f d e8d!ee< de(fd"d#Z=d$e(d%ee8e8f de(fd&d'Z>d(ee8e8f d)e8dee8e?f fd*d+Z@d,e8dee8e8e8f fd-d.ZAd,e8dee8e8e8f fd/d0ZBd,e8dee8e8e8f fd1d2ZCde8d3e8d4e8de8fd5d6ZDde8d3e8d4e8de8fd7d8ZEd9e$de8d3e8d4e8de8f
d:d;ZFd9e$d,e8ddfd<d=ZGd$e(d(e8de8fd>d?ZHd@ee8ef dee8 fdAdBZIdCdDdEdFdGdHdIdJdKdLdMdNdOZJdPe8de?fdQdRZKdSee8e8f de*fdTdUZLd9e$de8d3e8d%ee8e8f dVee) deMfdWdXZNd9e$dYe8de8d%ee8e8f dee8 f
dZd[ZOd9e$d%ee8e8f dee8 fd\d]ZPd9e$de8d3e8dee8ef dVee) deeee?f  fd^d_ZQd9e$d(edeee?f fd`daZRd9e$d(edeee?f fdbdcZSd9e$d$e(dddfdedfZTdge<dhe<de8fdidjZUd9e$d)e8d@ee8e8f dedd fdkdlZVd9e$d,e8d)e8dmee8 dneeW doee8 dddfdpdqZXd9e$d,e8deMfdrdsZY		dd9e$d)e8dte8deee8e8f  dueee8e8f  deee8ef  fdvdwZZG dxdy dye!Z[G dzd{ d{e"Z\d9e$d,e8d|e<d}e<d)e8d~e8ddfddZ]d9e$dej^j_de8de8deMdee8 dee8 fddZ`ddd9e$d,e8doee8 dee* fddZad9e$d,e8deMfddZbd9e$d,e8de8de8deMf
ddZcdd9e$d,e8dee8 dee% fddZdd,e8de%fddZed,e8de*de%fddZfd9e$de8d3e8dee8ef dee% f
ddZgd9e$d,e8dee8ee? f fddZhdd9e$d,e8de?doee8 deMf
ddZid9e$d,e8de8fddZjd9e$de8de8deMdee8 f
ddZkd9e$de8de8de8fddZld9e$de8d|e<d}e<de8d~e8ddfddZmd9e$dej^j_de8de8deMdee8 dee8 fddZne+eRdZoe+eSdZpdS )    N)AnyDictIteratorListMappingOptionalSequenceTuple)_common)_xml)DEFAULT_RETRY_CODESINVALID_HOSTNAME_STATUSBaseStreamingReadFileBaseStreamingWriteFileConcurrentWriteFailureConfigDirEntryErrorFileBodyRequestRequestFailureStatTokenManagerVersionMismatch	path_joinrngstrip_slashes
shared_keyoauth_token	anonymousi  iQ i iP  zx-ms-blob-cache-controlzx-ms-blob-content-typex-ms-blob-content-md5zx-ms-blob-content-encodingzx-ms-blob-content-languagezx-ms-blob-content-disposition)zCache-ControlContent-TypeContent-MD5Content-EncodingContent-LanguagezContent-Dispositionreturnc                  C   s  t dddkrddiS dD ]!} | t jv r/i }dt jv r#t jd |d< d	t j|  d
|  S qdt jv rZi }t jd dD ]}|d\}}}||| < q?d	|d |d dS dt jv rt jd }t j|srtd| dt	|}t
|}	d|	d |	d |	d dW  d    S 1 sw   Y  dt jv rdt jd t jd t jd dS t jd}
t j|
rt	|
.}t
|}|di  D ]}|d dkrqd|d  d!  W  d    S W d    n1 sw   Y  t jd"}t j|rWt	|J}t
|}d }|D ])}d#|vrqd|d# d!}	|d u r'|	}q|d$d%|d$d%kr6|	}q|d urF|W  d    S W d    i S 1 sRw   Y  i S )&NAZURE_USE_IDENTITY01_azure_authazure-identity)AZURE_STORAGE_KEYAZURE_STORAGE_ACCOUNT_KEYAZURE_STORAGE_ACCOUNTaccountsakey)r)   storage_account_keyAZURE_STORAGE_CONNECTION_STRING;=accountname
accountkey)r)   r.   r0   AZURE_APPLICATION_CREDENTIALSzCredentials not found at 'zC' specified by environment variable 'AZURE_APPLICATION_CREDENTIALS'svcactappIdpasswordtenant)r)   	client_idclient_secret	tenant_idAZURE_CLIENT_IDAZURE_CLIENT_SECRETAZURE_TENANT_IDz~/.azure/msal_token_cache.jsonRefreshTokencredential_typerefreshsecret)r)   refresh_tokenz~/.azure/accessTokens.jsonrefreshToken	expiresOn )osgetenvenvironsplit	partitionlowerpathexistsr   openjsonload
expandusergetvalues)varnamer.   connection_datapartkey_val
creds_pathfcredsmsal_tokens_pathtokenstokenaccess_tokens_path
best_token re   C/home/ubuntu/.local/lib/python3.10/site-packages/blobfile/_azure.py_load_credentialsB   s   







 
		







rg   c                  C   s   t jd} t j| sg S t| d}t| d}W d   n1 s(w   Y  |	dg }dt
ttf dtfdd	}|j|d
d dd |D S )z
    Return a list of subscription ids from the local azure profile
    the default subscription will appear first in the list
    z~/.azure/azureProfile.jsonrbz	utf-8-sigNsubscriptionsxr%   c                 S   s   | d S )N	isDefaultre   )rj   re   re   rf   key_fn   s   z%load_subscription_ids.<locals>.key_fnT)rZ   reversec                 S      g | ]}|d  qS )idre   ).0subre   re   rf   
<listcomp>       z)load_subscription_ids.<locals>.<listcomp>)rI   rO   rT   rP   rQ   rR   loadsreaddecoderU   r   strr   boolsort)default_profile_pathr^   profileri   rl   re   re   rf   load_subscription_ids   s   r|   r.   templatedatac                 K   s   t jd|  d|fi |S )Nhttps://.blob.core.windows.net)common	build_url)r.   r}   r~   re   re   rf   r      s   r      r_   scopesuccess_codesc                 C   s~   | d dkrd| d |d}d}n| d dkr'd| d | d	 |d
}| d }nt td| dddditj|d|dS )Nr)   rC   rE   )
grant_typerE   r   r   r7   client_credentialsr;   r<   )r   r;   r<   r   r=   z"https://login.microsoftonline.com/z/oauth2/v2.0/tokenPOSTr!   z!application/x-www-form-urlencodedutf8)urlmethodheadersr~   r   )AssertionErrorr   urllibparse	urlencodeencode)r_   r   r   r~   r=   re   re   rf   _create_access_token_request   s*   

r   reqauthc              
   C   s   | j d u ri }nt| j  }| jd u ri }nt| j }d|d< tjjtjjdd|d< | j	}|d urAt
|trAt|}t| j| j|||| jt| jt| jd}|\}}|tkrgt|||d< |S |tkrtd| |d< |S |tkry	 |S )	Nz
2019-02-02zx-ms-versiontzz%a, %d %b %Y %H:%M:%S GMTz	x-ms-date)r   r   paramsr   r~   preload_contentr   retry_codesAuthorizationzBearer )r   dictcopyr   datetimenowtimezoneutcstrftimer~   
isinstancexmlunparser   r   r   r   tupler   r   
SHARED_KEYsign_with_shared_keyOAUTH_TOKEN	ANONYMOUS)r   r   r   r   r~   resultkindrb   re   re   rf   create_api_request   sB   


r   rZ   r   c                 C   s  i d| d d| d d| d d| d d| d d	| d
 d| d d| d ddddddddddddddddddddd}t j|}|jdd }t jd | d!|jd"d   }|d |d |d ||d |d	 |d |d |d |d |d |d |d |d |d |d |d |d# |d$ |d f}d%|}t	t
t| d& |d'd(d'|d)< t jd*d+ | D }t| d d,d-d.}	t|	}
|d/ | |
fS )0NstSignedStartseSignedExpirysksSignedServicesktskesktid	SignedTidskoid	SignedOidskvSignedVersionsvz
2018-11-09srbsprsiprH   sisprz
https,httprsccrscdrsce)rsclrsct.r   z/blob//   r   r   
Valuer   sha256sigc                 S   s   i | ]\}}|d kr||qS )rH   re   rp   kvre   re   rf   
<dictcomp>6  s    z'generate_signed_url.<locals>.<dictcomp>ZGMTz%Y-%m-%dT%H:%M:%S%Z?)r   r   urlparsenetlocrL   unquoterO   joinbase64	b64encodehmacdigest	b64decoder   rv   r   itemstimestrptimereplacecalendartimegm)rZ   r   r   ustorage_accountcanonicalized_resourceparts_to_signstring_to_signquerytstre   re   rf   generate_signed_url   s   	
"

r   rO   c                 C   s4   |  dr	t| S |  drt| S td|  d)Naz://r   Invalid path: '')
startswithsplit_az_pathsplit_https_pathr   )rO   re   re   rf   
split_path=  s
   

r   c                 C   sv   | t dd  d}t |dk rtd|  d|d }|d }|dkr-td|  dd|dd  }|||fS )	Nr   r      r   r   r   r   rH   )lenrL   r   r   )rO   partsr.   	containerobjre   re   rf   r   F  s   
r   c                 C   s   | t dd  d}t |dk rtd|  d|d }|d }|dr*|d	kr2td|  dd|dd  }|d
d }|||fS )Nr   r   r   r   r   r   r   r   rH   r   )r   rL   r   endswithr   )rO   r   hostnamer   r   r.   re   re   rf   r   R  s   
r   r   r   c                 C   s   d|  d| d| S )Nr   .blob.core.windows.net/r   re   r.   r   r   re   re   rf   combine_https_path_     r   c                 C   s   d|  d| d| S )Nr   r   re   r   re   re   rf   combine_az_pathc  r   r  confc                 C   s   | j r	t|||S t|||S N)output_az_pathsr  r   )r  r.   r   r   re   re   rf   combine_pathg  s   r  c                 C   sh   | ds	|d7 }t|\}}}tt|d||ddddidd}t| |}|jd	kr2td
| dd S )Nr   /{container}/{blob}r   blobPUTx-ms-blob-type	BlockBlob)     r   r   r   r   r  z?Unable to create directory, account/container does not exist: 'r   )r   r   r   r   execute_api_requeststatusr   r  rO   r.   r   r  r   respre   re   rf   	mkdirfilen  s   


r  c                 C   s  g }| j d ur| j  D ]\}}| }|| d|  qtj| j}|j	dd }d| d|j
dd   }d|gtt| }	| jd u rOi }
nt| j}
g }|
 D ] \}}| }tdd| }|d	rz|| d|  qZdt|}|
d
d}| jd urtt| j}| j|
dd|
dd||
dd|
dd|
dd|
dd|
dd|
dd|
dd|
dd||	g}d|}ttt||ddd}d| d| S )N:r   r   r   r   r   z\s+ zx-ms-Content-LengthrH   r#   r$   r"   r!   DatezIf-Modified-SinceIf-MatchzIf-None-MatchzIf-Unmodified-SinceRanger   r   z
SharedKey ) r   r   rN   appendr   r   r   r   r   rL   rO   r   listsortedr   r   rerq   stripr   rU   r~   rw   r   r   r   r   r   r   r   r   rv   )r   rZ   params_to_signnamevaluecanonical_namer   r   canonical_urlr   r   headers_to_signcanonical_valuecanonicalized_headerscontent_lengthr   r   	signaturere   re   rf   r   }  sZ   















r   metadatac                 C   s.   d| v r| d }|d u rd S t | S d S )Nr"   )r   r   hex)r)  b64_encodedre   re   rf   _get_md5  s   r,  010203040506070809101112)JanFebMarAprMayJunJulAugSepOctNovDectextc                 C   sL   |   }|d  dt|d   d|d  d|d  d}tj|}| S )N   -r   r   T   z+00:00)rL   _MONTH_NAME_TO_INDEXr   fromisoformat	timestamp)rE  pdate_stringdtre   re   rf   _parse_timestamp  s   0rP  itemc                 C   sb   d| v r	| d }n| d }d| v rt | d }nt| d }tt| d |t|t| | d dS )NzCreation-Timezx-ms-creation-timex-ms-meta-blobfilemtimezLast-Modifiedr  Etag)sizemtimectimemd5version)floatrP  r   intr,  )rQ  	raw_ctimerU  re   re   rf   	make_stat  s   

r\  out_failuresc                    s   dddt gd tkrd dtf fdd}t| |}|jt kr*d	S |jdkrBd tkrB|td
| | dS |jdv rId	S |td
| | dS )Nr       r     r%   c                     s.   t dt ddddddd} t| d	S )
NGET/{container}r   r   r  r(   )restypecomp
maxresultsr   r   r   r   r   )r   r   r   r   r.   r   r   r   re   rf   	build_req  s   
z(_can_access_container.<locals>.build_reqTzCould not access containerFr   r_  )	r   r   r  r   r   execute_requestr  r   create_from_request_response)r  r.   r   r   r]  rk  r  re   rj  rf   _can_access_container  s,   
	

ro  subscription_idc                    s   d| dddi	 dt f fdd}t| |}|jd	v r#d S t|j}|d
 D ]}|d |kr;|d   S q-d|vrBd S |d d q)Nz+https://management.azure.com/subscriptions/z,/providers/Microsoft.Storage/storageAccountsapi-version
2019-04-01Tr%   c                     s   t ddd} t|  dS )Nra  )r     r^  rg  rh  r   r   ri  r   r   r   re   rf   rk  #  s   z*_get_storage_account_id.<locals>.build_req)rs  r^  r!  r   ro   nextLink)r   r   rm  r  rR   rt   r~   )r  rp  r.   r   rk  r  outr   re   ru  rf   _get_storage_account_id  s"   
rx  c                    sr   dddig }	 dt f fdd}t| |}t|j}|dd	 |d
 D  d|vr2|S |d d q	)Nz*https://management.azure.com/subscriptionsrq  z
2020-01-01Tr%   c                     s   t dd} t|  dS )Nra  r   r   r   rh  rt  ri  ru  re   rf   rk  ?  s   z(_get_subscription_ids.<locals>.build_reqc                 S   rn   )subscriptionIdre   )rp   rQ  re   re   rf   rr   E  rs   z)_get_subscription_ids.<locals>.<listcomp>r!  rv  )r   r   rm  rR   rt   r~   extend)r  r   r   rk  r  r~   re   ru  rf   _get_subscription_ids9  s   r|  c                    s:  dt ffdd}t| |}t|j}t|d f t d D ]}t| || d ur1 n#q"t	|  }	fdd|	D }
|
D ]}t| || d urQ nqBd S dt f fdd}t| |}t|j}|d	 D ]&}|d
 dkrt
|d f}t| ||||dr|  S td| d| dqntd| d)Nr%   c                         t  ddS )Nz%https://management.azure.com/.defaultr_   r   r   re   r_   re   rf   build_req_access_token[     z8_get_storage_account_key.<locals>.build_req_access_tokenaccess_tokenc                    s   g | ]}| vr|qS re   re   )rp   ro   )stored_subscription_idsre   rf   rr   p  s    z,_get_storage_account_key.<locals>.<listcomp>c                     s&   t dd dddid} t|  dS )Nr   zhttps://management.azure.comz	/listKeysrq  rr  ry  rh  rt  ri  )r   storage_account_idre   rf   build_req_list_keys|  s   
z5_get_storage_account_key.<locals>.build_req_list_keyskeyspermissionsFULLr!  r]  zIFound storage account key, but it was unable to access storage account: '' and container: 'r   zCStorage account was found, but storage account keys were missing: ')r   r   rm  rR   rt   r~   r   r|   rx  r|  r   ro  r   )r  r.   r   r_   r]  r  r  r   rp  subscription_idsunchecked_subscription_idsr  rZ   storage_key_authre   )r   r_   r  r  rf   _get_storage_account_keyL  sH   



r  c                    sj  |\ }t   }t d}g }|dkrDdv r-d  kr-tdd  d  dtd f}t|  |||drB||t fS n1|d	krd
tf fdd}t	| |}t
|j}	|jdkr|	d dkrod|	d v s|	d dkr{d|	d v s|	d dkrd|	d v rtdtd|	d  d|	d  dt|	d f}t|  |||dr||t|	d  fS | jrt|  ||d}
|
d ur|
|t fS n|dkrd
tffdd}t	| |}t
|j}	t|	d f}t|  |||dr||t|	d  fS | jrt|  ||d}
|
d ur|
|t fS n_|dkruzdd lm} W n ty/   td!w | }|d"}W d    n	1 sDw   Y  t|jf}t|  |||dr]||jfS | jrut|  ||d}
|
d uru|
|t fS td#f}t|  |||dr|td$fS d%  d&| d}t|dkr|D ]
}|d'| 7 }qtdkr|d(7 }t|))Nr)   r/   r.   z*Provided storage account key for account 'zA' via environment variables, but needed credentials for account 'r   r0   r  rC   r%   c                      s   t d  dddS )Nr   z.blob.core.windows.net/.default)r   r  )r_   r   r   r  re   r.   r_   re   rf   rk    s
   
z$_get_access_token.<locals>.build_reqr  errorinvalid_grantAADSTS700082error_descriptioninteraction_requiredAADSTS50078AADSTS50076zMYour refresh token is no longer valid, please run `az login` to get a new onez7Encountered an error when requesting an access token: `z: z9`.  You can attempt to fix this by re-running `az login`.r  
expires_in)r  r.   r   r_   r]  r7   c                      r}  )N"https://storage.azure.com/.defaultr~  r  re   r  re   rf   rk    r  r*   r   )DefaultAzureCredentialzSWhen setting AZURE_USE_IDENTITY=1, you must also install the azure-identity packager  rH   infzFCould not find any credentials that grant access to storage account: 'r  z
    Access Failure: a  

No Azure credentials were found.  If the container is not marked as public, please do one of the following:

* Log in with 'az login', blobfile will use your default credentials to lookup your storage account key
* Set the environment variable 'AZURE_STORAGE_KEY' to your storage account key which you can find by following this guide: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage
* Create an account with 'az ad sp create-for-rbac --name <name>' and set the 'AZURE_APPLICATION_CREDENTIALS' environment variable to the path of the output from that command or individually set the 'AZURE_CLIENT_ID', 'AZURE_CLIENT_SECRET', and 'AZURE_TENANT_ID' environment variables)r   rg   rU   r   r   ro  SHARED_KEY_EXPIRATION_SECONDSr   r   rm  rR   rt   r~   r  r   rY  &use_azure_storage_account_key_fallbackr  azure.identityr  ImportErrorRuntimeError	get_tokenrb   
expires_onr   r   )r  rZ   r   r   
azure_authaccess_failuresr   rk  r  r   storage_account_key_authr  credrb   anonymous_authmsgresp_failurere   r  rf   _get_access_token  s   








r  c                    s   t jd}|d tkrd t t fS \ dtf fdd}t|}|jdkr8t	d  dt
|j}t t }|d	 |fS )
NrZ   r   r%   c                     s   t j jt jjd} | t jdd d}| t jdd d}td  dd	td
ddd||didd}tj	d}|d t
krMtd  d dt||dS )Nr   )hoursz%Y-%m-%dT%H:%M:%SZ   )daysr   r   r   serviceuserdelegationkey)rd  re  KeyInfo)StartExpiry)r   r^  r   r   r   r~   r   r  r   zOnly OAuth tokens can be used to get SAS tokens. You should set the Storage Blob Data Reader or Storage Blob Data Contributor IAM role. You can run `az storage blob list --auth-mode login --account-name z --container z0` to confirm that the missing role is the issue.rh  )r   r   r   r   	timedeltar   r   r   access_token_managerr  r   r   r   )r   startexpiryr   r   r.   r  r   rZ   re   rf   rk  ,  s(   

z!_get_sas_token.<locals>.build_reqr^  z@You do not have permission to generate an SAS token for account zh. Try setting the Storage Blob Delegator or Storage Blob Data Contributor IAM role at the account level.UserDelegationKey)r  r  r   r   SAS_TOKEN_EXPIRATION_SECONDSr   r   rm  r  r   r   r   r~   )r  rZ   r   rk  r  rw  r   re   r  rf   _get_sas_token#  s   

r  urllib3.BaseHTTPResponsec                    sr   t jj}|jdd  |jd}t|dk rtd|jdd dt	f fdd	}t
|S )
Nr   r   r   r   zmissing container from pathr   r%   c                      s   t tj fddS )Nr  rh  )r   r  r  re   r.   r  r   r   re   rf   rk  V  s   z&execute_api_request.<locals>.build_req)r   r   r   r   r   rL   rO   r   r   r   r   rm  )r  r   r   
path_partsrk  re   r  rf   r  N  s   r  index	upload_idc                 C   s>   | dk sJ |d> |  }|dk sJ t |jddddS )Ni      l               big)	byteorderr   )r   r   to_bytesrv   )r  r  id_plus_indexre   re   rf   _block_index_to_block_id^  s   r  c                 C   s   t |tddddd}t| |}|jdkrd S tj|jdhd}|d	 d
 d u r*d S |d	 d
 d }d	ddd |D ii}dd | D }t D ]\}	}
|	|v rX||	 ||
< qJt |dtddi |d|d i|dd}t| |S )N	blocklistre  ra  rl  )r   r   r   r   r   Blockrepeated_tags	BlockListCommittedBlocksLatestc                 S   rn   )Namere   )rp   r   re   re   rf   rr   u  rs   z-_clear_uncommitted_blocks.<locals>.<listcomp>c                 S       i | ]\}}| d r||qS z
x-ms-meta-r   r   re   re   rf   r   w       z-_clear_uncommitted_blocks.<locals>.<dictcomp>r	  r  etag)r  r_    r   r   r   r   r~   r   )	r   r   r  r  r   r   r~   r   !RESPONSE_HEADER_TO_REQUEST_HEADER)r  r   r)  r   r  r   blocksbodyr   srcdstre   re   rf   _clear_uncommitted_blockse  s.   


	r  	block_ids
md5_digestrX  c                 C   s(  dd|ii}i }|d urt |d|d< |d ur||d< t|d|tdd|d	d
ddtfd}t| |}	|	jd
kr`t	|	j
}
|
d d dkrTtjd| d||	dtjd|	j ||	d|	jdksj|	jtkrrtd| d|	jdkr|	jd dkrtjd|	j ||	dtjd||	d|	S )Nr  r  r   r    r  r	  r  r  r  r  r_  r  )r   r   r   r   r~   r   r   CodeInvalidBlockListzMInvalid block list, most likely a concurrent writer wrote to the same path: ``)requestresponseunexpected status messager  r  No such file or directory: 'r   x-ms-error-codeConditionNotMetetag mismatch)r   r   rv   r   r   r   r  r  r   r   r~   r   rn  r   FileNotFoundErrorr   r   )r  rO   r   r  r  rX  r  r   r   r  r   re   re   rf   _finalize_blob  sJ   



r  c           	      C   s   | ds	|d7 }t|\}}}|dkr2tt|d||ddtdddd	tfd
}t| |}|jdkS t| t|d|ddtdd|dddd}|D ]}|d dur^d|d v p[d|d v   S qHdS )z8
    Return true if a path is an existing directory
    r   rH   rb  r  ra  r   )rd  r   r_  )r   r   r   r   rc  r  r(   )re  rd  prefix	delimiterrf  r   r   r   BlobsN
BlobPrefixBlobF)	r   r   r   r   r   r   r  r  create_page_iterator)	r  rO   r.   r   r  r   r  itr   re   re   rf   isdir  s2   



r  r   r   c           
      c   s    |d u ri }nt | }|d u rd }nt | }	 t||||ddtfd}t| |}|jdtfv r6d S tj|jddhdd }	|	V  |	d	 d u rMd S |	d	 |d
< q)NTr   r_  r  r  r  r  EnumerationResults
NextMarkermarker)	r   r   r   r   r  r  r   r   r~   )
r  r   r   r~   r   rM  dr   r  r   re   re   rf   r    s0   
r  c                
       s\   e Zd Zdededee dee ddf
 fddZ	dd	ed
edee ddfddZ	  Z
S )StreamingReadFiler  rO   rT  rX  r%   Nc                    sX   || _ |d u r!t|||d}|d u rtd| d|j}|j| _ t j|||d d S )NrX  r  r   )r  rO   rT  )_version
maybe_statr  rT  rX  super__init__)selfr  rO   rT  rX  r   	__class__re   rf   r    s   zStreamingReadFile.__init__	streamingr  endr  c           
      C   sz   t | j\}}}dtj||di}| jd ur| j|d< tt|d||dd|d| d}t| j|}	| jp9|	j	
d	| _|	S )
Nr  r  r  r  r  r  ra  )   i  )r   r   r   r   r   ETag)r   _pathr   
calc_ranger  r   r   r  _confr   rU   )
r  r  r  r  r.   r   r  r   r   r  re   re   rf   _request_chunk  s   

	z StreamingReadFile._request_chunkr  )__name__
__module____qualname__r   rw   r   rZ  r  rx   r  __classcell__re   re   r  rf   r    s,    r  c                       sH   e Zd Zdededee ddf fddZded	eddfd
dZ	  Z
S )StreamingWriteFiler  rO   rX  r%   Nc           	   	      sR  || _ t|\}}}t|d||d| _tdd| _d| _|| _t	| jd| jr,d| jini ddd	d
t
fd}t||}|jdkr^|jd dkrXt|| j|j}|rW|jd | _nAt|| n;|jdt
fv sq|jd	kry|jd dkrytd| d|jd
kr|jd dkrtjd|j ||dtjd||dt | _t j||jd d S )Nr  r  r       HEADr  r   r  r_  r  r  r
  r  r
  r  ContainerNotFoundz3No such file or container/account does not exist: 'r   r  r  r  r  )r  
chunk_size)r  r   r   _urlr   randint
_upload_id_block_indexr  r   r   r  r  r   r  remover  r   rn  r   hashlibrW  _md5r  r  azure_write_chunk_size)	r  r  rO   rX  r.   r   r  r   r  r  re   rf   r  %  sB   K



zStreamingWriteFile.__init__chunkfinalizec           
   	      sD  d}|t |k ru| jj }||| } j| t|}t jddt	
| ditdt j jd|ddt d	}zt j| W n tyU   ~~|`~ w   jd
7  _ jtkrmtdt dt d|}|t |k s|r fddt jD }t j j j| j  jd}	|	jdp j _d S d S )Nr   r	  r"   r   blockre  blockid)r  )r  )r   r   r   r   r~   r   r   r   zExceeded block count limit of z? for Azure Storage.  Increase `azure_write_chunk_size` so that zI * `azure_write_chunk_size` exceeds the size of the file you are writing.c                    s   g | ]}t | jqS re   )r  r  )rp   ir  re   rf   rr     s    z4StreamingWriteFile._upload_chunk.<locals>.<listcomp>r  rO   r   r  r  rX  r
  )r   r  r  r  updater  rW  r   r  r   r   r   rv   r   r  r  r  r   r  	Exceptionr~   BLOCK_COUNT_LIMITr   ranger  r  r  r   rU   )
r  r   r!  r  r  r~   	block_md5r   r  r  re   r&  rf   _upload_chunk  sZ   


%

z StreamingWriteFile._upload_chunk)r  r  r  r   rw   r   r  
memoryviewrx   r-  r  re   re   r  rf   r  $  s    "xr  r  rT  block_idc              
   C   sj   t |dtd|ddt|it|||| dddtfd}t| |}|jdks+|jtkr3td	| d
d S )Nr	  r"  r#  r  r  r  r_  r  r  r   )r   r   rw   r   r   r  r  r  )r  rO   r  rT  r   r/  r   r  re   re   rf   r-    s   


	r-  executorr  r  
return_md5dst_versionc              
   C   sL  t |d}t|}W d    n1 sw   Y  t|\}}	}
t|d|	|
d}tdd}t|}g }t	|dt
 p>d}ttt|j| tjt}d}d}g }||jk rt||}|t| ||t||j| ||}|| || |d7 }||7 }||jk sZ|D ]}|  qt| |||||d |rt|d	S d S )
Nrh   r  r  r   r  _max_workersr   r'  r   )rQ   r   r,  r   r   r   r  rI   statgetattr	cpu_countminmaxmathceilst_sizePARALLEL_COPY_MINIMUM_PART_SIZEMAX_BLOCK_SIZEr  submitr-  r  r   r  binasciihexlifyrv   )r  r0  r  r  r1  r2  r^   r  r.   r   r  dst_urlr  sr  max_workers	part_sizer%  r  futuresr/  futurere   re   rf   parallel_upload  sJ   





	
rG  r  c          	      C   s   t |\}}}|dkrd S tt|d||ddddtfd}t| |}|jdkr)d S t|j}|d ur?|j|kr?t	j
d||d	|S )
NrH   r  r  r  r   r_  r   r   r   r  r  )r   r   r   r   r  r  r\  r   rX  r   rn  )	r  rO   rX  r.   r   r  r   r  r4  re   re   rf   r     s"   


r   c                 C   sZ   t |\}}}|dkrtd| dtt|d||ddddtfd	}t| |}|jdkS )
NrH   ,The system cannot find the path specified: 'r   r  r  DELETE   r_  rH  )r   r  r   r   r   r  r  r  re   re   rf   r  )  s   

r  r  	hexdigestc                 C   s   t |\}}}tt|d||ddd|idd}t| |}|jdv r#dS i }	t D ]\}
}|
|jv r9|j|
 |	|< q)t	t
|d	|	d
< tt|d||ddtddi |	d|idd}t| |}|jdkS )Nr  r  r  r  r   r_  r  r  )r_  r  Fr   r    r	  
propertiesr  )r   r^  r_  r  r   r   r   r   r   r   )r   r   r   r  r  r  r   r   r   r   r?  	unhexlifyrv   r   )r  rO   r  rL  r.   r   r  r   r  r   r  r  re   re   rf   maybe_update_md56  s>   


	

rQ  r  c           	   
   c   sr    i }|d ur||d< t |\}}}t| t|d|ddtd	dd|d|d}|D ]}t| |||E d H  q*d S )
Nr  rb  rc  ra  r  r   )re  rd  r  r  re   )r   r  r   r   _get_entries)	r  rO   r  r   r.   r   r  r  r   re   re   rf   
list_blobs_  s   rS  c                 C   sD   |  dr| d d } t| \}}}|dd }t|| ddd dS )Nr   r  TFr   rO   is_diris_filer4  r   r   rL   r   )rO   r[   r   r   re   re   rf   entry_from_dirpatho  s
   
rX  r4  c                 C   s<   |  drJ t| \}}}|dd }t|| dd|dS )Nr   r  FTrT  rW  )rO   r4  r[   r   r   re   re   rf   entry_from_path_statw  s   rY  r   c           	      c   s    |d }|d u rd S d|v r$|d D ]}t | |||d }t|V  qd|v rQ|d D ]&}t | |||d }|d drDt|V  q,|d }t|t|V  q,d S d S )Nr  r  r  r  r   
Properties)r  rX  r   rY  r\  )	r  r.   r   r   blobsbprO   r   propsre   re   rf   rR  ~  s"   rR  c                 C   sP   t |\}}}t|d||d}tj| ||fd}|d u r"|tdfS t||dS )Nr  r  r  rZ   r  rZ   r   )r   r   sas_token_managerr  rY  r   )r  rO   r.   r   r  r   rb   re   re   rf   get_url  s   ra  rU  c           
      C   s   t |\}}}i }|d ur||d< tt|d||ddtdd|dd}t| |}	|	jd	kr5td
| d|	jdkr<dS dd |	j D }t	||d< |d urT||d< tt|d||ddtdd|dd}t| |}	|	jd	krxtd
| d|	jdkS )Nr  r  r  r  r)  r  rM  rO  r_  zNo such file: 'r   r  Fc                 S   r  r  r  r   re   re   rf   r     r  zset_mtime.<locals>.<dictcomp>rR  r	  r   )
r   r   r   r   r  r  r  r   r   rw   )
r  rO   rU  rX  r.   r   r  r   r   r  re   re   rf   	set_mtime  s>   





rb  c                 C   sZ   t |\}}}t|}d|v r"d|dd d }t| |||S t| ||dd d S )Nr   r  rH   )r   r   r   rL   r  )r  rO   r.   r   r   re   re   rf   dirname  s   rc  c                    s  t |\t |\dtf fdd}d }d }d }t }|D ]{}	t |}
|
jdkrG|
jd }|
jd }|
jd } n^|
jtkrWtd| d	| d
|
jdkrv|
jd dkrntd| d	| d
td| d
|
jdkrt	
|
j}|d d dkrtjd|
j | |
dntd|
j t|	 q)t }|dkrtt| ttdddd}t |}
|
jd |krtd|
jd }|
jd }|dks|dkrtd| d
|rt |}|d ur|j|kr|jS d S )Nr%   c                     s   t dd} kr!tj fd}|d ur!t|| d\} }tt dddd| iddd	tfd
}t|tj fddS )Nr  r  r^  r_  r	  x-ms-copy-sourcerK  r_  r`  r  rh  )r   r`  r  r   r   r   r   r  )src_url	sas_tokenr[   r   r  dst_accountdst_blobdst_containersrc_accountsrc_blobsrc_containerre   rf   rk    s$   
zremote_copy.<locals>.build_reqrK  zx-ms-copy-idzx-ms-copy-statusr  :Source container or destination container not found: src='' dst='r   r_  r  r  zSource file not found: 'r`  r   r  PendingCopyOperationr  r  zunhandled status pendingr  r  ra  )r   r   zCopy id mismatchsuccesszInvalid copy status: ')r   r   r   exponential_sleep_generatorrm  r  r   r   r  r   r   r~   r   rn  r   r   sleepnextr   r  r   rX  rW  )r  r  r  r1  rk  copy_idcopy_statusr  backoff_generatorbackoffr  r   r   r   re   rg  rf   remote_copy  sn   










rz  ar   c                 C   sN   | ds	|d7 }t|\}}}t||}|dr |dd  }t| |||S )Nr   r   )r   r   r   r   r  )r  r{  r   r.   r   r   re   re   rf   
join_paths%  s   


r|  c           
         s   t |\}t |\}td|dtd|ddtf 	f
dd}t|}	|	jdkrFtd| d| d	|	jtkrVtd
| d| d	d S )Nr  r  r%   c               
      s   t jfd} | d u r}nt| d\}}tdtd dd|tj		 ddd	d
tfd}t|t	jfddS )Nr^  r_  r	  r"  r#  r'   r  )r  rd  zx-ms-source-ranger  r_  rO  rh  )
r`  r  r   r   r   r   r  r   r   r  )rf  copy_src_urlr[   r   
r/  r  rh  rj  rA  rT  rk  rm  re  r  re   rf   rk  :  s"   
z&_put_block_from_url.<locals>.build_reqr_  z?Source file/container or destination container not found: src='ro  r   rn  )r   r   r   r   rm  r  r  r   )
r  r  r  rT  r  r/  rl  ri  rk  r  re   r~  rf   _put_block_from_url0  s   $

r  c              
   C   sB  t | |}|d u rtd| dd }|jd urt|j}tdd}g }	|jt }
|
t	ks1J t
| j|
}d}d}g }||jk rot||}|t| ||t||j| ||}|| |	| |d7 }||7 }||jk sB|D ]}|  qqt|\}}}t|d||d}t| |||	||d |r|d urt|d	S d S )
NrI  r   r   r  r   r  r  r'  r   )r   r  rW  r?  rP  r   r  rT  r*  r=  r8  r  r  r>  r  r7  r  r   r   r   r  r@  rv   )r  r0  r  r  r1  r2  r   r  r  r  min_block_sizerD  r%  r  rE  r/  rF  rh  rj  ri  rA  re   re   rf   parallel_remote_copy_  sV   







	

r  azure_access	azure_sas)r   )NNr  )qr   r?  r   concurrent.futures
concurrentr   r  r   rR   r9  rI   r  r   urllib.parser   typingr   r   r   r   r   r   r   r	   urllib3blobfiler
   r   r   r   blobfile._commonr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r=  r*  r  rw   rg   r|   r   rZ  r   r   rY  r   r   r   r   r   r  r  r  r   r,  rJ  rP  r\  rx   ro  rx  r|  r  r  r  r  r  r  bytesr  r  r  r  r  r-  rE  ExecutorrG  r   r  rQ  rS  rX  rY  rR  ra  rb  rc  rz  r|  r  r  r  r`  re   re   re   rf   <module>   s   (L
S

$&*@	7


1

"

H +

"
9%
!" &2
(0$)

"$
#"
\
/

8