o
    پij                     @   s"  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( dZ)dZ*dZ+de,fd	d
Z-de.de.fddZ/de0de.de.fddZ1de0dee0ef de.fddZ2de0de0dee0 de"fddZ3de0de0de0de"fddZ4dee0ef fddZ5d ee0ef dee0 de"fd!d"Z6d#e0de0de0fd$d%Z7d&e"d'ee0e0f de"fd(d)Z8	*		dd+e0d,e0d-e9d.e0d/eee0e0f  d0eee0e0f  dee0ee9 f fd1d2Z:d3ed4e0de,fd5d6Z;d3ed4e0ddfd7d8Z<d4e0dee0e0f fd9d:Z=d+e0d;e0de0fd<d=Z>d>ee0ef dee0 fd?d@Z?dAe0de9fdBdCZ@dDee0ef de%fdEdFZAd3edGedeee9f fdHdIZBd3ed&e"ddJfdKdLZCG dMdN dNeZDG dOdP dPeZEd3ed4e0dee% fdQdRZFd3ed4e0de,fdSdTZGd3ed4e0dUe0dVe0de,f
dWdXZHd3ed4e0dYeIdZeId[e0de0fd\d]ZJd3ed+e0d,e0ddfd^d_ZKd3ed`ejLjMdae0d[e0dbe,dee0 fdcddZNd3edee0d.e0d/ee0e0f deee0ef  f
dfdgZOdd3ed4e0dhee0 dee fdidjZPd+e0dkee0ef dee fdldmZQd4e0defdndoZRd4e0dpe%defdqdrZSd3ed4e0dee0ee9 f fdsdtZTdd3ed4e0due9dvee0 de,f
dwdxZUd3ed4e0de0fdydzZVd3edae0d[e0dbe,dee0 f
d{d|ZWd3ed}e0d~e0de0fddZXe&eBdZYdS )    N)AnyDictIteratorListMappingOptionalTuple)_common)GCP_BASE_URLBaseStreamingReadFileBaseStreamingWriteFileConfigDirEntryErrorFileBodyRequestRequestFailure RestartableStreamingWriteFailureStatTokenManager	path_joinstrip_slashesi:	 oauth_token	anonymousreturnc                   C   s*   z	t dd W dS  t jy   Y dS w )Nzmetadata.google.internalP   FT)socketgetaddrinfogaierror r   r   A/home/ubuntu/.local/lib/python3.10/site-packages/blobfile/_gcp.py_is_gce_instance(   s   r!   sc                 C   s
   t | S N)base64urlsafe_b64encode)r"   r   r   r    
_b64encode0   s   
r&   private_keymsgc                 C   sH   ddl m} ddlm} ddlm} || }||}|||S )Nr   )SHA256)RSA)pkcs1_15)	Cryptodome.Hashr)   Cryptodome.PublicKeyr*   Cryptodome.Signaturer+   
import_keynewsign)r'   r(   r)   r*   r+   keyhr   r   r    _sign4   s   

r4   datac                 C   s\   t tdddd}t t|d}|d | }t t| |}|d | d | S )NRS256JWT)algtyputf8   .)r&   jsondumpsencoder4   )r'   r5   
header_b64body_b64to_signsignature_b64r   r   r    _create_jwt>   s
   rC   client_emailscopesc                 C   sT   t   }| d|d|d |d}dt||d}tdddd	itj|d
dS )N *https://www.googleapis.com/oauth2/v4/tokeni  )issscopeaudexpiatz+urn:ietf:params:oauth:grant-type:jwt-bearer)
grant_type	assertionPOSTContent-Type!application/x-www-form-urlencodedr:   urlmethodheadersr5   )timejoinrC   r   urllibparse	urlencoder>   )rD   r'   rE   now	claim_setr5   r   r   r    _create_token_requestF   s    r]   	client_idclient_secretrefresh_tokenc                 C   s0   d|| |d}t dddditj|ddS )	Nr`   )rM   r`   r^   r_   rG   rO   rP   rQ   r:   rR   )r   rX   rY   rZ   r>   )r^   r_   r`   r5   r   r   r    _refresh_access_token_request\   s   ra   c                  C   s   dt jv r1t jd } t j| std|  dt| }t|W  d    S 1 s,w   Y  t	 dkrBt j
t jd d}n
t j
t jd d}t j|rkt|}t|W  d    S 1 sfw   Y  i S )	NGOOGLE_APPLICATION_CREDENTIALSzcredentials not found at zC specified by environment variable 'GOOGLE_APPLICATION_CREDENTIALS'WindowsAPPDATAz+gcloud/application_default_credentials.jsonHOMEz3.config/gcloud/application_default_credentials.json)osenvironpathexistsr   openr<   loadplatformsystemrW   )
creds_pathfdefault_creds_pathr   r   r    _load_credentialsn   s*   



 


 rq   credsc                 C   sF   d| v rt | d | d |S d| v rt| d | d | d dS td)Nr'   rD   r`   r^   r_   )r`   r^   r_   zCredentials not recognized)r]   ra   r   rr   rE   r   r   r    _create_access_token_request   s   rt   templatec                 K   s   t jt| fi |S r#   )common	build_urlr
   )ru   r5   r   r   r    rw      s   rw   reqauthc              
   C   s   | j d u ri }nt| j  }| jd u ri }nt| j }|\}}|tkr.d| |d< n|tkr3ntd| d| j}|d urYt|trYt	
|d}d|vsUJ d|d< t| j| j|||| jt| jt| jdS )	NzBearer Authorizationzunrecognized auth kind ``r:   rP   zapplication/json)rT   rS   paramsrU   r5   preload_contentsuccess_codesretry_codes)rU   dictcopyr|   OAUTH_TOKEN	ANONYMOUSr   r5   
isinstancer<   r=   r>   r   rT   rS   r}   tupler~   r   )rx   ry   rU   r|   kindtokenr5   r   r   r    create_api_request   s6   

r   GETbucketname
expirationrT   r|   rU   c           $      C   sV  |d u ri }nt | }|d u ri }nt | }t }d|vr%td|tkr1tdt dtjj|dd}	d|  d|	 }
tjj	tj
jd}|d	}|d
}| d}|d  d| }d|d< d}t| }|D ]\}}t| }t| }|| d| d7 }qpg }|D ]\}}t| }|| qd|}d|d< ||d< ||d< t||d< ||d< g }t| }|D ]"\}}tjjt|dd}tjjt|dd}|| d|  qd|}d||
|||dg}t|  }dd|||g} tt|d | dd}!d}"|" |
 d| d|! }#|#|fS )Nr'   zPrivate key not found in credentials.  Please set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to point to a JSON key for a service account to use this callz Expiration can't be longer than z	 seconds. )safe/)tzz%Y%m%dT%H%M%SZz%Y%m%dz/auto/storage/goog4_requestrD   zstorage.googleapis.comhost:
;zGOOG4-RSA-SHA256zX-Goog-AlgorithmzX-Goog-CredentialzX-Goog-DatezX-Goog-ExpireszX-Goog-SignedHeaders=&zUNSIGNED-PAYLOADr:   zhttps://storage.googleapis.com?z&X-Goog-Signature=)r   r   rq   r   MAX_EXPIRATIONrX   rY   quotedatetimer[   timezoneutcstrftimesorteditemsstrlowerappendrW   hashlibsha256r>   	hexdigestbinasciihexlifyr4   decode)$r   r   r   rT   r|   rU   pr3   rr   escaped_object_namecanonical_uridatetime_nowrequest_timestamp	datestampcredential_scope
credentialcanonical_headersordered_headerskvlower_kstrip_vsigned_headers_parts_signed_headerscanonical_query_string_partsordered_params	encoded_k	encoded_vcanonical_query_stringcanonical_requestcanonical_request_hashstring_to_sign	signature	host_name
signed_urlr   r   r    generate_signed_url   s~   





r   confrh   c                 C   s   | ds	|d7 }t|\}}|dkr(ttd|dddd}t| |}|jdkS ttd	|ddt|dd
ddd}t| |}|jdkrEdS t|j	}d|v pRd|v S )Nr   r   z/storage/v1/b/{bucket}r   r        rS   rT   r~   r   /storage/v1/b/{bucket}/o1)prefix	delimiter
maxResultsrS   rT   r|   r~   r   Fr   prefixes)
endswith
split_pathr   rw   execute_api_requeststatusr   r<   loadsr5   )r   rh   r   blobrx   respresultr   r   r    isdir  s,   






r   c                 C   sf   | ds	|d7 }t|\}}ttd|ddtd|ddd}t| |}|jd	kr1td
| dd S )Nr   /upload/storage/v1/b/{bucket}/or   rO   media
uploadTyper   r     r   r   z4Unable to create directory, bucket does not exist: '')r   r   r   rw   r   r   r   r   r   rh   r   r   rx   r   r   r   r    	mkdirfile2  s   




r   c                 C   sZ   |  dstd|  d| tdd  } | d\}}}|dkr)td|  d||fS )Ngs://zInvalid path: 'r   r   r   )
startswithr   len	partition)rh   r   r   objr   r   r    r   A  s   
r   r   c                 C   s   d|  d| S )Nr   r   r   )r   r   r   r   r    combine_pathK  s   r   metadatac                 C   s>   d| v rt | d  S d| v rd| d v r| d d S d S )Nmd5Hashr   md5)r$   	b64decodehexr   r   r   r    get_md5O  s
   r   textc                 C   s   t j | d S )Nz%Y-%m-%dT%H:%M:%S.%f%z)r   strptime	timestamp)r   r   r   r    _parse_timestampZ  s   r   itemc                 C   s\   d| v rd| d v rt | d d }nt| d }tt| d |t| d t| | d dS )Nr   blobfile-mtimeupdatedsizetimeCreated
generation)r   mtimectimer   version)floatr   r   intr   )r   r   r   r   r    	make_stat^  s   

r  r2   c           	         sf  t jdddkrtdftdfS t }t  t dkr|dtf fdd	}t	
| |}t|j}|jd
krg|d }|dd}d| d| }|dkrc|dr[|d7 }t||dkrc|d7 }t||jdksnJ t|d f|t|d  fS t jdd dkrt rdtfdd	}t	
| |}t|j}t|d f|t|d  fS tdftdfS )N$BLOBFILE_FORCE_GOOGLE_ANONYMOUS_AUTH0r   r   infr   r   c                     s   t  dgd} d| _| S )Nz7https://www.googleapis.com/auth/devstorage.full_controlrs   r   )rt   r~   )rx   rr   r   r    	build_reqv  s
   z$_get_access_token.<locals>.build_reqr   errorerror_descriptionz<missing description>z Error with google credentials: [z] invalid_grantzInvalid JWT:z1
Please verify that your system clock is correct.zBad Requestz
Your credentials may be expired, please run the following commands: `gcloud auth application-default revoke` (this may fail but ignore the error) then `gcloud auth application-default login`r   access_token
expires_inNO_GCE_CHECKfalsetruec                   S   s   t ddddidS )Nr   zZhttp://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/tokenzMetadata-FlavorGoogle)rT   rS   rU   )r   r   r   r   r    r    s
   )rf   rg   getr   r   rV   rq   r   r   rv   execute_requestr<   r   r5   r   r   r   r   r   r!   )	r   r2   r[   r  r   r   r  descriptionr(   r   r  r    _get_access_tokenl  s6   

r  urllib3.BaseHTTPResponsec                    s    dt f fdd}t |S )Nr   c                      s   t tj dddS )Nr   )r2   )ry   )r   access_token_manager	get_tokenr   r   rx   r   r    r    s   z&execute_api_request.<locals>.build_req)r   rv   r  )r   rx   r  r   r  r    r     s   r   c                	       sT   e Zd Zdededee ddf fddZ	dded	ed
ee ddfddZ	  Z
S )StreamingReadFiler   rh   r   r   Nc                    sF   |d u rt ||}|d u rtd| d|j}t j|||d d S )NNo such file or bucket: 'r   )r   rh   r   )
maybe_statFileNotFoundErrorr   super__init__)selfr   rh   r   st	__class__r   r    r    s   
zStreamingReadFile.__init__	streamingstartendr  c              	   C   sN   t | j\}}ttd||ddtdddtj||did| d	}t| j|S )
Nz/storage/v1/b/{bucket}/o/{name})r   r   r   r   )altRanger#  r$  )   i  )rS   rT   r|   rU   r~   r}   )	r   _pathr   rw   r   rv   
calc_ranger   _conf)r  r"  r#  r$  r   r   rx   r   r   r    _request_chunk  s   
z StreamingReadFile._request_chunkr#   )__name__
__module____qualname__r   r   r   r   r  boolr,  __classcell__r   r   r   r    r    s    "	r  c                       sZ   e Zd Zdededdf fddZdededdfd	d
Zde	dededdfddZ
  ZS )StreamingWriteFiler   rh   r   Nc                    s   t |\}}ttd|ddt|ddd}t||}|jdv r'td| d	|jd
 | _|j	d dks6J t
 j||j	d d S )Nz4/upload/storage/v1/b/{bucket}/o?uploadType=resumabler   rO   )r   )r   r   r   rS   rT   r5   r~   )r   r   r  r   Locationi   r   )r   
chunk_size)r   r   rw   r   r   r   r  rU   _upload_urlgoogle_write_chunk_sizer  r  )r  r   rh   r   r   rx   r   r   r   r    r    s   


zStreamingWriteFile.__init__chunkfinalizec                 C   s   | j }t|dkr|r| ||| d S d}|t|k rG|| jj }|t|k}||| }| |||o5| |}|t|7 }|t|k sd S d S )Nr   )_offsetr   _upload_piecer+  r7  )r  r8  r9  offsetr#  r$  
last_piecepiecer   r   r    _upload_chunk  s   z StreamingWriteFile._upload_chunkr<  r>  c           
   
   C   s   |}|t | d }d}|r|t | }dd| d| d| d}t |dkr1|r1d	| |d
< t| j||d|r;dndd}z	t| j| W d S  tyh }	 z|	jdv rct|	j|	j	|	j|	j
|	jd d }	~	ww )N   *zapplication/octet-streamzbytes -r   )rP   Content-Ranger   zbytes */rC  PUT)r      )i4  )rS   r5   rU   rT   r~   )r   i  )messagerequest_stringresponse_statusr  r  )r   r   r6  r   r+  r   rH  r   rF  rG  r  r  )
r  r<  r>  r9  r#  r$  
total_sizerU   rx   er   r   r    r;    s>   

z StreamingWriteFile._upload_piece)r-  r.  r/  r   r   r  
memoryviewr0  r?  r   r;  r1  r   r   r   r    r2    s    "r2  c                 C   sX   t |\}}|dkrd S ttd||dddd}t| |}|jdkr$d S tt|jS )Nr   !/storage/v1/b/{bucket}/o/{object}r   objectr   r   r   r   )	r   r   rw   r   r   r  r<   r   r5   r   r   r   r    r    s   

r  c                 C   sP   t |\}}|dkrtd| dttd||dddd}t| |}|jd	kS )
Nr   z,The system cannot find the path specified: 'r   rL  rM  DELETE   r   r   rQ  )r   r  r   rw   r   r   r   r   r   r    remove  s   

rR  r   r   c                 C   sL   t |\}}ttd||ddt|dtd|iddd}t| |}|jd	kS )
NrL  rM  PATCHifGenerationMatchr   r   r   r   i  rS   rT   r|   r5   r~   r   )r   r   rw   r   r   r   )r   rh   r   r   r   r   rx   r   r   r   r    maybe_update_md5"  s   
	
rX  r#  r   dstc           
   	   C   sZ   t |\}}ttd|ddtd|dt|||| ddd}t| |}t|j}	|	d	 S )
Nr   r   rO   r   r   r'  r   rW  r   )	r   r   rw   r   r   r   r<   r   r5   )
r   rh   r#  r   rY  r   r   rx   r   r   r   r   r    _upload_part1  s   


r[  c                 C   s&   t td||dddd}t| | d S )NrL  rM  rO  rP  r   )r   rw   r   )r   r   r   rx   r   r   r    _delete_part?  s   r\  executorsrc
return_md5c              
   C   s  t |d}t|}W d    n1 sw   Y  t|d}t|}|jdkrGt	| |}W d    n1 s<w   Y  |rE|S d S t
|\}	}
g }g }t|dt pYd}tt|j| tj}d}d}g }||jk rd| }|t| ||t||j| || }|| ||
|  |d7 }||7 }||jk srt||D ]\}}| }|||d|id qttd	|	|
d
dd|idd}t| |}t|j}t| ||d | g }|D ]}|t| |	|}|| q|D ]}|  q|r|S d S )Nrbr:   r   _max_workersr@  z.part.rU  )r   r   objectPreconditionsz?/storage/v1/b/{destinationBucket}/o/{destinationObject}/compose)destinationBucketdestinationObjectrO   sourceObjectsrZ  r3  r   )rj   rv   	block_md5r   r   r   rf   statst_sizer2  r   getattr	cpu_countmaxmathceilPARALLEL_COPY_MINIMUM_PART_SIZEsubmitr[  minr   zipr   r   rw   r   r<   r   r5   rX  r\  )r   r]  r^  rY  r_  ro   
md5_digestr   r"   	dstbucketdstnamesource_objectsobject_namesmax_workers	part_sizeir#  futuressuffixfuturer   r   rx   r   r   delete_futuresr   r   r    parallel_uploadH  sp   





	


r~  rS   c                 c   sd    t | }	 t|||dd}t| |}|jdkrd S t|j}|V  d|vr+d S |d |d< q)NTr   r   r   nextPageToken	pageToken)r   r   r   r   r   r<   r   r5   )r   rS   rT   r|   r   rx   r   r   r   r   r    _create_page_iterator  s   

r  r   c                 c   sf    i }|d ur||d< t |\}}t| td|ddtdd|i|d}|D ]
}t||E d H  q&d S )Nr   r   r   r   r   )r   rS   rT   r|   r   )r   r  rw   r   _get_entries)r   rh   r   r|   r   r   itr   r   r   r    
list_blobs  s   
r  r   c                 c   s    d|v r|d D ]}t | |}t|V  q	d|v r=|d D ] }t | |d }|d dr4t|V  qt|t|V  qd S d S )Nr   r   r   r   )r   entry_from_dirpathr   entry_from_path_statr  )r   r   r   rh   r   r   r   r    r    s   
r  c                 C   sB   |  dr| d d } t| \}}|dd }t|| ddd dS )Nr   TFr   rh   is_diris_filerg  r   r   splitr   )rh   r   r   r   r   r   r    r    s
   
r  rg  c                 C   s:   |  drJ t| \}}|dd }t|| dd|dS )Nr   r  FTr  r  )rh   rg  r   r   r   r   r   r    r    s   r  c                 C   s   t |\}}t||tdS )N)r   )r   r   r   )r   rh   r   r   r   r   r    get_url  s   r  r   r   c           	      C   sz   t |\}}d }|d urt|d}ttd||dd|tdt|iddd}t| |}|jd	kr8td
| d|jdkS )NrT  rL  rM  rS  r   r   rV  rW  r   zNo such file: 'r   r   )r   r   r   rw   r   r   r   r  )	r   rh   r   r   r   r   r|   rx   r   r   r   r    	set_mtime  s   



r  c                 C   sP   t |\}}t|}d|v rd|dd d }t||S t|dd d S )Nr   r  r   )r   r   rW   r  r   )r   rh   r   r   r   r   r    dirname  s   
r  c                 C   s   t |\}}t |\}}i }	 ttd||||dd|dd}	t| |	}
|
jdkr0td| d	t|
j}|d
 rD|rBt	|d S d S |d |d< q)NTzc/storage/v1/b/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject})sourceBucketsourceObjectrc  rd  rO   r   r   r   zSource file not found: 'r   doneresourcerewriteToken)
r   r   rw   r   r   r  r<   r   r5   r   )r   r^  rY  r_  	srcbucketsrcnamers  rt  r|   rx   r   r   r   r   r    remote_copy  s4   

r  abc                 C   sH   | ds	|d7 }t|\}}t||}|dr|dd  }t||S )Nr   r@  )r   r   r   r   r   )r   r  r  r   r   r   r   r    
join_paths  s   



r  gcp)r   NNr#   )Zr$   r   concurrent.futures
concurrentr   r   r<   rl  rf   rl   r   rV   urllib.parserX   typingr   r   r   r   r   r   r   urllib3blobfiler	   rv   blobfile._commonr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r0  r!   bytesr&   r   r4   rC   r]   ra   rq   rt   rw   r   r   r   r   r   r   r   r   r   r  r  r   r  r2  r  rR  rX  r   r[  r\  rz  Executorr~  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r    <module>   s    $@

"(
[
/J"	
C

$""$"
