o
    ;ikZ                     @   s  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlmZmZ d dlmZmZ d dlmZmZmZmZmZmZmZmZ d dlmZ d dlmZ d d	lmZ d
dl m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m*Z* erddl+m,Z, dZ-dZ.dZ/dZ0dZ1dZ2dZ3e4e5d< e$dddd		dqddd ee6 d!ee6 d"e6fd#d$Z7e1dd%d&eeeef d'e4d(e4d)e8e6 d*e6d+e4d,ee d"dfd-d.Z9d/ed"e4fd0d1Z:d2e8e6 d'e4d"e;e6e<e4f fd3d4Z=	drd5e&d/ee>ef d,ee d"e;e6e<e4f fd6d7Z?de>d8ed"e;e6e<e4f fd9d:Z@de>d8ed"e6fd;d<ZAd/e>d=ed"eBe6ef fd>d?ZC			dsd@ed8ed,ee dAee6 dBee6 d"e6fdCdDZDe$dEdFdddGe6d"e>fdHdIZEdJe6d8ed"e>fdKdLZFdJe6d8ed"ee> fdMdNZGejHG dOdP dPZIdQeg ee	ef f dRedSedTe4d"eIf
dUdVZJ	drdWedSedTee4 d"eIfdXdYZKdZedSedTe4d"eIfd[d\ZLeg ee f ZMejHG d]d^ d^ZNejHG d_d` d`ZOdQeMdae4dbe jPd"e8eN fdcddZQdQeMdee4d"eNfdfdgZRdQeMfdhdiZSdQeMdje4dke4d"ee4 fdldmZTdne6d"e<fdodpZUdS )t    N)AsyncIterator)AbstractContextManagercontextmanager)BytesIOFileIO)PathPurePosixPath)TYPE_CHECKINGAnyBinaryIOCallableContextManagerOptionalUnioncast)urlparse)api_pb2)ModalClientModal   )ExecutionError   )TaskContextretry)UploadHashesget_upload_hashes)ClientSessionRegistry)loggerBytesIOSegmentPayloadi    i    i  @    i   i   @i   
BLOCK_SIZE   g333333?)
n_attempts
base_delaytimeoutapplication/octet-streampayloadr   content_md5_b64content_typereturnc           
   
      s  |   i }|rt| r||d< |r||d< t j| |||du r&dgng d4 I dH }|jdkrAtd t	dI dH  |jd	kriz	|
 I dH }W n tyZ   d
}Y nw td|  d|j d| |jd  }|dr{|dd }|d dkr|d dkr|dd }|}|  }	|	|krtd|	 d| d|W  d  I dH  W  d   S 1 I dH sw   Y  W d   dS 1 sw   Y  dS )zMReturns etag of s3 object which is a md5 hex checksum of the uploaded contentzContent-MD5zContent-TypeNcontent-type)dataheadersskip_auto_headers  HReceived SlowDown signal from S3, sleeping for 1 second before retrying.r      	<no body>zPut to url z failed with status : ETag)zW/zw/r   r   "z.Local data and remote data checksum mismatch (z vs ))reset_on_erroruse_md5r   get_sessionputstatusr   debugasynciosleeptext	Exceptionr   r,   strip
startswithmd5_checksum	hexdigest)

upload_urlr&   r'   r(   r,   respr?   etag
remote_md5local_md5_hex rJ   K/home/ubuntu/.local/lib/python3.10/site-packages/modal/_utils/blob_utils.py_upload_to_s3_url=   sL   




"rL   )upload_chunk_sizeprogress_report_cb	data_filecontent_lengthmax_part_size	part_urlscompletion_urlrM   rN   c                   s  ddl m} g }d}	|}
t| tr$|  fddtt|D }n| j  fddtt|D }tt	||ddD ]&\}\}}t
|
|}|||	|||d}|t||d d	 |
|8 }
|	|7 }	q=tj| I d H }d
g}t|dD ]\}}|d| d| d qt|d d|}dd |D }td| dt|  }t j||ddgdI d H }|jdkrz	| I d H }W n ty   d}Y nw td|j d| | I d H }||vrtd| d| d S )Nr   r   r   c                    s   g | ]}t  qS rJ   r   .0_)viewrJ   rK   
<listcomp>   s    z,perform_multipart_upload.<locals>.<listcomp>c                    s   g | ]}t  d qS )rbopenrU   filenamerJ   rK   rY          )start)segment_startsegment_length
chunk_sizerN   )r&   r(   z<CompleteMultipartUpload>z<Part>
<PartNumber>z</PartNumber>
<ETag>"z"</ETag>
</Part>z</CompleteMultipartUpload>
c                 S   s   g | ]}t |qS rJ   )bytesfromhex)rV   rG   rJ   rJ   rK   rY      r_       -asciir*   )r+   r-   r0   r1   z(Error when completing multipart upload: z,Hash mismatch on multipart upload assembly: z not in )bytes_io_segment_payloadr   
isinstancer   	getbufferrangelenname	enumeratezipminappendrL   r   gatherjoinhashlibmd5rD   r   r9   postencoder;   r?   r@   r   )rO   rP   rQ   rR   rS   rM   rN   r   upload_corosfile_offsetnum_bytes_leftdata_file_readerspart_numberdata_file_rdrpart_urlpart_length_bytespart_payload
part_etagscompletion_partsrG   completion_bodybin_hash_partsexpected_multipart_etagrF   msgresponse_bodyrJ   )r^   rX   rK   perform_multipart_uploadn   s\   





"
r   r+   c                 C   s0   |   }| dtj |   }| | || S Nr   )tellseekosSEEK_END)r+   posrP   rJ   rJ   rK   get_content_length   s
   
r   blob_idsc              	      s   d}d}t t| |D ]N\}\}}z/|dr3t }	||I dH  t |	 }
|d t|
d }n||I dH  |||fW   S  tyZ   |drOd}|t| d krX Y qw td)	zBTry uploading to each provider in order, with fallback on failure.r   Fz:r2Ni ʚ;r   TzFailed to upload blob)	rp   rq   endswithtimemonotonic_nsmaxr@   rn   r   )itemsr   callbackrP   r2_throughput_bytes_s	r2_failedidxitemblob_idt0dt_nsrJ   rJ   rK   _blob_upload_with_fallback   s(   

r   upload_hashesc                    s   t tr
tt tjjj d}||I d H }|	ddkrA fdd}t
|jj|j| dI d H \}}}	n&ddlm}
 |
d	 d
fdd}t
|jj|j| dI d H \}}}	rndd |||	fS )N)content_md5content_sha256_base64rP   upload_types_oneof
multipartsc              	      s$   t  | j| j| jtdI d H S )N)rP   rQ   rR   rS   rM   rN   )r   part_lengthupload_urlsrS   DEFAULT_SEGMENT_CHUNK_SIZE)part)rP   r+   rN   rJ   rK   upload_multipart_upload   s   z-_blob_upload.<locals>.upload_multipart_upload)rP   r   r   r   )ra   rb   rN   c                    s   t |  jdI d H S )N)r'   )rL   
md5_base64)url)r&   r   rJ   rK   upload_to_s3_url   s   z&_blob_upload.<locals>.upload_to_s3_urlT)complete)rk   re   r   r   r   BlobCreateRequestr   sha256_base64
BlobCreate
WhichOneofr   r   r   r   rj   r   r   )r   r+   stubrN   reqrF   r   r   r   r   r   r   rJ   )rP   r+   r&   rN   r   rK   _blob_upload   s@   


r   r   c           
   
      s   t | d d }td|dd t }t| tr&td | d} t| }t|| |I d H \}}}t	t | d}|| }	td|dd	|	dd
|dd|  |||fS )N   zUploading large blob of size .2fz MiBz8Blob uploading string, not bytes - auto-encoding as utf8utf8MbP?zUploaded large blob of size  MiB ( MiB/s, total zs). )
rn   r   r<   r   rk   strry   r   r   r   )
r&   r   size_mibr   r   r   r   r   dur_sthroughput_mib_srJ   rJ   rK    blob_upload_with_r2_failure_info  s   


 
r   c                    s   t | |I d H \}}}|S N)r   )r&   r   r   rW   rJ   rJ   rK   blob_upload$  s   r   api_stubc                    s*   t | tkrdt| |I d H iS d| iS )Ndata_blob_idr+   )rn   MAX_OBJECT_SIZE_BYTESr   )r+   r   rJ   rJ   rK   format_blob_data)  s   (r   file_obj
sha256_hexmd5_hexc                    s.   t | ||d}t|| ||I d H \}}}|S )N)r   r   )r   r   )r   r   rN   r   r   r   r   rW   rJ   rJ   rK   blob_upload_file-  s   r      g?download_urlc              	      s   t  | 4 I d H ;}|jdkrtd tdI d H  |jdkr6| I d H }t	d|j d| |
 I d H W  d   I d H  S 1 I d H sMw   Y  d S )Nr.   r/   r   r0    Get from url failed with status r2   )r   r9   getr;   r   r<   r=   r>   r?   r   read)r   s3_respr?   rJ   rJ   rK   _download_from_url9  s   


0r   r   c           	         s   t d|   t }tj| d}||I dH }t|jI dH }t|d d }t	t | d}|| }t d|  d|dd	|dd
|dd	 |S )zHConvenience function for reading all of the downloaded file into memory.zDownloading large blob r   Nr   r   zDownloaded large blob z	 of size r   r   r   zs))
r   r<   r   r   BlobGetRequestBlobGetr   r   rn   r   )	r   r   r   r   rF   r+   r   r   r   rJ   rJ   rK   blob_downloadG  s   "r   c              	   C  s   t j| d}||I d H }|j}t |4 I d H E}|jdkr0t	d t
dI d H  |jdkrG| I d H }td|j d| |j 2 z	3 d H W }|V  qL6 W d   I d H  d S 1 I d H shw   Y  d S )Nr   r.   r/   r   r0   r   r2   )r   r   r   r   r   r9   r   r;   r   r<   r=   r>   r?   r   contentiter_any)r   r   r   rF   r   r   r?   chunkrJ   rJ   rK   	blob_iterW  s   


.r   c                   @   s   e Zd ZU eg eeef f ed< eed< e	ed< e
ed< e	ed< e
ed< e
ed< eed< eed	< d
Zee ed< defddZd
S )FileUploadSpecsourcesource_descriptionsource_is_pathmount_filenameuse_blobr   r   modesizeNr   r)   c                 C   s>   |   }|d | W  d   S 1 sw   Y  dS )zRead content from source.r   N)r   r   r   )selffprJ   rJ   rK   read_contentw  s   

$zFileUploadSpec.read_content)__name__
__module____qualname__r   r   r   r   __annotations__r
   boolr   intr   r   re   r   rJ   rJ   rJ   rK   r   i  s   
 r   r   r   r   r   c           
      C   s   d }|  E}| dtj | }| d |tkr+|tkr dnd }d}t||d}	nd}|dk r?| d | }t|}	nt|}	W d    n1 sMw   Y  t| |t	|t
| ||	 |	 |d@ ||d
S )	Nr    baadbaadbaadbaadbaadbaadbaadbaadT)r   Fi     )
r   r   r   r   r   r   r   r   r   r   )r   r   r   r   LARGE_FILE_LIMITMULTIPART_UPLOAD_THRESHOLDr   r   r   rk   r   as_posixr   r   )
r   r   r   r   r   r   r   r   r   hashesrJ   rJ   rK   _get_file_upload_spec~  s:   


r   r^   c                    s:   |pt  jt dkrdnd@ }t fdd ||S )NWindowsr     c                      
   t  dS NrZ   r[   rJ   r]   rJ   rK   <lambda>  s   
 z0get_file_upload_spec_from_path.<locals>.<lambda>)r   statst_modeplatformsystemr   )r^   r   r   rJ   r]   rK   get_file_upload_spec_from_path  s   $
r   r   c                    s"   t  fdd}t|t ||S )Nc                   3   s      d  V  d S r   )r   rJ   r   rJ   rK   r     s   

z1get_file_upload_spec_from_fileobj.<locals>.source)r   r   r   )r   r   r   r   rJ   r   rK   !get_file_upload_spec_from_fileobj  s   r   c                   @   s&   e Zd ZU eed< eed< eed< dS )FileUploadBlockr`   endcontents_sha256N)r   r   r   r   r   re   rJ   rJ   rJ   rK   r     s   
 r   c                   @   s   e Zd ZU eed< eeef ed< eed< ee	 ed< e
ed< e
ed< e	dded	ed
ejdee
 dd f
ddZedeeef d	ed
ejde
dd f
ddZededeeef d	ede
d
ejdd fddZdS )FileUploadSpec2r   r   pathblocksr   r   Nr^   r   hash_semaphorer)   c                    sJ   |pt  jt dkrdnd@ } fdd}t| |||I d H S )Nr   r   r   c                      r   r   r[   rJ   r]   rJ   rK   r     s   
z)FileUploadSpec2.from_path.<locals>.source)r   r   r   r   r   r  _create)r^   r   r  r   r   rJ   r]   rK   	from_path  s   $	
zFileUploadSpec2.from_path	source_fpc                    sh   z|   fdd}W n ty%   tt| } |    fdd}Y nw t|t||||I d H S )Nc                     s$   t  } t | d}|d |S )NrZ   r   )r   dupfdopenr   )new_fdr   )filenorJ   rK   r     s   

z,FileUploadSpec2.from_fileobj.<locals>.sourcec                      s   t  S r   rT   rJ   )bufferrJ   rK   r   
  s   )r  OSErrorr   r   rl   r  r  r   )r  r   r  r   r   rJ   )r  r  rK   from_fileobj  s    

zFileUploadSpec2.from_fileobjc                    sl   |  }| dtj | }W d    n1 sw   Y  t| ||I d H }t| || ||d@ |dS )Nr   r   )r   r   r  r  r   r   )r   r   r   r   _gather_blocksr  r   )r   r   r   r   r  r  r   r  rJ   rJ   rK   r    s   	
zFileUploadSpec2._creater   )r   r   r   _FileUploadSource2r   r   r   r   listr   r   staticmethodr   r=   	Semaphorer   r  r   r   r  r  rJ   rJ   rJ   rK   r    s\   
 

r  r   r  c                    s`   dt dt dt fdd}||t}dt dtffdd  fd	d
t|D }tj| I d H S )Nabr)   c                 S   s   | |   S r   rJ   )r  r  rJ   rJ   rK   ceildiv3  s   z_gather_blocks.<locals>.ceildiv	block_idxc              	      sR    4 I d H  t t| I d H W  d   I d H  S 1 I d H s"w   Y  d S r   )r=   	to_thread_gather_block)r  )r  r   rJ   rK   gather_block8  s   0z$_gather_blocks.<locals>.gather_blockc                 3   s    | ]} |V  qd S r   rJ   )rV   r   )r  rJ   rK   	<genexpr><  s    z!_gather_blocks.<locals>.<genexpr>)r   r    r   rm   r=   rt   )r   r   r  r  
num_blockstasksrJ   )r  r  r   rK   r  .  s   
r  r  c                 C   s2   |t  }t| ||t  }t| ||}t|||dS )N)r`   r   r   )r    _find_end_of_block_hash_range_sha256r   )r   r  r`   r   r   rJ   rJ   rK   r  @  s   r  c                 C   s   t  }|| }|  C}|| d}||k r:||| }|s!n#|t|7 }|| ||k sW d    | S W d    | S W d    | S 1 sQw   Y  | S r   )rv   sha256r   r   rn   updatedigest)r   r`   r   sha256_hash
range_sizer   num_bytes_readr   rJ   rJ   rK   r  G  s,   





r  r`   r   c           	      C   s   || }|}|  G}| | d}||k r@||| }|sn)|d}|r.|| t| }|t|7 }||k sW d   |S W d   |S W d   |S 1 sSw   Y  |S )a_  Finds the appropriate end of a block, which is the index of the byte just past the last non-zero byte in the
    block.

    >>> _find_end_of_block(lambda: BytesIO(b"abc123   "), 0, 1024)
    6
    >>> _find_end_of_block(lambda: BytesIO(b"abc123   "), 3, 1024)
    6
    >>> _find_end_of_block(lambda: BytesIO(b"abc123   "), 0, 3)
    4
    >>> _find_end_of_block(lambda: BytesIO(b"abc123   a"), 0, 9)
    6
    >>> _find_end_of_block(lambda: BytesIO(b"   "), 0, 3)
    0
    >>> _find_end_of_block(lambda: BytesIO(b"      "), 3, 6)
    3
    >>> _find_end_of_block(lambda: BytesIO(b""), 0, 1024)
    0
    r       N)r   r   rstriprn   )	r   r`   r   r   new_endblock_fpr%  r   stripped_chunkrJ   rJ   rK   r  [  s0   





r  r   c                 C   sF   t | jdd }|ds|drdS |dv rdS td| )	zThis takes an upload URL in S3 and returns whether we should attach a checksum.

    It's only a workaround for missing functionality in moto.
    https://github.com/spulec/moto/issues/816
    :r   z.amazonaws.comz.r2.cloudflarestorage.comT)z	127.0.0.1	localhostz
172.21.0.1FzUnknown S3 host: )r   netlocsplitr   r@   )r   hostrJ   rJ   rK   r8     s   r8   )Nr%   r   )NNN)Vr=   dataclassesrv   r   r   r   collections.abcr   
contextlibr   r   ior   r   pathlibr   r   typingr	   r
   r   r   r   r   r   r   urllib.parser   modal_protor   modal_proto.modal_api_grpcr   	exceptionr   async_utilsr   r   
hash_utilsr   r   
http_utilsr   r   rj   r   r   MAX_ASYNC_OBJECT_SIZE_BYTESr   BLOB_MAX_PARALLELISMr   r   r    r   r   r   rL   r  r   r   tupler   r   re   r   r   r   dictr   r   r   r   r   	dataclassr   r   r   r   r  r   r  r  r  r  r  r  r8   rJ   rJ   rJ   rK   <module>   s  
(
7	
E	


 >

-
	Y
)