o
    dih                     @   s$  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZmZm Z m!Z!m"Z" d dl#m$Z$ d dl%m&Z& e'e(Z)ee*gdf Z+G dd dZ,ee,ef Z-dd Z.dd Z/dd Z0dd Z1eeed			d:de2de2de2de
ee2ef  de
e+ de
e fdd Z3	d;d!e2d"e2d#ee2e2f d$ee2e2f d%e*d&e-d'e j4d(e j5d)e
e j6 d*dfd+d,Z7eeed			d:de2de2d-e-de
ee2ef  de
e+ de
e fd.d/Z8eeed				d<d-e-de2de2de
ee2ef  de
e+ de
e d0ee9ge9f fd1d2Z:eeed			d:de2de2de2de
ee2ef  de
e+ de
e fd3d4Z;eeed				d<d5ee2ef de2de2de
ee2ef  de
e+ de
e fd6d7Z<d8d9 Z=dS )=    N)partial)BytesIO)OptionalCallableBinaryIODictAnyUnion)abstractmethod)with_current_context)ClientError)register_feature_id)utils)S3TransferConfig
S3Transfer)bucket_upload_filebucket_download_filebucket_copybucket_upload_fileobjbucket_download_fileobj)UploadSubmissionTask)CopySubmissionTaskc                   @   sF   e Zd ZeddededefddZedeeef defdd	Z	d
S )_AsyncBinaryIOr   offsetwhencereturnc                       d S N )selfr   r   r   r   F/home/ubuntu/.local/lib/python3.10/site-packages/aioboto3/s3/inject.pyseek      z_AsyncBinaryIO.seeksc                    r   r   r   )r   r#   r   r   r    write   r"   z_AsyncBinaryIO.writeN)r   )
__name__
__module____qualname__r
   intr!   r	   bytes	bytearrayr$   r   r   r   r    r      s
     r   c                 K   sJ   t | dt t | dt t | dt t | dt t | dt d S )Nupload_filedownload_filecopyupload_fileobjdownload_fileobj)r   inject_attributer+   r,   r-   r.   r/   class_attributeskwargsr   r   r    inject_s3_transfer_methods'   s   r4   c                 K   s   t | dt d S )Nload)r   r0   object_summary_loadr1   r   r   r    inject_object_summary_methods1   s   r7   c                 K   sX   t | dt t | dt t | dt t | dt t | dt t | dt d S )Nr5   r+   r,   r-   r.   r/   )r   r0   bucket_loadr   r   r   r   r   r1   r   r   r    inject_bucket_methods5   s   r9   c                    s@   | j jj| j| jdI d H }d|v r|d|d< || j _d S )NBucketKeyContentLengthSize)metaclienthead_objectbucket_namekeypopdata)r   argsr3   responser   r   r    r6   D   s   r6   S3_TRANSFERr;   r<   Filename	ExtraArgsCallbackConfigc              
      sd   t |d4 I dH }t| ||||||dI dH  W d  I dH  dS 1 I dH s+w   Y  dS )aR  Download an S3 object to a file asynchronously.

    Usage::

        import aioboto3

        async with aioboto3.resource('s3') as s3:
            await s3.meta.client.download_file('mybucket', 'hello.txt', '/tmp/hello.txt')

    Similar behaviour as S3Transfer's download_file() method,
    except that parameters are capitalised.
    wbNrJ   rK   rL   )aiofilesopenr/   )r   r;   r<   rI   rJ   rK   rL   fileobjr   r   r    r,   M   s   .r,   bucketrC   	extraArgsheadersstartfile	semaphore
write_lockio_queuer   c              
      s^  |4 I d H  | j d|||d d|I d H }|d  I d H }|
r/|
||fI d H  nCt|jrht|jrh|4 I d H  ||I d H  ||I d H  W d   I d H  n1 I d H sbw   Y  n
|| || |	rz|	t| W n   Y W d   I d H  d S W d   I d H  d S W d   I d H  d S 1 I d H sw   Y  d S )NRange)r;   r<   rZ   Bodyr   )
get_objectreadputinspectiscoroutinefunctionr!   r$   len)r   rR   rC   rS   rT   rU   rV   rW   rX   callbackrY   rG   contentr   r   r    _download_parto   s8   
(

.rd   Fileobjc                    s  |pt  }|p	i }z| jd||d|I dH }W n  ty: } z|jd d dkr5tddddid	 d}~ww t|j}	t }
|d
 |j d |j }d fdd}t	d}g t
 fdd}d}|szt| }zTg }t|D ]0}||j }t||j }dd| d|d  i}|t| ||||||	|
||snd qtj| I dH  |r|I dH  td| d|  W dS  ty } ztd| d| |d}~ww )a  Download an object from S3 to a file-like object.

    The file-like object must be in binary mode.

    This is a managed transfer which will perform a multipart download
    with asyncio if necessary.

    Usage::

        import aioboto3
        s3 = aioboto3.client('s3')

        async with aiofiles.open('filename', 'wb') as data:
            await s3.download_fileobj('mybucket', 'mykey', data)

    :type Fileobj: a file-like object
    :param Fileobj: A file-like object to download into. At a minimum, it must
        implement the `write` method and must accept bytes.

    :type Bucket: str
    :param Bucket: The name of the bucket to download from.

    :type Key: str
    :param Key: The name of the key to download from.

    :type ExtraArgs: dict
    :param ExtraArgs: Extra arguments that may be passed to the
        client operation.

    :type Callback: method
    :param Callback: A method which takes a number of bytes transferred to
        be periodically called during the download.

    :type Config: boto3.s3.transfer.TransferConfig
    :param Config: The transfer configuration to be used when performing the
        download.
    r:   NErrorCode	NoSuchKey404	Not Foundrg   Message
HeadObjectr=      r   c                    s,   | 7  rz  W d S    Y d S d S r   r   )bytes_transferred)rK   total_downloadedr   r    wrapper_callback   s   z*download_fileobj.<locals>.wrapper_callbackr!   c                     s   t  j} z\d}|k ra I dH  d}|sZd}g }tD ]'\}\}}||krK| r9 |I dH  n | || |t|7 }d}q$t|D ]}| qP|r|k sW dS W dS  t	j
yn   Y dS w )z
        Pretty much, get things off queue, add them to list
        Go through list, write things to file object in order
        r   NFT)r_   r`   r$   appendget	enumeratera   reversedrD   asyncioCancelledError)is_asyncwritten_posdone_nothingindexes_to_removeindexchunk_startrE   )re   io_listrY   
total_sizer   r    queue_reader   s4   

z&download_fileobj.<locals>.queue_readerrZ   bytes=-zDownloaded file from /zCouldn't download file from r   )r   rA   r   rG   rv   	Semaphoremax_request_concurrencyLockmultipart_chunksizehasattrQueueensure_futurerangeminrr   rd   gatherloggerdebug	Exception)r   r;   r<   re   rJ   rK   rL   head_responseerrrW   write_mutextotal_partsrq   is_seekabler   queue_reader_futuretasksirU   endrT   er   )rK   re   r~   rY   rp   r   r    r/      s^   
0 
	#

"
r/   
Processingc                    s  |pi dd   D dd   D }pt dtdtffdd	dtjk rH	jI d	H }	|	dkr=n|	7 tjk s0tjk r~rUjd d
I d	H  r|trvtI d	H  d	S t d	S j	d dI d	H }
|
d g 
dt
jjdt
 d	ddtf
fddd  	fdd}t
| }fddtdjD }z|I d	H  W n ty } z|  W Y d	}~nd	}~ww t
jt
 t
 ht
jdI d	H \}} st
kr)j dI d	H  nDz
jdd d jd d
id|I d	H  W n' tyl } z|zj dI d	H  W n   Y W Y d	}~nd	}~ww |D ]}| s|  z|I d	H  W qo   Y qoqog }|D ]}| s|  || q| }t d!| q|rt
j"|ddiI d	H D ]}t#|trt d!| qrԈd	S )!a  Upload a file-like object to S3.

    The file-like object must be in binary mode.

    This is a managed transfer which will perform a multipart upload in
    multiple threads if necessary.

    Usage::

        import aioboto3
        s3 = aioboto3.client('s3')

        async with aiofiles.open('filename', 'rb') as data:
            await s3.upload_fileobj(data, 'mybucket', 'mykey')

    :type Fileobj: a file-like object
    :param Fileobj: A file-like object to upload. At a minimum, it must
        implement the `read` method, and must return bytes.

    :type Bucket: str
    :param Bucket: The name of the bucket to upload to.

    :type Key: str
    :param Key: The name of the key to upload to.

    :type ExtraArgs: dict
    :param ExtraArgs: Extra arguments that may be passed to the
        client operation.

    :type Callback: method
    :param Callback: A method which takes a number of bytes transferred to
        be periodically called during the upload.

    :type Config: boto3.s3.transfer.TransferConfig
    :param Config: The transfer configuration to be used when performing the
        upload.

    :type Processing: method
    :param Processing: A method which takes a bytes buffer and convert it
        by custom logic.
    c                 S       i | ]\}}|t jv r||qS r   )r   UPLOAD_PART_ARGS.0kvr   r   r    
<dictcomp>`       z"upload_fileobj.<locals>.<dictcomp>c                 S   r   r   )r   COMPLETE_MULTIPART_ARGSr   r   r   r    r   a  r   	num_bytesr   c                    s8     | }t|r|I d H }|S tdI d H  |S )Ng        )r]   r_   isawaitablerv   sleep)r   rE   )re   r   r    fileobj_readd  s   


z$upload_fileobj.<locals>.fileobj_read    N)r;   r<   r[   r:   UploadIdr   )maxsizec               
      s4  d} sz	  I d H }W n tjy   Y | S w zjd
i |I d H }W n ty@ } z|  W Y d }~| S d }~ww i }dv rW|D ]}|drV|| ||< qI|d |d d| t|d }|7 | d7 } t	
d	  rzt r |I d H  n | W n   Y   r| S )Nr   ChecksumAlgorithmChecksumETag
PartNumberr   r   r[   rn   zUploaded part to S3r   )rs   rv   rw   upload_partr   set
startswithrr   ra   r   r   r_   r`   	task_done)uploaded_parts	part_argsrespr   finished_parts_kwargsrC   current_bytes)rK   	exceptionexception_eventfinished_partsrY   r3   r   
sent_bytesr   r    uploader  sT   (
	


,z upload_fileobj.<locals>.uploaderc               
      s,  d} d}s|s| d7 } t  }| dkr|7 }d}t|jk rcz
jI d H }W n tyH } z|  d}t  }W Y d }~nd }~ww |dkrT|dkrTd}n||7 }|d7 }t|jk s!|sk| dkrkd S rq|}	| | 
dI d H  td d7 s|r	d S d S d S d S )Nr   Frn   Tr   )r[   r;   r<   r   r   zAdded part to io_queue)	r*   ra   r   io_chunksizer   r   r^   r   r   )parteofmultipart_payloadloop_counterrE   r   )r;   rL   r<   r   r   r   expected_partsr   initial_datarY   	upload_idupload_part_argsr   r    file_reader  sL   


z#upload_fileobj.<locals>.file_readerc                    s   g | ]}t   qS r   )rv   r   )r   _)r   r   r    
<listcomp>  s    z"upload_fileobj.<locals>.<listcomp>)return_whenr;   r<   r   c                 S      | d S Nr   r   itemr   r   r    <lambda>      z upload_fileobj.<locals>.<lambda>rC   Partsr;   r<   r   MultipartUploadzFuture uploaded {0} partsreturn_exceptionsTr   )r   N)$itemsr   r(   r)   ra   multipart_threshold
put_objectr_   r`   create_multipart_uploadrv   r   max_io_queue_sizeEventr   r   r   r   r   waitcreate_taskjoinFIRST_COMPLETEDis_setabort_multipart_uploadsortcomplete_multipart_uploaddonecancelrr   resultr   r   formatr   
isinstance)r   re   r;   r<   rJ   rK   rL   r   complete_upload_argsnew_datar   r   file_reader_futurefuturesr   r   pendingcoro	cancelledfuturer   r   )r;   rK   rL   re   r<   r   r   r   r   r   r   r   rY   r3   r   r   r   r   r   r    r.   +  s   4


 $4/	

r.   c              
      sd   t |d4 I dH }t| ||||||dI dH  W d  I dH  dS 1 I dH s+w   Y  dS )a;  Upload a file to an S3 object.

    Usage::

        import aioboto3
        async with aioboto3.resource('s3') as s3:
            await s3.meta.client.upload_file('/tmp/hello.txt', 'mybucket', 'hello.txt')

    Similar behavior as S3Transfer's upload_file() method,
    except that parameters are capitalized.
    rbNrN   )rO   rP   r.   )r   rI   r;   r<   rJ   rK   rL   	open_filer   r   r    r+   =  s   .r+   
CopySourcec                    s   d|v sJ d|v sJ |p}|pt  }|pi }z+i }| D ]\}	}
|	tjv r1|
|tj|	 < q!|jd$|d |d d|I d H }W n  tye } z|jd d dkr`tdddd	id
 d }~ww |d |jk r~jd$|||d|I d H  d S dd | D }j	d$||d|I d H }g dt
|jdtdtttf f fdd}tt|d t|j }g }dd | D }|||||d d td|d D ]7}| }||d< |d |j }||j d }||kr|d d }d| d| |d< |||| | qz;t
j| I d H  t|ks*J djdd d d d | D }jd$|||d d!id"|I d H  W d S  ty } z!zj|||d d#I d H  W | tyz } z||d }~ww d }~ww )%Nr;   r<   r:   rf   rg   rh   ri   rj   rk   rm   r=   )r   r;   r<   c                 S   s    i | ]\}}|t jvr||qS r   )r   CREATE_MULTIPART_ARGS_BLACKLISTr   r   r   r    r     r   zcopy.<locals>.<dictcomp>r   sizer   c              	      s   4 I d H  j di |I d H }W d   I d H  n1 I d H s#w   Y  |d d |d d  rJz| 7   W d S    Y d S d S )NCopyPartResultr   r   r   r   )upload_part_copyrr   )r   r   upload_part_responserK   r   r   semr   r   r    r     s   (zcopy.<locals>.uploaderc                 S   r   r   )r   UPLOAD_PART_COPY_ARGSr   r   r   r    r     r   r   )r;   r<   r   r   rn   r   r   r   CopySourceRangez=Number of finished upload parts does not match expected partsc                 S   r   r   r   r   r   r   r    r     r   zcopy.<locals>.<lambda>r   c                 S   r   r   )r   r   r   r   r   r    r     r   r   r   r   r   )r   r   r   EXTRA_ARGS_TO_HEAD_ARGS_MAPPINGrA   r   rG   r   copy_objectr   rv   r   r   r(   r   strr   mathceilfloatr   updater   r-   rr   r   ra   r   r   r   r   )r   r   r;   r<   rJ   rK   SourceClientrL   head_object_kwargsparamvaluer   r   create_multipart_kwargscreate_multipart_upload_respr   	num_partsr   upload_kwargspart_numberpart_upload_kwargsrange_start	range_endr   err2r   r   r    r-   ^  s   

(&
r-   c              
      s   i | j _z"| j j I dH }|d D ]}|d | jkr$|| j _ W dS qW dS  tyG } z|jdi ddks< W Y d}~dS d}~ww )z]
    Calls s3.Client.list_buckets() to update the attributes of the Bucket
    resource.
    NBucketsNamerf   rg   AccessDenied)r?   rE   r@   list_bucketsnamer   rG   rs   )r   rF   r3   rG   bucket_datar   r   r   r    r8     s    r8   )NNN)NN)NNNN)>rv   rO   r_   loggingr  	functoolsr   ior   typingr   r   r   r   r   r	   abcr
   aiobotocore.contextr   botocore.exceptionsr   botocore.useragentr   boto3r   boto3.s3.transferr   r   boto3.s3.injectr   r   r   r   r   s3transfer.uploadr   s3transfer.copiesr   	getLoggerr%   r   r(   TransferCallbackr   AnyFileObjectr4   r7   r9   r6   r  r,   r   r   r   rd   r/   r)   r.   r+   r-   r8   r   r   r   r    <module>   s     


	"8
    
g